From 297fead1a0cd997b0e3e6e55f77fbe83b376be1a Mon Sep 17 00:00:00 2001 From: Ravi Kumar L Date: Wed, 26 Aug 2026 02:05:52 +0200 Subject: [PATCH 001/116] =?UTF-8?q?=F0=9F=A9=B9=20fix(codeapi):=20add=20se?= =?UTF-8?q?ssion=20cache=20recovery=20utility=20(#60)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(codeapi): add session cache recovery utility * fix(codeapi): accept emitted session keys --- service/Dockerfile | 3 + service/Dockerfile.api | 3 + service/scripts/rehydrate-session-cache.ts | 486 ++++++++++++++++++++ service/src/rehydrate-session-cache.test.ts | 244 ++++++++++ 4 files changed, 736 insertions(+) create mode 100644 service/scripts/rehydrate-session-cache.ts create mode 100644 service/src/rehydrate-session-cache.test.ts diff --git a/service/Dockerfile b/service/Dockerfile index 8ae43ef4..762790dc 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -16,10 +16,12 @@ RUN cd /temp/prod && bun install --frozen-lockfile --production FROM base AS builder COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src +COPY service/scripts ./scripts COPY shared /shared COPY service/tsconfig.json ./ RUN bun build ./src/file-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' RUN bun build ./src/api-server.ts --minify --outdir .build-api --target bun --external '@opentelemetry/*' +RUN bun build ./scripts/rehydrate-session-cache.ts --minify --outdir .build-migrations --target bun --external '@opentelemetry/*' RUN bun build ./src/worker-server.ts --minify --outdir .build-worker --target bun --external '@opentelemetry/*' RUN bun build ./src/egress-gateway.ts --minify --outdir .build-egress-gateway --target bun --external '@opentelemetry/*' @@ -37,6 +39,7 @@ ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules COPY --from=builder /app/.build-api ./.build-api +COPY --from=builder /app/.build-migrations ./.build-migrations COPY --from=builder /app/src/*.py ./src/ CMD ["bun", "run", ".build-api/api-server.js"] diff --git a/service/Dockerfile.api b/service/Dockerfile.api index 0e0d1a9c..419bdc95 100644 --- a/service/Dockerfile.api +++ b/service/Dockerfile.api @@ -18,9 +18,11 @@ RUN cd /temp/prod && bun install --frozen-lockfile --production FROM base AS builder COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src +COPY service/scripts ./scripts COPY shared /shared COPY service/tsconfig.json ./ RUN bun build ./src/api-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' +RUN bun build ./scripts/rehydrate-session-cache.ts --minify --outdir .build-migrations --target bun --external '@opentelemetry/*' # Production stage FROM oven/bun:1.3.14 AS production @@ -30,6 +32,7 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* COPY --from=install /temp/prod/node_modules ./node_modules COPY --from=builder /app/.build ./.build +COPY --from=builder /app/.build-migrations ./.build-migrations COPY --from=builder /app/src ./src # Copy matplotlib templates to /app where __dirname points at runtime COPY --from=builder /app/src/matplotlib.py /app/src/matplotlib-async.py ./ diff --git a/service/scripts/rehydrate-session-cache.ts b/service/scripts/rehydrate-session-cache.ts new file mode 100644 index 00000000..c21afa92 --- /dev/null +++ b/service/scripts/rehydrate-session-cache.ts @@ -0,0 +1,486 @@ +import IORedis from 'ioredis'; +import type * as tls from 'tls'; +import { isValidId } from '../src/utils'; +import { redisKeepAliveOptions } from '../src/redis-options'; + +/** + * One-time recovery for session ownership keys that expired before the + * associated files were removed. + * + * Build the input from a trusted source that provides the exact session id and + * expected session key pairs. Keep recovery manifests outside this repository + * because session keys can contain tenant and user identifiers. + * + * Pipe newline-delimited JSON to this script. Run it without `--apply` first: + * + * {"type":"source","environment":"example","region":"region-1","namespace":"codeapi","query_start_utc":"2026-01-01T00:00:00Z","query_end_utc":"2026-01-02T00:00:00Z"} + * {"session_id":"<21-character id>","expected_session_key":""} + * + * The apply path uses SET NX and never replaces an existing owner. + */ + +const DEFAULT_SESSION_CACHE_TTL_SECONDS = 86400; +const MAX_RECOVERY_CONTEXT_LENGTH = 128; +const MAX_RECOVERY_SESSION_KEY_LENGTH = 512; +const MAX_RECONNECT_ATTEMPTS = 5; +const RECONNECT_DELAY_MS = 2000; + +export interface RecoverySource { + type: 'source'; + environment: string; + region: string; + namespace: string; + query_start_utc: string; + query_end_utc: string; +} + +export interface RecoveryRecord { + session_id: string; + expected_session_key: string; +} + +export interface RecoveryStore { + get(key: string): Promise; + set( + key: string, + value: string, + expiryMode: 'EX', + ttlSeconds: number, + condition: 'NX', + ): Promise<'OK' | null>; +} + +export interface RecoverySummary { + input: number; + missing: number; + restored: number; + matching: number; + conflicts: number; +} + +export class RecoveryInterruptedError extends Error { + readonly summary: RecoverySummary; + + constructor(error: unknown, summary: RecoverySummary) { + super(error instanceof Error ? error.message : String(error)); + this.name = 'RecoveryInterruptedError'; + this.summary = { ...summary }; + } +} + +interface RecoveryScope { + environment: string; + region: string; + namespace: string; +} + +interface Options extends RecoveryScope { + apply: boolean; + inputPath?: string; + ttlSeconds: number; +} + +function usage(): string { + return `Usage: bun run /app/.build-migrations/rehydrate-session-cache.js [options] + +Restores missing Redis session ownership keys so retained files can be cleaned +up normally. Input is JSONL on stdin by default. The first record must be a +source header whose environment, region, and namespace match the configured +recovery scope. + +Options: + --apply Write missing keys. Without this flag, only inspect. + --input Read JSONL from a file instead of stdin. + --environment Expected source environment. Defaults to + SESSION_RECOVERY_ENVIRONMENT. + --region Expected source region. Defaults to + SESSION_RECOVERY_REGION. + --namespace Expected source namespace. Defaults to + SESSION_RECOVERY_NAMESPACE. + --ttl-seconds Redis TTL for restored keys. Defaults to + SESSION_CACHE_TTL or ${DEFAULT_SESSION_CACHE_TTL_SECONDS}. + --help Show this help. + +Keep recovery manifests outside the repository because expected_session_key +values can contain tenant and user identifiers. Add --apply only after +reviewing the dry-run summary. + +Exit codes: + 0 Completed without conflicts (dry-run missing keys are expected). + 1 Invalid input, configuration error, or interrupted Redis operation. + 2 Apply left a key missing or found an ownership conflict.`; +} + +function parsePositiveInteger(raw: string, name: string): number { + if (!/^\d+$/.test(raw)) { + throw new Error(`${name} must be a positive integer`); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + +function parseRecoveryContext(raw: string | undefined, name: string): string { + const value = raw?.trim(); + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + if ( + value.length > MAX_RECOVERY_CONTEXT_LENGTH + || !/^[A-Za-z0-9][A-Za-z0-9_.:/-]*$/.test(value) + ) { + throw new Error( + `${name} must be ${MAX_RECOVERY_CONTEXT_LENGTH} or fewer safe characters`, + ); + } + return value; +} + +function optionValue(args: string[], index: number, name: string): string { + if (index + 1 >= args.length) { + throw new Error(`${name} requires a value`); + } + const value = args[index + 1]; + if (value === '' || value.startsWith('--')) { + throw new Error(`${name} requires a value`); + } + return value; +} + +export function parseOptions(args: string[], env: NodeJS.ProcessEnv = process.env): Options { + const configuredTtl = env.SESSION_CACHE_TTL?.trim(); + let ttlSeconds = configuredTtl != null && configuredTtl !== '' + ? parsePositiveInteger(configuredTtl, 'SESSION_CACHE_TTL') + : DEFAULT_SESSION_CACHE_TTL_SECONDS; + let environment = env.SESSION_RECOVERY_ENVIRONMENT; + let region = env.SESSION_RECOVERY_REGION; + let namespace = env.SESSION_RECOVERY_NAMESPACE; + let apply = false; + let inputPath: string | undefined; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + switch (arg) { + case '--apply': + apply = true; + break; + case '--input': + inputPath = optionValue(args, index, '--input'); + index += 1; + break; + case '--environment': + environment = optionValue(args, index, '--environment'); + index += 1; + break; + case '--region': + region = optionValue(args, index, '--region'); + index += 1; + break; + case '--namespace': + namespace = optionValue(args, index, '--namespace'); + index += 1; + break; + case '--ttl-seconds': + ttlSeconds = parsePositiveInteger( + optionValue(args, index, '--ttl-seconds'), + '--ttl-seconds', + ); + index += 1; + break; + case '--help': + break; + default: + throw new Error(`Unknown option: ${arg}`); + } + } + + return { + apply, + inputPath, + environment: parseRecoveryContext(environment, 'Recovery environment'), + region: parseRecoveryContext(region, 'Recovery region'), + namespace: parseRecoveryContext(namespace, 'Recovery namespace'), + ttlSeconds, + }; +} + +function isUtcTimestamp(value: unknown): value is string { + return typeof value === 'string' + && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(value) + && Number.isFinite(Date.parse(value)); +} + +function parseRecoverySource(value: unknown, expected: RecoveryScope): RecoverySource { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('The first input record must be a recovery source header'); + } + const source = value as Record; + let environment: string; + let region: string; + let namespace: string; + try { + environment = parseRecoveryContext( + typeof source.environment === 'string' ? source.environment : undefined, + 'Source environment', + ); + region = parseRecoveryContext( + typeof source.region === 'string' ? source.region : undefined, + 'Source region', + ); + namespace = parseRecoveryContext( + typeof source.namespace === 'string' ? source.namespace : undefined, + 'Source namespace', + ); + } catch { + throw new Error('Recovery source contains invalid environment, region, or namespace'); + } + if ( + source.type !== 'source' + || environment !== expected.environment + || region !== expected.region + || namespace !== expected.namespace + || !isUtcTimestamp(source.query_start_utc) + || !isUtcTimestamp(source.query_end_utc) + ) { + throw new Error( + `Recovery source must match ${expected.environment}/${expected.region}/${expected.namespace}`, + ); + } + if (Date.parse(source.query_start_utc) >= Date.parse(source.query_end_utc)) { + throw new Error('Recovery source query_start_utc must be before query_end_utc'); + } + return { + type: 'source', + environment, + region, + namespace, + query_start_utc: source.query_start_utc, + query_end_utc: source.query_end_utc, + }; +} + +function isRecoveryRecord(value: unknown): value is RecoveryRecord { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const record = value as Record; + return typeof record.session_id === 'string' + && isValidId(record.session_id) + && typeof record.expected_session_key === 'string' + && record.expected_session_key.length > 0 + && record.expected_session_key.length <= MAX_RECOVERY_SESSION_KEY_LENGTH + && !/\p{Cc}/u.test(record.expected_session_key); +} + +export function parseRecoveryManifest( + input: string, + expected: RecoveryScope, +): { source: RecoverySource; records: RecoveryRecord[] } { + const bySessionId = new Map(); + const lines = input.split(/\r?\n/); + let source: RecoverySource | undefined; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index].trim(); + if (!line || line.startsWith('#')) { + continue; + } + + let value: unknown; + try { + value = JSON.parse(line); + } catch { + throw new Error(`Input line ${index + 1} is not valid JSON`); + } + if (!source) { + source = parseRecoverySource(value, expected); + continue; + } + if (!isRecoveryRecord(value)) { + throw new Error( + `Input line ${index + 1} must contain a valid session_id and expected_session_key`, + ); + } + + const previous = bySessionId.get(value.session_id); + if (previous && previous.expected_session_key !== value.expected_session_key) { + throw new Error(`Input contains conflicting owners for session ${value.session_id}`); + } + bySessionId.set(value.session_id, value); + } + + if (bySessionId.size === 0) { + throw new Error('Input contains no recovery records'); + } + if (!source) { + throw new Error('Input contains no recovery source header'); + } + return { source, records: [...bySessionId.values()] }; +} + +export async function recoverSessionCache( + records: RecoveryRecord[], + store: RecoveryStore, + options: Pick, +): Promise { + const summary: RecoverySummary = { + input: records.length, + missing: 0, + restored: 0, + matching: 0, + conflicts: 0, + }; + + for (const record of records) { + try { + const redisKey = `session:${record.session_id}`; + const current = await store.get(redisKey); + if (current === record.expected_session_key) { + summary.matching += 1; + continue; + } + if (current !== null) { + summary.conflicts += 1; + // eslint-disable-next-line no-console + console.error(`Conflict: ${record.session_id} already has a different owner`); + continue; + } + + if (!options.apply) { + summary.missing += 1; + continue; + } + + const result = await store.set( + redisKey, + record.expected_session_key, + 'EX', + options.ttlSeconds, + 'NX', + ); + if (result === 'OK') { + summary.restored += 1; + continue; + } + + const racedValue = await store.get(redisKey); + if (racedValue === record.expected_session_key) { + summary.matching += 1; + } else if (racedValue === null) { + summary.missing += 1; + // eslint-disable-next-line no-console + console.error(`Missing: ${record.session_id} disappeared during recovery`); + } else { + summary.conflicts += 1; + // eslint-disable-next-line no-console + console.error(`Conflict: ${record.session_id} changed during recovery`); + } + } catch (error) { + throw new RecoveryInterruptedError(error, summary); + } + } + + return summary; +} + +function redisRetryStrategy(times: number): number | null { + return times > MAX_RECONNECT_ATTEMPTS ? null : RECONNECT_DELAY_MS; +} + +function createRedisClient(): IORedis { + const options = { + host: process.env.REDIS_HOST ?? 'redis', + port: Number(process.env.REDIS_PORT) || 6379, + password: process.env.REDIS_PASSWORD, + maxRetriesPerRequest: 1, + retryStrategy: redisRetryStrategy, + enableReadyCheck: true, + connectTimeout: 10000, + disconnectTimeout: 2000, + ...redisKeepAliveOptions(), + tls: process.env.REDIS_TLS === 'true' + ? { rejectUnauthorized: false } as tls.ConnectionOptions + : undefined, + ...(process.env.REDIS_USE_ALTERNATIVE_DNS_LOOKUP === 'true' + ? { + dnsLookup: ( + address: string, + callback: (err: Error | null, addr: string) => void, + ): void => callback(null, address), + } + : {}), + }; + return new IORedis(options); +} + +async function readInput(inputPath?: string): Promise { + if (inputPath != null && inputPath !== '') { + return Bun.file(inputPath).text(); + } + if (process.stdin.isTTY === true) { + throw new Error('No manifest input: pipe JSONL on stdin or use --input '); + } + return Bun.stdin.text(); +} + +function recoveryOutput( + options: Options, + source: RecoverySource, + summary: RecoverySummary, +): Record { + return { + mode: options.apply ? 'apply' : 'dry-run', + environment: source.environment, + region: source.region, + namespace: source.namespace, + ttlSeconds: options.ttlSeconds, + ...summary, + }; +} + +export async function main(args: string[] = process.argv.slice(2)): Promise { + if (args.includes('--help')) { + // eslint-disable-next-line no-console + console.log(usage()); + return 0; + } + + let client: IORedis | undefined; + let options: Options | undefined; + let source: RecoverySource | undefined; + try { + options = parseOptions(args); + const manifest = parseRecoveryManifest(await readInput(options.inputPath), options); + source = manifest.source; + client = createRedisClient(); + const summary = await recoverSessionCache(manifest.records, client, options); + // eslint-disable-next-line no-console + console.log(JSON.stringify(recoveryOutput(options, source, summary))); + const incompleteApply = options.apply && summary.missing > 0; + return summary.conflicts === 0 && !incompleteApply ? 0 : 2; + } catch (error) { + if ( + error instanceof RecoveryInterruptedError + && options != null + && source != null + ) { + // eslint-disable-next-line no-console + console.log(JSON.stringify(recoveryOutput(options, source, error.summary))); + } + // eslint-disable-next-line no-console + console.error(error instanceof Error ? error.message : String(error)); + return 1; + } finally { + if (client != null) { + await client.quit().catch(() => client?.disconnect()); + } + } +} + +if (require.main === module) { + void main().then((exitCode) => { + process.exitCode = exitCode; + }); +} diff --git a/service/src/rehydrate-session-cache.test.ts b/service/src/rehydrate-session-cache.test.ts new file mode 100644 index 00000000..22e391f8 --- /dev/null +++ b/service/src/rehydrate-session-cache.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it } from 'bun:test'; +import { + parseOptions, + parseRecoveryManifest, + recoverSessionCache, + RecoveryInterruptedError, + type RecoverySource, + type RecoveryStore, +} from '../scripts/rehydrate-session-cache'; + +const SESSION_ID = 'ABCDEFGHIJKLMNOPQRSTU'; +const OTHER_SESSION_ID = 'ZYXWVUTSRQPONMLKJIHGF'; +const SESSION_KEY = 'example-tenant:user:example-user'; +const SCOPE = { + environment: 'example', + region: 'region-1', + namespace: 'codeapi', +}; +const SOURCE = { + type: 'source', + ...SCOPE, + query_start_utc: '2026-01-01T00:00:00Z', + query_end_utc: '2026-01-02T00:00:00Z', +} as const satisfies RecoverySource; + +class MemoryStore implements RecoveryStore { + readonly values = new Map(); + + async get(key: string): Promise { + return this.values.get(key) ?? null; + } + + async set( + key: string, + value: string, + _expiryMode: 'EX', + _ttlSeconds: number, + _condition: 'NX', + ): Promise<'OK' | null> { + if (this.values.has(key)) { + return null; + } + this.values.set(key, value); + return 'OK'; + } +} + +describe('rehydrate-session-cache', () => { + it('parses, validates, and deduplicates JSONL records', () => { + const input = [ + '# trusted recovery source', + JSON.stringify(SOURCE), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: SESSION_KEY }), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: SESSION_KEY }), + '', + ].join('\n'); + + expect(parseRecoveryManifest(input, SCOPE)).toEqual({ + source: SOURCE, + records: [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + }); + }); + + it('rejects a manifest outside the configured recovery scope', () => { + const input = [ + JSON.stringify(SOURCE), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: SESSION_KEY }), + ].join('\n'); + + expect(() => parseRecoveryManifest(input, { ...SCOPE, region: 'region-2' })).toThrow( + 'Recovery source must match example/region-2/codeapi', + ); + }); + + it('rejects invalid recovery context values', () => { + expect(() => parseOptions([ + '--environment', 'example', + '--region', 'region with spaces', + '--namespace', 'codeapi', + ], {})).toThrow('Recovery region'); + }); + + it('rejects conflicting owners before connecting to Redis', () => { + const input = [ + JSON.stringify(SOURCE), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: SESSION_KEY }), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: 'tenant-id:user:other' }), + ].join('\n'); + + expect(() => parseRecoveryManifest(input, SCOPE)).toThrow('conflicting owners'); + }); + + it('accepts a composite session key longer than one resource id', () => { + const longSessionKey = `${'t'.repeat(128)}:skill:${'s'.repeat(128)}:v:1`; + const input = [ + JSON.stringify(SOURCE), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: longSessionKey }), + ].join('\n'); + + expect(parseRecoveryManifest(input, SCOPE).records).toEqual([ + { session_id: SESSION_ID, expected_session_key: longSessionKey }, + ]); + }); + + it('accepts session keys containing identity punctuation and Unicode', () => { + const emittedSessionKey = 'tenant+東京@example.com/user:user+東京@example.com'; + const input = [ + JSON.stringify(SOURCE), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: emittedSessionKey }), + ].join('\n'); + + expect(parseRecoveryManifest(input, SCOPE).records).toEqual([ + { session_id: SESSION_ID, expected_session_key: emittedSessionKey }, + ]); + }); + + it('rejects empty session keys and control characters', () => { + for (const expectedSessionKey of ['', 'tenant:user:user\nid']) { + const input = [ + JSON.stringify(SOURCE), + JSON.stringify({ session_id: SESSION_ID, expected_session_key: expectedSessionKey }), + ].join('\n'); + + expect(() => parseRecoveryManifest(input, SCOPE)).toThrow( + 'must contain a valid session_id and expected_session_key', + ); + } + }); + + it('accepts recovery scope from arguments or configuration', () => { + expect(parseOptions([ + '--environment', 'argument-env', + '--region', 'argument-region', + '--namespace', 'argument-namespace', + '--ttl-seconds', '7200', + ], {})).toMatchObject({ + environment: 'argument-env', + region: 'argument-region', + namespace: 'argument-namespace', + ttlSeconds: 7200, + }); + expect(parseOptions([], { + SESSION_RECOVERY_ENVIRONMENT: 'configured-env', + SESSION_RECOVERY_REGION: 'configured-region', + SESSION_RECOVERY_NAMESPACE: 'configured-namespace', + SESSION_CACHE_TTL: '86400', + })).toMatchObject({ + environment: 'configured-env', + region: 'configured-region', + namespace: 'configured-namespace', + ttlSeconds: 86400, + }); + }); + + it('requires the recovery scope', () => { + expect(() => parseOptions([], {})).toThrow('Recovery environment'); + }); + + it('does not write during a dry run', async () => { + const store = new MemoryStore(); + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: false, ttlSeconds: 86400 }, + ); + + expect(summary).toEqual({ input: 1, missing: 1, restored: 0, matching: 0, conflicts: 0 }); + expect(store.values.size).toBe(0); + }); + + it('restores only absent keys and reports existing owners', async () => { + const store = new MemoryStore(); + store.values.set(`session:${OTHER_SESSION_ID}`, 'tenant-id:user:someone-else'); + + const summary = await recoverSessionCache( + [ + { session_id: SESSION_ID, expected_session_key: SESSION_KEY }, + { session_id: OTHER_SESSION_ID, expected_session_key: SESSION_KEY }, + ], + store, + { apply: true, ttlSeconds: 86400 }, + ); + + expect(summary).toEqual({ input: 2, missing: 0, restored: 1, matching: 0, conflicts: 1 }); + expect(store.values.get(`session:${SESSION_ID}`)).toBe(SESSION_KEY); + expect(store.values.get(`session:${OTHER_SESSION_ID}`)).toBe('tenant-id:user:someone-else'); + }); + + it('preserves partial counts when a Redis operation fails', async () => { + let reads = 0; + const store: RecoveryStore = { + async get(): Promise { + reads += 1; + if (reads === 1) { + return SESSION_KEY; + } + throw new Error('Redis unavailable'); + }, + async set(): Promise<'OK' | null> { + throw new Error('unexpected set'); + }, + }; + + try { + await recoverSessionCache( + [ + { session_id: SESSION_ID, expected_session_key: SESSION_KEY }, + { session_id: OTHER_SESSION_ID, expected_session_key: SESSION_KEY }, + ], + store, + { apply: true, ttlSeconds: 86400 }, + ); + throw new Error('expected recovery to fail'); + } catch (error) { + expect(error).toBeInstanceOf(RecoveryInterruptedError); + expect((error as RecoveryInterruptedError).summary).toEqual({ + input: 2, + missing: 0, + restored: 0, + matching: 1, + conflicts: 0, + }); + } + }); + + it('reports a key that disappears during an apply race as missing', async () => { + const store: RecoveryStore = { + async get(): Promise { + return null; + }, + async set(): Promise { + return null; + }, + }; + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400 }, + ); + + expect(summary).toEqual({ input: 1, missing: 1, restored: 0, matching: 0, conflicts: 0 }); + }); +}); From 587c79b3b8bb67d245e43dc077b2d4d82cfaf04d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 02:08:30 -0400 Subject: [PATCH 002/116] chore(codeapi): remediate SCA dependency findings (#3414) (#71) Source: ClickHouse/ai@573e859e52c5e8566843556438bdc2e479f011b4 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- api/bun.lock | 4 ++-- api/package-lock.json | 19 ++++++++++--------- api/package.json | 2 +- service/bun.lock | 12 ++++++------ service/package.json | 6 ++++-- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/api/bun.lock b/api/bun.lock index 2c714c0f..4341186d 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -11,7 +11,7 @@ "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", "express": "^4.22.2", - "nanoid": "^3.3.7", + "nanoid": "^5.1.16", "pino": "^10.3.0", "prom-client": "^15.1.3", "semver": "^7.8.0", @@ -168,7 +168,7 @@ "ms": ["ms@2.0.0", "", {}, "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="], - "nanoid": ["nanoid@3.3.8", "", { "bin": "bin/nanoid.cjs" }, "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w=="], + "nanoid": ["nanoid@5.1.16", "", { "bin": { "nanoid": "bin/nanoid.js" } }, "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ=="], "negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], diff --git a/api/package-lock.json b/api/package-lock.json index c03cb12f..998bdca7 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -15,7 +15,7 @@ "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", "express": "^4.22.2", - "nanoid": "^3.3.7", + "nanoid": "^5.1.16", "pino": "^10.3.0", "prom-client": "^15.1.3", "semver": "^7.8.0" @@ -834,20 +834,21 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "node_modules/nanoid": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", - "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "version": "5.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", + "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { - "nanoid": "bin/nanoid.cjs" + "nanoid": "bin/nanoid.js" }, "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + "node": "^18 || >=20" } }, "node_modules/negotiator": { @@ -1915,9 +1916,9 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "nanoid": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", - "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==" + "version": "5.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", + "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==" }, "negotiator": { "version": "0.6.3", diff --git a/api/package.json b/api/package.json index b36ff5fa..8ed06d85 100644 --- a/api/package.json +++ b/api/package.json @@ -16,7 +16,7 @@ "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", "express": "^4.22.2", - "nanoid": "^3.3.7", + "nanoid": "^5.1.16", "pino": "^10.3.0", "prom-client": "^15.1.3", "semver": "^7.8.0" diff --git a/service/bun.lock b/service/bun.lock index 46660bc3..0a0dea4a 100644 --- a/service/bun.lock +++ b/service/bun.lock @@ -20,7 +20,7 @@ "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", "minio": "^8.0.5", - "nanoid": "^3.3.7", + "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "winston": "^3.14.2", @@ -35,7 +35,6 @@ "@types/busboy": "^1.5.4", "@types/express": "^4.17.21", "@types/ioredis-mock": "^8.2.7", - "@types/nanoid": "^3.0.0", "@types/node": "^22.5.5", "@typescript-eslint/eslint-plugin": "^8.0.1", "@typescript-eslint/parser": "^8.0.1", @@ -51,6 +50,9 @@ }, }, }, + "overrides": { + "decode-uri-component": "0.5.0", + }, "packages": { "@aws-sdk/checksums": ["@aws-sdk/checksums@3.1000.19", "", { "dependencies": { "@aws-sdk/core": "^3.976.0", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.29.4", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-Hc4N100RdkuWshKBnhPzmpdftfi9mCLz+OHFELHM1QIgMH4QRUUWyWgfiebta/YX2Bd62wTcm3EqAP8TeXv0gA=="], @@ -236,8 +238,6 @@ "@types/minimatch": ["@types/minimatch@6.0.0", "", { "dependencies": { "minimatch": "*" } }, "sha512-zmPitbQ8+6zNutpwgcQuLcsEpn/Cj54Kbn7L5pX0Os5kdWplB7xPgEh/g+SWOB/qmows2gpuCaPyduq8ZZRnxA=="], - "@types/nanoid": ["@types/nanoid@3.0.0", "", { "dependencies": { "nanoid": "*" } }, "sha512-UXitWSmXCwhDmAKe7D3hNQtQaHeHt5L8LO1CB8GF8jlYVzOv5cBWDNqiJ+oPEWrWei3i3dkZtHY/bUtd0R/uOQ=="], - "@types/node": ["@types/node@22.19.3", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-1N9SBnWYOJTrNZCdh/yJE+t910Y128BoyY+zBLWhL3r0TYzlTmFdXrPwHL9DyFZmlEXNQQolTZh3KHV31QDhyA=="], "@types/qs": ["@types/qs@6.14.0", "", {}, "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ=="], @@ -436,7 +436,7 @@ "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - "decode-uri-component": ["decode-uri-component@0.2.2", "", {}, "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ=="], + "decode-uri-component": ["decode-uri-component@0.5.0", "", {}, "sha512-1BiQVoK8C9gUbQU6NzAtO/tkz2qOFpEObMWpcFvhx4fYnj4Oc5yzaJN/LD36ihkVUdXyh5ZekzX+yM+ty/SrPg=="], "deep-is": ["deep-is@0.1.4", "", {}, "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ=="], @@ -778,7 +778,7 @@ "msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="], - "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + "nanoid": ["nanoid@5.1.16", "", { "bin": { "nanoid": "bin/nanoid.js" } }, "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ=="], "napi-postinstall": ["napi-postinstall@0.3.4", "", { "bin": { "napi-postinstall": "lib/cli.js" } }, "sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ=="], diff --git a/service/package.json b/service/package.json index e1b280f6..c5a49a82 100644 --- a/service/package.json +++ b/service/package.json @@ -41,7 +41,7 @@ "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", "minio": "^8.0.5", - "nanoid": "^3.3.7", + "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", "winston": "^3.14.2" @@ -60,7 +60,6 @@ "@types/busboy": "^1.5.4", "@types/express": "^4.17.21", "@types/ioredis-mock": "^8.2.7", - "@types/nanoid": "^3.0.0", "@types/node": "^22.5.5", "@typescript-eslint/eslint-plugin": "^8.0.1", "@typescript-eslint/parser": "^8.0.1", @@ -73,5 +72,8 @@ "rollup-plugin-sourcemaps": "^0.6.3", "ts-node": "^10.9.2", "typescript": "^5.5.4" + }, + "overrides": { + "decode-uri-component": "0.5.0" } } From da8775fd829d362df10372fe0c02d29761bb0c89 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 1 Sep 2026 12:07:40 -0400 Subject: [PATCH 003/116] feat: add outbound stateful code bridge (#66) * feat: add outbound stateful code bridge * test: cover remote bridge startup policy * fix: harden remote bridge lifecycle fencing * fix: harden remote bridge assignment lifecycle * fix: close remote bridge commit races * fix: surface shutdown workspace quarantine * fix: fail closed across bridge lifecycle gaps * fix: persist stateful settlement commit barriers * fix: bound bridge control-plane timing * fix: recover abandoned bridge leases * fix: fence stateful bridge workspaces * fix: recover bridge lease read failures * fix: add safe workspace fence recovery * fix: bound bridge liveness timers * fix: bound bridge cleanup recovery * fix: preserve bridge deadline fencing * fix: acknowledge bridge lease delivery * fix: close bridge deadline gaps * fix: harden bridge recovery edges * fix: preserve bridge rejection recovery * fix: fence bridge lease recovery * fix: bound bridge control lifetimes * fix: retain bridge recovery ownership * fix: isolate bridge control state * fix: bound bridge workspace reset * fix: validate bridge deployment inputs * fix: close bridge registration and deadline races * fix: anchor bridge lease freshness * fix: preserve bridge response status --- .env.example | 7 + .gitignore | 1 + README.md | 14 + api/Dockerfile | 5 +- api/src/entrypoint.sh | 9 +- docker/Dockerfile.worker-sandbox | 6 +- docker/rootfs-setup.c | 92 + docker/start-direct-sandbox.sh | 33 +- docs/remote-bridge/README.md | 109 + launcher/Dockerfile | 7 +- packages/code/Dockerfile | 15 + packages/code/README.md | 57 + packages/code/package-lock.json | 54 + packages/code/package.json | 42 + packages/code/src/cli.test.ts | 49 + packages/code/src/cli.ts | 93 + packages/code/src/index.ts | 2 + packages/code/src/protocol.test.ts | 56 + packages/code/src/protocol.ts | 129 ++ packages/code/src/worker.test.ts | 1665 +++++++++++++++ packages/code/src/worker.ts | 797 ++++++++ packages/code/tsconfig.json | 15 + service/Dockerfile | 2 + service/Dockerfile.api | 2 + service/Dockerfile.local | 2 + service/Dockerfile.node | 2 + service/Dockerfile.service | 1 + service/Dockerfile.worker | 2 + service/rollup.config.js | 6 +- service/src/api-server.ts | 2 + service/src/bridge/router.ts | 381 ++++ service/src/bridge/store.test.ts | 1813 +++++++++++++++++ service/src/bridge/store.ts | 1300 ++++++++++++ service/src/config.test.ts | 3 +- service/src/config.ts | 9 +- service/src/lifecycle.ts | 4 + service/src/local-api.ts | 10 +- .../src/runtime-session/job-policy.test.ts | 13 + service/src/runtime-session/job-policy.ts | 7 +- service/src/sandbox-backend/index.test.ts | 6 + service/src/sandbox-backend/index.ts | 22 + service/src/sandbox-backend/remote-bridge.ts | 82 + service/src/sandbox-backend/types.ts | 6 +- service/src/secure-startup.test.ts | 97 +- service/src/secure-startup.ts | 42 +- service/src/service-api.ts | 2 + service/src/utils.test.ts | 38 + service/src/utils.ts | 12 +- service/tsconfig.json | 6 +- 49 files changed, 7074 insertions(+), 55 deletions(-) create mode 100644 docker/rootfs-setup.c create mode 100644 docs/remote-bridge/README.md create mode 100644 packages/code/Dockerfile create mode 100644 packages/code/README.md create mode 100644 packages/code/package-lock.json create mode 100644 packages/code/package.json create mode 100644 packages/code/src/cli.test.ts create mode 100644 packages/code/src/cli.ts create mode 100644 packages/code/src/index.ts create mode 100644 packages/code/src/protocol.test.ts create mode 100644 packages/code/src/protocol.ts create mode 100644 packages/code/src/worker.test.ts create mode 100644 packages/code/src/worker.ts create mode 100644 packages/code/tsconfig.json create mode 100644 service/src/bridge/router.ts create mode 100644 service/src/bridge/store.test.ts create mode 100644 service/src/bridge/store.ts create mode 100644 service/src/sandbox-backend/remote-bridge.ts diff --git a/.env.example b/.env.example index 54a2971a..3bd113e5 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,13 @@ SANDBOX_RUN_CPU_TIME=10000 SANDBOX_RUN_TIMEOUT=15000 SANDBOX_OUTPUT_MAX_SIZE=65536 +# Remote stateful code bridge (Code API deployment) +# CODEAPI_SANDBOX_BACKEND=remote-bridge +# CODEAPI_EXECUTION_PROFILE=stateful +# CODEAPI_RUNTIME_SESSION_MODE=affinity +# CODEAPI_BRIDGE_WORKER_ID=my-vm +# CODEAPI_BRIDGE_TOKEN=replace-with-a-strong-random-secret + # Service Configuration PYTHON_CONCURRENCY=5 OTHER_CONCURRENCY=15 diff --git a/.gitignore b/.gitignore index db2b8a28..a1a0c6ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ data/ node_modules +packages/*/dist/ .env .git .npmrc diff --git a/README.md b/README.md index 6612cb5f..716941d9 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Code Interpreter (internally `codeapi`, the prefix used by its env vars, images, - **Package Delivery** - Bakes Python, Node, and Bun into the default microVM block-root image; a package-init PVC mode remains available for direct NsJail development +- **Remote Code Bridge** - Lets an operator-owned VM connect outbound and serve + as a fenced, stateful sandbox through the `@librechat/code` worker ## Architecture @@ -65,6 +67,18 @@ Two modes are supported: - **NsJail mode** (`kvmEnabled: false`): Direct NsJail sandboxing with Linux namespaces and cgroups - **MicroVM mode** (`kvmEnabled: true`): libkrun microVM with its own kernel, NsJail runs inside the guest +## Remote stateful environments + +The `remote-bridge` backend keeps the Code API as the policy and queue boundary +while moving execution to a sandbox on an operator-selected VM. The worker only +makes outbound authenticated requests, so the VM does not need a public ingress +port. Assignments carry a deadline, a single-active-worker lock, a monotonically +increasing generation, and a one-time lease token to fence stale workers. + +See [Remote Code Bridge](docs/remote-bridge/README.md) for deployment and threat +model details. The worker protocol and CLI live in the provider-neutral +[`@librechat/code`](packages/code/README.md) package. + ## Security disclaimer This service exists to run arbitrary, untrusted code — treat every diff --git a/api/Dockerfile b/api/Dockerfile index f8d6713c..3526e98e 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -29,8 +29,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ - && chmod 0111 /usr/local/bin/spec-guard + && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ + && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup # ============================================================================ # Stage 1b: Build language runtime packages (only consumed by sandbox-runner-baked) @@ -212,6 +214,7 @@ RUN dnf install -y --setopt=install_weak_deps=False \ && dnf clean all COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher +COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh index b532a67c..4fce5ba4 100755 --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -171,28 +171,33 @@ fi chmod 777 "$SMOKE_DIR" fi SMOKE_LOG=$(mktemp) +SMOKE_STDERR=$(mktemp) NSJAIL_CGROUP_ARGS=() if [ "$SANDBOX_USE_CGROUPV2" = "true" ]; then NSJAIL_CGROUP_ARGS=(--use_cgroupv2) fi -if timeout 10 /usr/sbin/nsjail --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ +if timeout 10 "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ "${NSJAIL_CGROUP_ARGS[@]}" --log "$SMOKE_LOG" \ --user "65534:${SMOKE_OUTSIDE_UID}:1" --group "65534:${SMOKE_OUTSIDE_GID}:1" \ -s /usr/bin:/bin -s /usr/lib:/lib -s /usr/lib64:/lib64 \ -B "$SMOKE_DIR:/mnt/data" \ - -- /bin/sh -c 'printf "%s\n" sandbox_ok > /mnt/data/smoke.txt && test "$(cat /mnt/data/smoke.txt)" = sandbox_ok' > /dev/null 2>&1; then + -- /bin/sh -c 'printf "%s\n" sandbox_ok > /mnt/data/smoke.txt && test "$(cat /mnt/data/smoke.txt)" = sandbox_ok' > /dev/null 2>"$SMOKE_STDERR"; then echo "NsJail smoke test passed" else echo "FATAL: NsJail smoke test failed — sandbox cannot start" echo "NsJail log output:" cat "$SMOKE_LOG" 2>/dev/null || true + echo "NsJail stderr:" + cat "$SMOKE_STDERR" 2>/dev/null || true rm -f "$SMOKE_LOG" + rm -f "$SMOKE_STDERR" rm -rf "$SMOKE_DIR" exit 1 fi rm -f "$SMOKE_LOG" +rm -f "$SMOKE_STDERR" rm -rf "$SMOKE_DIR" echo "Starting sandbox API server..." diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index cd3edd3d..c18eab82 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -42,8 +42,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ - && chmod 0111 /usr/local/bin/spec-guard + && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ + && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup # ============================================================================ # Stage 1b: Build language runtime packages for the baked KVM root disk @@ -83,6 +85,7 @@ WORKDIR /app COPY service/package.json service/bun.lock ./ RUN bun install --frozen-lockfile COPY service/src ./src +COPY packages/code/src /packages/code/src COPY shared /shared COPY service/tsconfig.json ./ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' @@ -231,6 +234,7 @@ ENV PATH="/root/.bun/bin:${PATH}" # --- Launcher (runs on host, boots microVM) --- COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher +COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup # --- Launcher entrypoint (DNS resolution + socat relay before VM boot) --- COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh diff --git a/docker/rootfs-setup.c b/docker/rootfs-setup.c new file mode 100644 index 00000000..b0b817cf --- /dev/null +++ b/docker/rootfs-setup.c @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +static int bind_mount(const char *source, const char *target, int read_only) { + if (mount(source, target, NULL, MS_BIND | MS_REC, NULL) != 0) { + fprintf(stderr, "bind %s -> %s failed: %s\n", source, target, strerror(errno)); + return -1; + } + + if (read_only && + mount(NULL, target, NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, NULL) != 0) { + fprintf(stderr, "read-only remount of %s failed: %s\n", target, strerror(errno)); + return -1; + } + + return 0; +} + +static int bind_rootfs_path(const char *rootfs, const char *path) { + char source[PATH_MAX]; + int written = snprintf(source, sizeof(source), "%s%s", rootfs, path); + if (written < 0 || (size_t)written >= sizeof(source)) { + fprintf(stderr, "rootfs path is too long: %s%s\n", rootfs, path); + return -1; + } + + return bind_mount(source, path, 1); +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: sandbox-rootfs-setup ROOTFS [COMMAND ...]\n"); + return 2; + } + + const char *rootfs = argv[1]; + if (rootfs[0] != '/') { + fprintf(stderr, "rootfs must be an absolute path\n"); + return 2; + } + + if (mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0) { + fprintf(stderr, "making the mount namespace private failed: %s\n", strerror(errno)); + return 1; + } + + if ((mkdir("/sandbox_api", 0755) != 0 && errno != EEXIST) || + (mkdir("/pkgs", 0755) != 0 && errno != EEXIST)) { + fprintf(stderr, "creating rootfs mount targets failed: %s\n", strerror(errno)); + return 1; + } + + /* + * Keep this process statically linked: the final /usr mount replaces + * the Fedora launcher's dynamic userspace with the Debian sandbox rootfs. + * A shell cannot safely perform this sequence because its next command may + * try to load a host binary against guest libraries (or vice versa). + */ + const char *paths[] = {"/sandbox_api", "/pkgs"}; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + if (bind_rootfs_path(rootfs, paths[i]) != 0) { + return 1; + } + } + + if (access("/host-packages", F_OK) == 0 && + bind_mount("/host-packages", "/pkgs", 0) != 0) { + fprintf(stderr, "warning: sandbox will run without host packages\n"); + } + + /* Bind all guest userspace last, then immediately enter it. */ + if (bind_rootfs_path(rootfs, "/usr") != 0) { + return 1; + } + + setenv("PATH", "/root/.bun/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 1); + setenv("LD_LIBRARY_PATH", "/usr/lib/aarch64-linux-gnu:/usr/lib/x86_64-linux-gnu", 1); + + setenv("NSJAIL_PATH", "/usr/sbin/nsjail", 1); + + char *default_argv[] = {"/sandbox_api/entrypoint.sh", NULL}; + char **command_argv = argc > 2 ? &argv[2] : default_argv; + execv(command_argv[0], command_argv); + fprintf(stderr, "starting sandbox entrypoint failed: %s\n", strerror(errno)); + return 1; +} diff --git a/docker/start-direct-sandbox.sh b/docker/start-direct-sandbox.sh index a171a6df..bf7c9805 100644 --- a/docker/start-direct-sandbox.sh +++ b/docker/start-direct-sandbox.sh @@ -44,35 +44,4 @@ else fi export SANDBOX_ROOTFS="$ROOTFS" - -exec unshare --mount bash -c ' - ROOTFS="${SANDBOX_ROOTFS:-/sandbox-rootfs}" - - mount -o bind,ro "$ROOTFS/usr/sbin" /usr/sbin || { echo "FATAL: cannot bind /usr/sbin"; exit 1; } - mount -o bind,ro "$ROOTFS/usr/lib" /usr/lib || { echo "FATAL: cannot bind /usr/lib"; exit 1; } - - if [ -d "$ROOTFS/usr/lib64" ] && ! [ -L "$ROOTFS/usr/lib64" ]; then - mount -o bind,ro "$ROOTFS/usr/lib64" /usr/lib64 2>/dev/null || \ - echo "[sandbox] WARNING: could not bind /usr/lib64 - sandboxed binaries may fail to exec" - fi - - mount -o bind,ro "$ROOTFS/usr/local" /usr/local || { echo "FATAL: cannot bind /usr/local"; exit 1; } - mount -o bind,ro "$ROOTFS/sandbox_api" /sandbox_api || { echo "FATAL: cannot bind /sandbox_api"; exit 1; } - mount -o bind,ro "$ROOTFS/pkgs" /pkgs || { echo "FATAL: cannot bind /pkgs"; exit 1; } - - if [ -d /host-packages ]; then - mount --bind /host-packages /pkgs 2>/dev/null || \ - echo "WARNING: could not bind /host-packages - sandbox will run without packages" - fi - - mount -o bind,ro "$ROOTFS/usr/bin" /usr/bin || { echo "FATAL: cannot bind /usr/bin"; exit 1; } - - multiarch_libdir=$(find /usr/lib -maxdepth 1 -type d -name "*-linux-gnu" -print -quit) - if [ -n "$multiarch_libdir" ]; then - export LD_LIBRARY_PATH="$multiarch_libdir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - fi - - export PATH="/root/.bun/bin:$PATH" - - exec /sandbox_api/entrypoint.sh -' +exec unshare --mount /sandbox-rootfs-setup "$ROOTFS" diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md new file mode 100644 index 00000000..dd22e73b --- /dev/null +++ b/docs/remote-bridge/README.md @@ -0,0 +1,109 @@ +# Remote Code Bridge + +Remote Code Bridge makes an operator-owned VM a stateful Code API execution +environment without exposing that VM to inbound internet traffic. + +```text +LibreChat -> Code API -> Redis assignment + ^ | + | outbound v + @librechat/code -> local sandbox +``` + +Code API remains the public authentication, policy, manifest, timeout, and +result-normalization boundary. The bridge worker has a separate operator +credential and never accepts end-user bearer tokens directly. + +## Code API configuration + +Run this as an isolated stateful Code API deployment: + +```dotenv +CODEAPI_SANDBOX_BACKEND=remote-bridge +CODEAPI_EXECUTION_PROFILE=stateful +CODEAPI_RUNTIME_SESSION_MODE=affinity +CODEAPI_BRIDGE_WORKER_ID=my-vm +CODEAPI_BRIDGE_TOKEN= +``` + +Use `strict` instead of `affinity` if every request must include a runtime +session hint. In hardened mode, startup requires the bridge token to be at least +32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a +remote execution cannot retain an open Code API process across tool callbacks. + +Start the CLI beside a sandbox using the same worker ID and secret; see +[`@librechat/code`](../../packages/code/README.md). +Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` +and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, +persistent local runner per session. A single sandbox endpoint is stateless and +is rejected for runtime-session assignments. + +## LibreChat configuration + +Expose the Code API deployment as an environment under the Agents endpoint: + +```yaml +endpoints: + agents: + statefulCodeSessions: + environments: + - id: my-vm + name: My VM + type: attached + baseURL: https://code.example.com/v1 + default: true +``` + +Agents may select this environment with `code_environment_id: my-vm`. +LibreChat derives a stable per-conversation runtime session ID, so commands in +later turns reuse the same workspace. Attached environments deliberately skip +background prewarming: the single worker lease is reserved for explicit user +execution. + +## Lifecycle and fencing + +- Registration is ephemeral in Redis and must be refreshed by the worker. +- Code API permits one active assignment per configured worker. +- Each assignment has an absolute deadline, generation, and random lease token. +- Settlements with the wrong worker, generation, token, or expired deadline are + rejected. +- Assignments are queued for the exact registered worker incarnation, so an + outstanding poll from a replaced process cannot consume replacement work. +- Assignment records and the worker lock live through the full configured job + deadline plus cleanup grace. +- Ambiguous settlement delivery is retried through the assignment deadline. If + a stateful settlement remains ambiguous, the CLI exits and the affected local + session runner must be reset or discarded before restart. +- Enqueueing stateful work atomically creates a durable in-flight workspace + marker. A definite rejection or successful result finalization clears it; + worker or VM loss leaves it in place so later reuse fails closed. Settlement + receipts outlive assignment cleanup briefly so retries are idempotent and + cannot recreate a cleared marker. +- To recover a fenced session, stop the normal worker process and discard/reset + that session's local sandbox workspace. While it remains stopped, run + `librechat-code reset-workspace ` with the same worker + configuration; the command temporarily registers its own incarnation and + exits. Start the normal worker only after the reset command succeeds. Code API + refuses the acknowledgement while work is active or when it is not made by + the currently registered incarnation. +- Request cancellation is polled by the worker and aborts the local sandbox + request. +- A leased assignment remains in a Redis-backed delivery claim until the worker + explicitly acknowledges it; reconnecting before acknowledgement redelivers + the same fenced assignment instead of losing it after an HTTP disconnect. +- The sandbox receives the stable runtime session ID separately from the lease; + workspace state belongs to that session, not to a transient assignment. + +## Security boundaries + +The bridge removes inbound VM exposure; it does not replace sandbox isolation. +For internet-facing LibreChat deployments, use the hardened microVM/NsJail +stack, default-deny sandbox egress, signed execution manifests, least-privilege +host credentials, resource limits, and host/network monitoring. Bind the local +sandbox endpoint to loopback or a private container network. Rotate a leaked +bridge token immediately; the initial protocol intentionally uses a static +operator secret and supports one configured worker per Code API deployment. + +The next control-plane layer can add short-lived pairing credentials and a +multi-worker directory without changing the execution protocol or moving code +tools into the Agents SDK. diff --git a/launcher/Dockerfile b/launcher/Dockerfile index 0d252c28..1a077e12 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -33,8 +33,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ - && chmod 0111 /usr/local/bin/spec-guard + && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ + && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup FROM oven/bun:1.3.14-debian AS sandbox-build @@ -126,6 +128,7 @@ RUN dnf install -y --setopt=install_weak_deps=False \ && dnf clean all COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher +COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup COPY --from=sandbox-build / /sandbox-rootfs/ @@ -136,6 +139,6 @@ RUN mkdir -p /host-packages COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh COPY docker/sandbox-entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /sandbox-rootfs-setup /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh /usr/local/bin/entrypoint.sh ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/packages/code/Dockerfile b/packages/code/Dockerfile new file mode 100644 index 00000000..21fb09bd --- /dev/null +++ b/packages/code/Dockerfile @@ -0,0 +1,15 @@ +FROM node:24-alpine AS build +WORKDIR /app +COPY package.json package-lock.json tsconfig.json ./ +RUN npm ci +COPY src ./src +RUN npm run build + +FROM node:24-alpine +ENV NODE_ENV=production +RUN addgroup -S librechat-code && adduser -S librechat-code -G librechat-code +WORKDIR /app +COPY --from=build /app/package.json ./package.json +COPY --from=build /app/dist ./dist +USER librechat-code +ENTRYPOINT ["node", "dist/cli.js"] diff --git a/packages/code/README.md b/packages/code/README.md new file mode 100644 index 00000000..2439ab25 --- /dev/null +++ b/packages/code/README.md @@ -0,0 +1,57 @@ +# `@librechat/code` + +Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed +code environment to LibreChat Code API. + +The CLI is a transport bridge, not a sandbox. Run it beside a Code Interpreter +sandbox (NsJail for trusted local development, or the hardened microVM stack for +untrusted internet traffic). It connects outbound to Code API, long-polls for +assignments, forwards them to the local sandbox, and returns fenced results. +The VM does not need an inbound public port. + +## Run + +```bash +npm install -g @librechat/code + +LIBRECHAT_CODE_URL=https://code.example.com/v1 \ +LIBRECHAT_CODE_WORKER_TOKEN='' \ +LIBRECHAT_CODE_WORKER_ID=my-vm \ +LIBRECHAT_CODE_SANDBOX_ENDPOINT=http://127.0.0.1:2000/api/v2 \ +librechat-code +``` + +Optional environment variables: + +- `LIBRECHAT_CODE_SANDBOX_PROFILE`: capability label; defaults to `nsjail`. +- `LIBRECHAT_CODE_RUNTIMES`: comma-separated capability labels. +- `LIBRECHAT_CODE_POLICY`: local policy description hashed into the worker's + registration; defaults to `default-deny`. +- `LIBRECHAT_CODE_STATEFUL_WORKSPACE`: defaults to `false`. Set it to `true` + only when the local sandbox supervisor provides a distinct persistent runner + for every runtime session. In that mode the endpoint must contain a + `{runtimeSessionId}` placeholder, for example + `http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2`. The worker URL- + encodes and substitutes the assigned session ID before execution. Hintless + assignments use an ephemeral `assignment-` session so affinity-mode + stateless work never reaches a literal placeholder route. + +A single built-in sandbox runner binds itself to one runtime session and must +not be advertised as stateful. Use the default stateless capability until a +session-routing supervisor is configured. + +The worker retries result settlement through the assignment deadline. If a +stateful result remains ambiguous, it exits with a quarantine error instead of +accepting another assignment. Reset or discard that session's local runner +before restarting the worker; its workspace may contain mutations that Code +API did not commit. + +After discarding or resetting that session's local runner, acknowledge recovery +with `librechat-code reset-workspace `. The command uses the +configured worker credentials, registers a fresh incarnation, and only clears +the server fence when no assignment is active. Run it while the normal worker +process is stopped, then restart the normal worker after the command exits. + +Use a unique worker ID and secret per Code API deployment, expose only the +sandbox loopback endpoint to the CLI, and enforce VM/container egress policy +independently of the bridge transport. diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json new file mode 100644 index 00000000..15ca1ad6 --- /dev/null +++ b/packages/code/package-lock.json @@ -0,0 +1,54 @@ +{ + "name": "@librechat/code", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@librechat/code", + "version": "0.1.0", + "license": "Apache-2.0", + "bin": { + "librechat-code": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^22.5.5", + "typescript": "^5.5.4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/packages/code/package.json b/packages/code/package.json new file mode 100644 index 00000000..f195881e --- /dev/null +++ b/packages/code/package.json @@ -0,0 +1,42 @@ +{ + "name": "@librechat/code", + "version": "0.1.0", + "description": "LibreChat stateful code environment protocol and worker CLI", + "license": "Apache-2.0", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./protocol": { + "types": "./dist/protocol.d.ts", + "import": "./dist/protocol.js" + }, + "./worker": { + "types": "./dist/worker.d.ts", + "import": "./dist/worker.js" + } + }, + "bin": { + "librechat-code": "./dist/cli.js" + }, + "files": [ + "dist", + "!dist/*.test.*" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test dist/*.test.js", + "prepack": "npm run build" + }, + "devDependencies": { + "@types/node": "^22.5.5", + "typescript": "^5.5.4" + }, + "engines": { + "node": ">=20" + } +} diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts new file mode 100644 index 00000000..179bf813 --- /dev/null +++ b/packages/code/src/cli.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +test('CLI rejects an invalid worker ID before entering the run loop', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering/vm', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /LIBRECHAT_CODE_WORKER_ID must match the bridge worker ID format/, + ); +}); + +test('CLI rejects invalid advertised capabilities before registration', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_SANDBOX_PROFILE: '', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid/, + ); +}); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts new file mode 100644 index 00000000..cb1b100d --- /dev/null +++ b/packages/code/src/cli.ts @@ -0,0 +1,93 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto'; +import { BridgeWorker } from './worker.js'; +import { + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, +} from './protocol.js'; + +function required(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function list(value: string | undefined): string[] { + return ( + value + ?.split(',') + .map((item) => item.trim()) + .filter(Boolean) ?? [] + ); +} + +const controller = new AbortController(); +process.once('SIGINT', () => controller.abort()); +process.once('SIGTERM', () => controller.abort()); + +const policy = process.env.LIBRECHAT_CODE_POLICY ?? 'default-deny'; +const statefulWorkspace = + process.env.LIBRECHAT_CODE_STATEFUL_WORKSPACE?.trim().toLowerCase() === 'true'; +const sandboxEndpoint = + process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? + 'http://127.0.0.1:2000/api/v2'; +if (statefulWorkspace && !sandboxEndpoint.includes('{runtimeSessionId}')) { + throw new Error( + 'LIBRECHAT_CODE_STATEFUL_WORKSPACE requires LIBRECHAT_CODE_SANDBOX_ENDPOINT to contain {runtimeSessionId}', + ); +} +const workerId = required('LIBRECHAT_CODE_WORKER_ID'); +if (!isValidBridgeWorkerId(workerId)) { + throw new Error( + 'LIBRECHAT_CODE_WORKER_ID must match the bridge worker ID format', + ); +} +const capabilities = { + statefulWorkspace, + sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? 'nsjail', + runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), + policyDigest: createHash('sha256').update(policy).digest('hex'), +}; +if (!isValidBridgeWorkerCapabilities(capabilities)) { + throw new Error( + 'LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid', + ); +} +const worker = new BridgeWorker({ + codeApiUrl: required('LIBRECHAT_CODE_URL'), + token: required('LIBRECHAT_CODE_WORKER_TOKEN'), + workerId, + sandboxEndpoint, + capabilities, + onError: (error) => { + const message = error instanceof Error ? error.message : 'unknown bridge error'; + process.stderr.write(`librechat-code: reconnecting after ${message}\n`); + }, +}); + +async function main(): Promise { + const command = process.argv[2]; + if (command === 'reset-workspace') { + const runtimeSessionId = process.argv[3]?.trim(); + if (!runtimeSessionId) { + throw new Error( + 'Usage: librechat-code reset-workspace ', + ); + } + await worker.register(controller.signal); + await worker.resetWorkspace(runtimeSessionId, controller.signal); + process.stdout.write( + `librechat-code: reset acknowledged for ${runtimeSessionId}\n`, + ); + return; + } + if (command != null) { + throw new Error(`Unknown command: ${command}`); + } + await worker.run(controller.signal); +} + +main().catch((error: Error) => { + process.stderr.write(`librechat-code: ${error.message}\n`); + process.exitCode = 1; +}); diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts new file mode 100644 index 00000000..c5eeaafc --- /dev/null +++ b/packages/code/src/index.ts @@ -0,0 +1,2 @@ +export * from './protocol.js'; +export * from './worker.js'; diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts new file mode 100644 index 00000000..61c50922 --- /dev/null +++ b/packages/code/src/protocol.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + bridgeWorkerPath, + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, +} from './protocol.js'; + +test('bridgeWorkerPath encodes worker-controlled path segments', () => { + assert.equal( + bridgeWorkerPath('vm/example worker'), + '/bridge/workers/vm%2Fexample%20worker', + ); +}); + +test('bridge worker IDs reject path, whitespace, and oversized values', () => { + assert.equal(isValidBridgeWorkerId('engineering-vm:1'), true); + assert.equal(isValidBridgeWorkerId('engineering/vm'), false); + assert.equal(isValidBridgeWorkerId('engineering vm'), false); + assert.equal(isValidBridgeWorkerId('a'.repeat(129)), false); +}); + +test('bridge worker capabilities enforce registration limits', () => { + const valid = { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + policyDigest: 'a'.repeat(64), + }; + assert.equal(isValidBridgeWorkerCapabilities(valid), true); + assert.equal( + isValidBridgeWorkerCapabilities({ ...valid, sandboxProfile: '' }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + sandboxProfile: 'a'.repeat(129), + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + runtimes: Array.from({ length: 33 }, () => 'bash'), + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + runtimes: ['a'.repeat(65)], + }), + false, + ); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts new file mode 100644 index 00000000..9a5d2ae0 --- /dev/null +++ b/packages/code/src/protocol.ts @@ -0,0 +1,129 @@ +export const BRIDGE_PROTOCOL_VERSION = 1 as const; +export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +export const BRIDGE_SANDBOX_PROFILE_MAX_LENGTH = 128; +export const BRIDGE_RUNTIME_MAX_COUNT = 32; +export const BRIDGE_RUNTIME_MAX_LENGTH = 64; + +export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; + +export interface BridgeWorkerCapabilities { + statefulWorkspace: boolean; + sandboxProfile: string; + runtimes: string[]; + policyDigest?: string; +} + +export interface BridgeWorkerRegistration { + protocolVersion: BridgeProtocolVersion; + workerId: string; + incarnationId: string; + capabilities: BridgeWorkerCapabilities; +} + +export interface BridgeWorkerRegistrationResponse { + protocolVersion: BridgeProtocolVersion; + workerId: string; + incarnationId: string; + registeredAt: string; + leaseTtlMs: number; +} + +export interface BridgeSandboxRequest { + body: TBody; + headers: Record; +} + +export interface BridgeAssignment { + protocolVersion: BridgeProtocolVersion; + assignmentId: string; + workerId: string; + incarnationId: string; + generation: number; + leaseToken: string; + expiresAt: string; + /** Server-calculated execution budget at lease time; avoids VM clock skew. */ + remainingMs?: number; + runtimeSessionId?: string; + request: BridgeSandboxRequest; +} + +export interface BridgeLeaseResponse { + protocolVersion: BridgeProtocolVersion; + /** Time spent handling the lease request on Code API, excluding transit. */ + serverElapsedMs?: number; + assignment?: BridgeAssignment; +} + +export interface BridgeFulfilledSettlement { + protocolVersion: BridgeProtocolVersion; + generation: number; + leaseToken: string; + incarnationId: string; + status: 'fulfilled'; + result: TResult; +} + +export interface BridgeRejectedSettlement { + protocolVersion: BridgeProtocolVersion; + generation: number; + leaseToken: string; + incarnationId: string; + status: 'rejected'; + error: string; +} + +export type BridgeSettlement = + BridgeFulfilledSettlement | BridgeRejectedSettlement; + +export interface BridgeSettlementResponse { + protocolVersion: BridgeProtocolVersion; + accepted: true; +} + +export interface BridgeCancellationResponse { + protocolVersion: BridgeProtocolVersion; + cancelled: boolean; +} + +export class BridgeProtocolError extends Error { + constructor( + message: string, + public readonly status?: number, + public readonly code?: string, + ) { + super(message); + this.name = 'BridgeProtocolError'; + } +} + +export function bridgeWorkerPath(workerId: string): string { + return `/bridge/workers/${encodeURIComponent(workerId)}`; +} + +export function isValidBridgeWorkerId(workerId: string): boolean { + return BRIDGE_WORKER_ID_PATTERN.test(workerId); +} + +export function isValidBridgeWorkerCapabilities( + value: unknown, +): value is BridgeWorkerCapabilities { + if (typeof value !== 'object' || value === null) return false; + const capabilities = value as Record; + return ( + typeof capabilities.statefulWorkspace === 'boolean' && + typeof capabilities.sandboxProfile === 'string' && + capabilities.sandboxProfile.trim().length > 0 && + capabilities.sandboxProfile.length <= BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && + Array.isArray(capabilities.runtimes) && + capabilities.runtimes.length <= BRIDGE_RUNTIME_MAX_COUNT && + capabilities.runtimes.every( + (runtime) => + typeof runtime === 'string' && + runtime.length > 0 && + runtime.length <= BRIDGE_RUNTIME_MAX_LENGTH, + ) && + (capabilities.policyDigest === undefined || + (typeof capabilities.policyDigest === 'string' && + /^[a-f0-9]{64}$/.test(capabilities.policyDigest))) + ); +} diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts new file mode 100644 index 00000000..d633366e --- /dev/null +++ b/packages/code/src/worker.test.ts @@ -0,0 +1,1665 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; + +import type { BridgeAssignment } from './protocol.js'; + +test('worker forwards a fenced assignment to the sandbox and settles the result', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + requests.push({ url, init }); + if (url.endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1/', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2/', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + const assignment: BridgeAssignment = { + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 3, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { + body: { language: 'bash' }, + headers: { 'X-Execution-Manifest': 'signed' }, + }, + }; + + await worker.executeAndSettle(assignment); + + assert.equal(requests.length, 2); + assert.equal( + requests[0].url, + 'http://127.0.0.1:2000/sessions/rt-user-1/api/v2/execute', + ); + assert.equal( + (requests[0].init?.headers as Record)[ + 'X-Runtime-Session-Id' + ], + 'rt-user-1', + ); + assert.match(requests[1].url, /assignments\/assignment-1\/settle$/); + assert.deepEqual(JSON.parse(String(requests[1].init?.body)), { + protocolVersion: 1, + generation: 3, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + incarnationId: 'incarnation-00000001', + status: 'fulfilled', + result: { session_id: 'run-1', files: [] }, + }); +}); + +test('worker acknowledges a discarded workspace through the reset endpoint', async () => { + let requestBody: Record | undefined; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + assert.match(String(input), /workers\/vm-1\/workspaces\/reset$/); + requestBody = JSON.parse(String(init?.body)) as Record; + return new Response(JSON.stringify({ protocolVersion: 1, reset: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }, + }); + + await worker.resetWorkspace('rt-user-1'); + assert.deepEqual(requestBody, { + protocolVersion: 1, + incarnationId: 'incarnation-00000001', + runtimeSessionId: 'rt-user-1', + confirmDiscarded: true, + }); +}); + +test('worker bounds a stalled workspace reset request', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + resetTransportTimeoutMs: 20, + fetchImpl: async (_input, init) => + await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }), + }); + + await assert.rejects(worker.resetWorkspace('rt-user-1'), { + name: 'AbortError', + }); +}); + +test('worker continues after an assignment-scoped settlement conflict', async () => { + const controller = new AbortController(); + let registrations = 0; + let leases = 0; + let leaseAcknowledged = false; + let observedError: unknown; + const assignment: BridgeAssignment = { + protocolVersion: 1, + assignmentId: 'expired-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + request: { body: { language: 'bash' }, headers: {} }, + }; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/workers/register')) { + registrations += 1; + if (registrations === 2) controller.abort(); + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (init?.signal?.aborted === true) { + throw new DOMException('aborted', 'AbortError'); + } + if (url.endsWith('/lease')) { + leases += 1; + return new Response( + JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (url.endsWith('/ack')) { + leaseAcknowledged = true; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (url.endsWith('/execute')) { + assert.equal(leaseAcknowledged, true); + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: 'Bridge assignment has expired', + code: 'ASSIGNMENT_EXPIRED', + }), + { status: 409, headers: { 'Content-Type': 'application/json' } }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + reconnectDelayMs: 0, + fetchImpl, + onError: (error) => { + observedError = error; + }, + }); + + await worker.run(controller.signal); + assert.equal(registrations, 2); + assert.equal(leases, 1); + assert.equal( + observedError instanceof Error ? observedError.message : undefined, + 'Bridge assignment has expired', + ); +}); + +test('worker aborts sandbox execution at the absolute assignment deadline', async () => { + let settlement: Record | undefined; + const fetchImpl: typeof fetch = async (input, init) => { + if (String(input).endsWith('/execute')) { + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + settlement = JSON.parse(String(init?.body)) as Record; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-deadline', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 30).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.incarnationId, 'incarnation-00000001'); +}); + +test('worker refreshes its registration during a long assignment', async () => { + let registrations = 0; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/workers/register')) { + registrations += 1; + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 100, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (url.endsWith('/execute')) { + await new Promise((resolve) => setTimeout(resolve, 20)); + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + protocolVersion: 1, + accepted: true, + body: init?.body, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + await worker.register(); + await new Promise((resolve) => setTimeout(resolve, 45)); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-heartbeat', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.ok(registrations >= 2); +}); + +test('worker schedules registration freshness from request start', async () => { + let registrations = 0; + const fetchImpl: typeof fetch = async (input) => { + const url = String(input); + if (url.endsWith('/workers/register')) { + registrations += 1; + if (registrations === 1) { + await new Promise((resolve) => setTimeout(resolve, 40)); + } + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 50, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (url.endsWith('/execute')) { + await new Promise((resolve) => setTimeout(resolve, 10)); + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (url.endsWith('/cancelled')) { + return new Response( + JSON.stringify({ protocolVersion: 1, cancelled: false }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + registrationTransportTimeoutMs: 100, + cancellationPollIntervalMs: 100, + fetchImpl, + }); + await worker.register(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-registration-transit', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.ok(registrations >= 2); +}); + +test('worker continues cancellation polling after a stalled response', async () => { + let cancellationAttempts = 0; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + if (url.endsWith('/cancellation')) { + cancellationAttempts += 1; + if (cancellationAttempts === 1) { + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + return new Response(JSON.stringify({ cancelled: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempted = true; + return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 5, + cancellationTransportTimeoutMs: 10, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'cancel-after-stall', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.equal(cancellationAttempts, 2); + assert.equal(settlementAttempted, true); +}); + +test('worker routes a hintless assignment to an ephemeral template session', async () => { + let executeUrl = ''; + let runtimeSessionHeader = ''; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + executeUrl = url; + runtimeSessionHeader = (init?.headers as Record)[ + 'X-Runtime-Session-Id' + ]; + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'hintless-assignment', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal( + executeUrl, + 'http://127.0.0.1:2000/sessions/assignment-hintless-assignment/api/v2/execute', + ); + assert.equal(runtimeSessionHeader, 'assignment-hintless-assignment'); +}); + +test('worker quarantines a fulfilled stateful settlement rejected by Code API', async () => { + const fetchImpl: typeof fetch = async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: 'assignment was fenced' }), { + status: 409, + headers: { 'Content-Type': 'application/json' }, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'fenced-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + BridgeWorkspaceQuarantinedError, + ); +}); + +test('worker surfaces a definite stateless settlement rejection directly', async () => { + const fetchImpl: typeof fetch = async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: 'assignment was fenced' }), { + status: 409, + headers: { 'Content-Type': 'application/json' }, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'fenced-stateless-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }), + (error: unknown) => + error instanceof Error && + error.name === 'BridgeProtocolError' && + error.message === 'assignment was fenced', + ); +}); + +test('worker preserves status for a non-JSON settlement rejection', async () => { + let settlementAttempts = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempts += 1; + return new Response('assignment fenced', { + status: 409, + headers: { 'Content-Type': 'text/html' }, + }); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'non-json-fenced-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }), + (error: unknown) => + error instanceof Error && + error.name === 'BridgeProtocolError' && + 'status' in error && + error.status === 409, + ); + assert.equal(settlementAttempts, 1); +}); + +test('worker retries an ambiguous settlement before the deadline', async () => { + let settlementAttempts = 0; + const fetchImpl: typeof fetch = async (input) => { + const url = String(input); + if (url.endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempts += 1; + if (settlementAttempts === 1) throw new TypeError('connection reset'); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'retry-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(settlementAttempts, 2); +}); + +test('worker quarantines stateful reuse after settlement stays ambiguous', async () => { + const fetchImpl: typeof fetch = async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new TypeError('connection reset'); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'ambiguous-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 50).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + BridgeWorkspaceQuarantinedError, + ); +}); + +test('worker keeps a definite stateful rejection nonfatal when settlement is ambiguous', async () => { + const fetchImpl: typeof fetch = async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ error: 'syntax_error' }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new TypeError('connection reset'); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + rejectionAckGraceMs: 0, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'rejected-ambiguous-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 50).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + (error: unknown) => + error instanceof TypeError && + !(error instanceof BridgeWorkspaceQuarantinedError), + ); +}); + +test('worker retries a known-clean rejection after shutdown until acknowledged', async () => { + const controller = new AbortController(); + let settlementAttempts = 0; + let registrations = 0; + const fetchImpl: typeof fetch = async (input) => { + if (String(input).endsWith('/workers/register')) { + registrations += 1; + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 50, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ error: 'syntax_error' }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempts += 1; + if (settlementAttempts === 1) { + controller.abort(); + throw new TypeError('connection reset'); + } + return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + rejectionAckGraceMs: 500, + fetchImpl, + }); + + await worker.register(); + await worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'late-clean-rejection', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 20).toISOString(), + remainingMs: 20, + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }, + controller.signal, + ); + assert.equal(controller.signal.aborted, true); + assert.equal(settlementAttempts, 2); + assert.ok(registrations > 1); +}); + +test('worker preserves a definite rejection when its heartbeat fails', async () => { + let registrations = 0; + let rejectedSettlement = false; + let settlementAttempts = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/workers/register')) { + registrations += 1; + if (registrations === 2) { + throw new TypeError('registration unavailable'); + } + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 50, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (String(input).endsWith('/execute')) { + await new Promise((resolve) => setTimeout(resolve, 40)); + return new Response(JSON.stringify({ error: 'syntax_error' }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempts += 1; + rejectedSettlement = + JSON.parse(String(init?.body) || '{}').status === 'rejected'; + if (settlementAttempts === 1) { + return new Response(JSON.stringify({ error: 'unavailable' }), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await worker.register(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'clean-rejection-after-heartbeat-error', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.ok(registrations >= 3); + assert.equal(rejectedSettlement, true); + assert.equal(settlementAttempts, 2); +}); + +test('worker quarantines a stateful workspace after a sandbox 5xx response', async () => { + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ error: 'upstream failed' }), { + status: 502, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempted = true; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'ambiguous-5xx', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.equal(settlementAttempted, false); +}); + +test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => { + let rejectedSettlement = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/execute')) { + return new Response('not found', { + status: 404, + headers: { 'Content-Type': 'text/html' }, + }); + } + rejectedSettlement = + JSON.parse(String(init?.body) || '{}').status === 'rejected'; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'non-json-404', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.equal(rejectedSettlement, true); +}); + +test('worker quarantines a stateful workspace after the sandbox request aborts', async () => { + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + if (String(input).endsWith('/execute')) { + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + settlementAttempted = true; + return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'aborted-execution', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 30).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.equal(settlementAttempted, false); +}); + +test('worker surfaces quarantine when shutdown aborts stateful execution', async () => { + const controller = new AbortController(); + let executeStarted = false; + const assignment: BridgeAssignment = { + protocolVersion: 1, + assignmentId: 'shutdown-execution', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/workers/register')) { + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (url.endsWith('/lease')) { + return new Response( + JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (url.endsWith('/execute')) { + executeStarted = true; + setTimeout(() => controller.abort(), 10); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await assert.rejects( + worker.run(controller.signal), + BridgeWorkspaceQuarantinedError, + ); + assert.equal(executeStarted, true); +}); + +test('worker does not start execution after shutdown is already aborted', async () => { + const controller = new AbortController(); + controller.abort(new DOMException('shutdown', 'AbortError')); + let executeStarted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async () => { + executeStarted = true; + return new Response('{}', { status: 200 }); + }, + }); + + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'shutdown-before-execution', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + request: { body: { language: 'bash' }, headers: {} }, + }, + controller.signal, + ), + { name: 'AbortError' }, + ); + assert.equal(executeStarted, false); +}); + +test('worker does not start settlement after shutdown is already aborted', async () => { + const controller = new AbortController(); + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) { + controller.abort(new DOMException('shutdown', 'AbortError')); + throw new DOMException('aborted', 'AbortError'); + } + settlementAttempted = true; + return new Response('{}', { status: 200 }); + }, + }); + + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'shutdown-before-settlement', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + request: { body: { language: 'bash' }, headers: {} }, + }, + controller.signal, + ), + { name: 'AbortError' }, + ); + assert.equal(settlementAttempted, false); +}); + +test('worker bounds a stalled lease transport beyond its long poll', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + leaseWaitMs: 10, + leaseTransportGraceMs: 20, + fetchImpl: async (_input, init) => + await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }), + }); + + await assert.rejects(worker.lease(), { name: 'AbortError' }); +}); + +test('worker subtracts lease response transit from the server budget', async () => { + const originalNow = Date.now; + let now = 10_000; + Date.now = () => now; + try { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/ack')) { + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + now += 50; + return new Response( + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 20, + assignment: { + protocolVersion: 1, + assignmentId: 'transit-budget', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(0).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + const assignment = await worker.lease(); + assert.equal(assignment?.remainingMs, 970); + } finally { + Date.now = originalNow; + } +}); + +test('worker rejects a lease whose acknowledgement exhausts its budget', async () => { + const originalNow = Date.now; + let now = 100_000; + let abandonedSettlement: Record | undefined; + let registrations = 0; + let settlementAttempts = 0; + Date.now = () => now; + try { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/workers/register')) { + registrations += 1; + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 50, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (String(input).endsWith('/ack')) { + now += 10; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (String(input).endsWith('/settle')) { + settlementAttempts += 1; + abandonedSettlement = JSON.parse( + String(init?.body), + ) as Record; + if (settlementAttempts === 1) { + return new Response(JSON.stringify({ error: 'unavailable' }), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + return new Response( + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment: { + protocolVersion: 1, + assignmentId: 'expired-after-ack', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(0).toISOString(), + remainingMs: 10, + request: { body: { language: 'bash' }, headers: {} }, + }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await assert.rejects(worker.lease(), /expired during lease acknowledgement/); + assert.equal(abandonedSettlement?.status, 'rejected'); + assert.ok(registrations > 0); + assert.equal(settlementAttempts, 2); + } finally { + Date.now = originalNow; + } +}); + +test('worker rejects an assignment after ambiguous acknowledgement delivery', async () => { + let rejectedSettlement = false; + let acknowledgementAttempts = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + rejectionAckGraceMs: 500, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/ack')) { + acknowledgementAttempts += 1; + throw new TypeError('acknowledgement response lost'); + } + if (String(input).endsWith('/settle')) { + rejectedSettlement = + JSON.parse(String(init?.body) || '{}').status === 'rejected'; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (String(input).endsWith('/workers/register')) { + return new Response( + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + return new Response( + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment: { + protocolVersion: 1, + assignmentId: 'ambiguous-ack', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await assert.rejects(worker.lease(), /acknowledgement response lost/); + assert.equal(acknowledgementAttempts, 1); + assert.equal(rejectedSettlement, true); +}); + +test('worker clamps rejected settlement errors to the protocol limit', async () => { + let rejection = ''; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ error: 'x'.repeat(5_000) }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }); + } + const settlement = JSON.parse(String(init?.body)) as { error: string }; + rejection = settlement.error; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'long-rejection', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + runtimeSessionId: 'rt-long-rejection', + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.equal(rejection.length, 4_096); +}); + +test('worker quarantines an explicitly dirty stateful sandbox response', async () => { + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) { + return new Response( + JSON.stringify({ + error: 'session_workspace_dirty', + message: 'restore required', + }), + { status: 409, headers: { 'Content-Type': 'application/json' } }, + ); + } + settlementAttempted = true; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'dirty-execution', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.equal(settlementAttempted, false); +}); + +test('worker bounds a stalled registration below its lease TTL', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + registrationTransportTimeoutMs: 20, + fetchImpl: async (_input, init) => + await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }), + }); + + await assert.rejects(worker.register(), { name: 'AbortError' }); +}); + +test('worker preserves status for a non-JSON registration rejection', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async () => + new Response('unauthorized', { + status: 401, + headers: { 'Content-Type': 'text/html' }, + }), + }); + + await assert.rejects( + worker.register(), + (error: unknown) => + error instanceof Error && + error.name === 'BridgeProtocolError' && + 'status' in error && + error.status === 401, + ); +}); + +test('worker uses the server-relative lease budget despite VM clock skew', async () => { + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) { + return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + settlementAttempted = true; + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'skewed-clock-assignment', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(0).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.equal(settlementAttempted, true); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts new file mode 100644 index 00000000..5c1afb95 --- /dev/null +++ b/packages/code/src/worker.ts @@ -0,0 +1,797 @@ +import { randomBytes } from 'node:crypto'; + +import { + BRIDGE_PROTOCOL_VERSION, + BridgeProtocolError, + bridgeWorkerPath, +} from './protocol.js'; + +import type { + BridgeAssignment, + BridgeLeaseResponse, + BridgeSettlement, + BridgeSettlementResponse, + BridgeWorkerCapabilities, + BridgeWorkerRegistrationResponse, +} from './protocol.js'; + +export interface BridgeWorkerOptions { + codeApiUrl: string; + token: string; + workerId: string; + sandboxEndpoint: string; + capabilities: BridgeWorkerCapabilities; + leaseWaitMs?: number; + leaseTransportGraceMs?: number; + registrationTransportTimeoutMs?: number; + leaseAckTransportTimeoutMs?: number; + resetTransportTimeoutMs?: number; + cancellationPollIntervalMs?: number; + cancellationTransportTimeoutMs?: number; + rejectionAckGraceMs?: number; + reconnectDelayMs?: number; + fetchImpl?: typeof fetch; + onError?: (error: unknown) => void; + incarnationId?: string; +} + +const DEFAULT_LEASE_WAIT_MS = 25_000; +const MAX_LEASE_WAIT_MS = 30_000; +const DEFAULT_LEASE_TRANSPORT_GRACE_MS = 5_000; +const DEFAULT_RECONNECT_DELAY_MS = 1_000; +const DEFAULT_REGISTRATION_TTL_MS = 60_000; +const DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS = 10_000; +const DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS = 10_000; +const DEFAULT_CANCELLATION_POLL_INTERVAL_MS = 500; +const DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS = 2_000; +const MIN_REGISTRATION_HEARTBEAT_MS = 25; +const REGISTRATION_RETRY_DELAY_MS = 100; +const SETTLEMENT_RETRY_DELAY_MS = 100; +const REJECTION_ACK_GRACE_MS = 30_000; +const MAX_SETTLEMENT_ERROR_LENGTH = 4_096; +const RUNTIME_SESSION_PLACEHOLDER = '{runtimeSessionId}'; + +function normalizedBaseUrl(value: string): string { + return value.replace(/\/+$/, ''); +} + +function errorMessage(value: object): string | undefined { + if ('error' in value && typeof value.error === 'string') return value.error; + return undefined; +} + +function errorCode(value: object): string | undefined { + if ('code' in value && typeof value.code === 'string') return value.code; + return undefined; +} + +export class BridgeWorkspaceQuarantinedError extends Error { + constructor( + message: string, + public readonly cause?: unknown, + ) { + super(message); + this.name = 'BridgeWorkspaceQuarantinedError'; + } +} + +export class BridgeWorker { + private readonly fetchImpl: typeof fetch; + private readonly codeApiUrl: string; + private readonly sandboxEndpoint: string; + private readonly incarnationId: string; + private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; + private lastRegisteredAtMs = 0; + + constructor(private readonly options: BridgeWorkerOptions) { + this.fetchImpl = options.fetchImpl ?? fetch; + this.codeApiUrl = normalizedBaseUrl(options.codeApiUrl); + this.sandboxEndpoint = normalizedBaseUrl(options.sandboxEndpoint); + this.incarnationId = + options.incarnationId ?? randomBytes(18).toString('base64url'); + } + + async register( + signal?: AbortSignal, + ): Promise { + const registrationController = new AbortController(); + const abortRegistration = (): void => registrationController.abort(); + if (signal?.aborted) { + abortRegistration(); + } else { + signal?.addEventListener('abort', abortRegistration, { once: true }); + } + const timeoutMs = Math.min( + Math.max(1, this.registrationTtlMs - 1), + Math.max( + 1, + this.options.registrationTransportTimeoutMs ?? + DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS, + ), + ); + const timeout = setTimeout(abortRegistration, timeoutMs); + const registrationStartedAtMs = Date.now(); + let registration: BridgeWorkerRegistrationResponse; + try { + registration = await this.request( + `${this.codeApiUrl}/bridge/workers/register`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: this.options.workerId, + incarnationId: this.incarnationId, + capabilities: this.options.capabilities, + }, + registrationController.signal, + ); + } finally { + clearTimeout(timeout); + signal?.removeEventListener('abort', abortRegistration); + } + if (registration.incarnationId !== this.incarnationId) { + throw new BridgeProtocolError( + 'Code API registered a different worker incarnation', + ); + } + this.registrationTtlMs = registration.leaseTtlMs; + this.lastRegisteredAtMs = registrationStartedAtMs; + return registration; + } + + async resetWorkspace( + runtimeSessionId: string, + signal?: AbortSignal, + ): Promise { + if (runtimeSessionId.trim().length === 0) { + throw new BridgeProtocolError('Runtime session ID is required'); + } + await this.timedRequest( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId: this.incarnationId, + runtimeSessionId, + confirmDiscarded: true, + }, + Math.max( + 1, + this.options.resetTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, + ), + signal, + ); + } + + async lease(signal?: AbortSignal): Promise { + const waitMs = Math.min( + MAX_LEASE_WAIT_MS, + Math.max(0, this.options.leaseWaitMs ?? DEFAULT_LEASE_WAIT_MS), + ); + const leaseController = new AbortController(); + const abortLease = (): void => leaseController.abort(); + if (signal?.aborted) { + abortLease(); + } else { + signal?.addEventListener('abort', abortLease, { once: true }); + } + const timeout = setTimeout( + abortLease, + waitMs + + Math.max( + 0, + this.options.leaseTransportGraceMs ?? + DEFAULT_LEASE_TRANSPORT_GRACE_MS, + ), + ); + let response: BridgeLeaseResponse; + const requestStartedAtMs = Date.now(); + try { + response = await this.request( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/lease`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + waitMs, + incarnationId: this.incarnationId, + }, + leaseController.signal, + ); + } finally { + clearTimeout(timeout); + signal?.removeEventListener('abort', abortLease); + } + if ( + response.assignment != null && + response.assignment.incarnationId !== this.incarnationId + ) { + throw new BridgeProtocolError( + 'Code API leased an assignment for a different worker incarnation', + ); + } + if ( + response.assignment != null && + (!Number.isSafeInteger(response.assignment.remainingMs) || + (response.assignment.remainingMs ?? -1) < 0) + ) { + throw new BridgeProtocolError( + 'Code API leased an assignment without a valid server-relative deadline', + ); + } + if (response.assignment == null) return undefined; + if ( + !Number.isSafeInteger(response.serverElapsedMs) || + (response.serverElapsedMs ?? -1) < 0 + ) { + throw new BridgeProtocolError( + 'Code API leased an assignment without valid server timing', + ); + } + const transportElapsedMs = Math.max( + 0, + Date.now() - requestStartedAtMs - (response.serverElapsedMs ?? 0), + ); + const adjustedAssignment = { + ...response.assignment, + remainingMs: Math.max( + 0, + (response.assignment.remainingMs ?? 0) - transportElapsedMs, + ), + }; + const acknowledgementStartedAtMs = Date.now(); + try { + await this.timedRequest( + this.assignmentUrl(adjustedAssignment, 'ack'), + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId: this.incarnationId, + generation: adjustedAssignment.generation, + leaseToken: adjustedAssignment.leaseToken, + }, + Math.max( + 1, + this.options.leaseAckTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, + ), + signal, + ); + } catch (error) { + const definiteRejection = + error instanceof BridgeProtocolError && + error.status != null && + error.status < 500 && + error.status !== 408 && + error.status !== 429; + if (!definiteRejection) { + await this.rejectUnexecutedAssignment( + adjustedAssignment, + 'Bridge lease acknowledgement delivery was ambiguous', + ); + } + throw error; + } + const remainingMs = Math.max( + 0, + (adjustedAssignment.remainingMs ?? 0) - + (Date.now() - acknowledgementStartedAtMs), + ); + if (remainingMs <= 0) { + await this.rejectUnexecutedAssignment( + adjustedAssignment, + 'Bridge assignment expired during lease acknowledgement', + ); + throw new BridgeProtocolError( + 'Bridge assignment expired during lease acknowledgement', + ); + } + return { + ...adjustedAssignment, + remainingMs, + }; + } + + async run(signal?: AbortSignal): Promise { + while (!signal?.aborted) { + try { + await this.register(signal); + const assignment = await this.lease(signal); + if (!assignment) continue; + await this.executeAndSettle(assignment, signal); + } catch (error) { + if (error instanceof BridgeWorkspaceQuarantinedError) { + throw error; + } + if (signal?.aborted) return; + if ( + error instanceof BridgeProtocolError && + (error.status === 401 || + error.status === 403 || + error.code === 'WORKER_FENCED' || + error.code === 'WORKER_QUARANTINED') + ) { + throw error; + } + this.options.onError?.(error); + const delay = + this.options.reconnectDelayMs ?? DEFAULT_RECONNECT_DELAY_MS; + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + } + + async executeAndSettle( + assignment: BridgeAssignment, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted === true) { + throw signal.reason instanceof Error + ? signal.reason + : new DOMException('aborted', 'AbortError'); + } + const executionController = new AbortController(); + const abortExecution = (): void => executionController.abort(); + signal?.addEventListener('abort', abortExecution, { once: true }); + const deadlineDelay = this.assignmentRemainingMs(assignment); + const localDeadlineAtMs = Date.now() + deadlineDelay; + const deadlineTimer = setTimeout( + () => executionController.abort(), + deadlineDelay, + ); + if (this.lastRegisteredAtMs === 0) { + this.lastRegisteredAtMs = Date.now(); + } + const heartbeatController = new AbortController(); + let heartbeatError: unknown; + const heartbeat = this.maintainRegistration( + heartbeatController.signal, + ).catch((error) => { + heartbeatError = error; + executionController.abort(); + }); + const cancellationController = new AbortController(); + const cancellationWatcher = this.watchCancellation( + assignment, + executionController, + cancellationController.signal, + ); + let settlement: BridgeSettlement; + let ambiguousSandboxError: unknown; + let sandboxRejectedExecution = false; + try { + const sandboxSessionId = this.sandboxSessionIdFor(assignment); + const headers = { + ...assignment.request.headers, + ...(sandboxSessionId + ? { 'X-Runtime-Session-Id': sandboxSessionId } + : {}), + }; + const response = await this.fetchImpl( + `${this.sandboxEndpointFor(assignment)}/execute`, + { + method: 'POST', + headers: { + ...headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(assignment.request.body), + signal: executionController.signal, + }, + ); + let payload: object = {}; + try { + payload = (await response.json()) as object; + } catch (error) { + if (response.ok) throw error; + } + if (!response.ok) { + sandboxRejectedExecution = + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 && + errorMessage(payload) !== 'session_workspace_dirty'; + throw new BridgeProtocolError( + errorMessage(payload) ?? + `Sandbox rejected execution with HTTP ${response.status}`, + response.status, + ); + } + if (heartbeatError != null) throw heartbeatError; + settlement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + incarnationId: this.incarnationId, + status: 'fulfilled', + result: payload, + }; + } catch (error) { + if ( + assignment.runtimeSessionId != null && + !sandboxRejectedExecution + ) { + ambiguousSandboxError = error; + } + settlement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + incarnationId: this.incarnationId, + status: 'rejected', + error: + (error instanceof Error + ? error.message + : 'Sandbox execution failed' + ).slice(0, MAX_SETTLEMENT_ERROR_LENGTH), + }; + } + + clearTimeout(deadlineTimer); + cancellationController.abort(); + await cancellationWatcher; + try { + if (ambiguousSandboxError != null) { + throw new BridgeWorkspaceQuarantinedError( + `Stateful workspace ${assignment.runtimeSessionId} was quarantined after an ambiguous sandbox execution`, + ambiguousSandboxError, + ); + } + const knownCleanStatefulRejection = + assignment.runtimeSessionId != null && + settlement.status === 'rejected' && + sandboxRejectedExecution; + if (knownCleanStatefulRejection) { + heartbeatController.abort(); + await heartbeat; + const recoveryHeartbeatController = new AbortController(); + const recoveryHeartbeat = this.maintainRegistration( + recoveryHeartbeatController.signal, + true, + ).catch(() => undefined); + try { + await this.settleWithRetry( + assignment, + settlement, + localDeadlineAtMs + + Math.max( + 0, + this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, + ), + ); + } finally { + recoveryHeartbeatController.abort(); + await recoveryHeartbeat; + } + } else { + await this.settleWithRetry( + assignment, + settlement, + localDeadlineAtMs, + signal, + ); + } + } finally { + heartbeatController.abort(); + await heartbeat; + signal?.removeEventListener('abort', abortExecution); + } + } + + private sandboxSessionIdFor( + assignment: BridgeAssignment, + ): string | undefined { + if (assignment.runtimeSessionId != null) { + return assignment.runtimeSessionId; + } + if (this.sandboxEndpoint.includes(RUNTIME_SESSION_PLACEHOLDER)) { + return `assignment-${assignment.assignmentId}`; + } + return undefined; + } + + private assignmentRemainingMs(assignment: BridgeAssignment): number { + if ( + Number.isSafeInteger(assignment.remainingMs) && + (assignment.remainingMs ?? -1) >= 0 + ) { + return assignment.remainingMs ?? 0; + } + return Math.max(0, Date.parse(assignment.expiresAt) - Date.now()); + } + + private sandboxEndpointFor(assignment: BridgeAssignment): string { + if (assignment.runtimeSessionId == null) { + if (!this.sandboxEndpoint.includes(RUNTIME_SESSION_PLACEHOLDER)) { + return this.sandboxEndpoint; + } + return this.sandboxEndpoint.replace( + RUNTIME_SESSION_PLACEHOLDER, + encodeURIComponent(`assignment-${assignment.assignmentId}`), + ); + } + if ( + this.options.capabilities.statefulWorkspace !== true || + !this.sandboxEndpoint.includes(RUNTIME_SESSION_PLACEHOLDER) + ) { + throw new BridgeProtocolError( + 'Stateful assignments require a sandbox endpoint template containing {runtimeSessionId}', + ); + } + return this.sandboxEndpoint.replace( + RUNTIME_SESSION_PLACEHOLDER, + encodeURIComponent(assignment.runtimeSessionId), + ); + } + + private async maintainRegistration( + signal: AbortSignal, + retryTransient = false, + ): Promise { + while (!signal.aborted) { + const heartbeatIntervalMs = Math.max( + MIN_REGISTRATION_HEARTBEAT_MS, + Math.floor(this.registrationTtlMs / 2), + ); + await this.delay( + Math.max( + 0, + this.lastRegisteredAtMs + heartbeatIntervalMs - Date.now(), + ), + signal, + ); + if (signal.aborted) return; + try { + await this.register(signal); + } catch (error) { + const terminal = + error instanceof BridgeProtocolError && + (error.status === 401 || + error.status === 403 || + error.code === 'WORKER_FENCED' || + error.code === 'WORKER_QUARANTINED'); + if (!retryTransient || terminal || signal.aborted) throw error; + await this.delay(REGISTRATION_RETRY_DELAY_MS, signal); + } + } + } + + private async rejectUnexecutedAssignment( + assignment: BridgeAssignment, + error: string, + ): Promise { + const heartbeatController = new AbortController(); + const heartbeat = this.maintainRegistration( + heartbeatController.signal, + true, + ).catch(() => undefined); + try { + await this.settleWithRetry( + assignment, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + incarnationId: this.incarnationId, + status: 'rejected', + error, + }, + Date.now() + + Math.max( + 0, + this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, + ), + ); + } finally { + heartbeatController.abort(); + await heartbeat; + } + } + + private async delay(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) return; + await new Promise((resolve) => { + const onAbort = (): void => { + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal.addEventListener('abort', onAbort, { once: true }); + }); + } + + private assignmentUrl(assignment: BridgeAssignment, action: string): string { + return ( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}` + + `/assignments/${encodeURIComponent(assignment.assignmentId)}/${action}` + ); + } + + private async settleWithRetry( + assignment: BridgeAssignment, + settlement: BridgeSettlement, + deadlineAtMs: number, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted === true) { + if (assignment.runtimeSessionId != null) { + throw new BridgeWorkspaceQuarantinedError( + `Stateful workspace ${assignment.runtimeSessionId} was quarantined before settlement during shutdown`, + signal.reason, + ); + } + throw signal.reason instanceof Error + ? signal.reason + : new DOMException('aborted', 'AbortError'); + } + const settlementController = new AbortController(); + const abortSettlement = (): void => settlementController.abort(); + signal?.addEventListener('abort', abortSettlement, { once: true }); + const deadlineTimer = setTimeout( + () => settlementController.abort(), + Math.max(0, deadlineAtMs - Date.now()), + ); + let lastError: unknown; + try { + while (!settlementController.signal.aborted) { + try { + await this.request( + this.assignmentUrl(assignment, 'settle'), + settlement, + settlementController.signal, + ); + return; + } catch (error) { + lastError = error; + if (signal?.aborted) break; + if ( + error instanceof BridgeProtocolError && + error.status != null && + error.status < 500 && + error.status !== 408 && + error.status !== 429 + ) { + if ( + assignment.runtimeSessionId != null && + settlement.status === 'fulfilled' + ) { + throw new BridgeWorkspaceQuarantinedError( + `Stateful workspace ${assignment.runtimeSessionId} was quarantined after Code API rejected its fulfilled settlement`, + error, + ); + } + throw error; + } + const remainingMs = deadlineAtMs - Date.now(); + if (remainingMs <= 0) break; + await this.delay( + Math.min(SETTLEMENT_RETRY_DELAY_MS, remainingMs), + settlementController.signal, + ); + } + } + } finally { + clearTimeout(deadlineTimer); + signal?.removeEventListener('abort', abortSettlement); + } + if ( + assignment.runtimeSessionId != null && + settlement.status === 'fulfilled' + ) { + throw new BridgeWorkspaceQuarantinedError( + `Stateful workspace ${assignment.runtimeSessionId} was quarantined after ambiguous settlement delivery`, + lastError, + ); + } + if (lastError instanceof Error) throw lastError; + throw new BridgeProtocolError('Bridge settlement deadline expired'); + } + + private async watchCancellation( + assignment: BridgeAssignment, + executionController: AbortController, + signal: AbortSignal, + ): Promise { + while (!signal.aborted && !executionController.signal.aborted) { + await this.delay( + Math.max( + 1, + this.options.cancellationPollIntervalMs ?? + DEFAULT_CANCELLATION_POLL_INTERVAL_MS, + ), + signal, + ); + if (signal.aborted || executionController.signal.aborted) return; + const pollController = new AbortController(); + const abortPoll = (): void => pollController.abort(); + signal.addEventListener('abort', abortPoll, { once: true }); + const timeout = setTimeout( + abortPoll, + Math.max( + 1, + this.options.cancellationTransportTimeoutMs ?? + DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS, + ), + ); + try { + const response = await this.request<{ cancelled: boolean }>( + this.assignmentUrl(assignment, 'cancellation'), + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId: this.incarnationId, + }, + pollController.signal, + ); + if (response.cancelled) { + executionController.abort(); + return; + } + } catch (error) { + if (signal.aborted) return; + if (error instanceof BridgeProtocolError && error.status === 404) { + executionController.abort(); + return; + } + } finally { + clearTimeout(timeout); + signal.removeEventListener('abort', abortPoll); + } + } + } + + private async request( + url: string, + body: object, + signal?: AbortSignal, + ): Promise { + const response = await this.fetchImpl(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.options.token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + signal, + }); + let payload: unknown; + try { + payload = await response.json(); + } catch (error) { + if (response.ok) throw error; + payload = {}; + } + if (!response.ok) { + const errorPayload = + typeof payload === 'object' && payload !== null ? payload : {}; + throw new BridgeProtocolError( + errorMessage(errorPayload) ?? + `Bridge request failed with HTTP ${response.status}`, + response.status, + errorCode(errorPayload), + ); + } + return payload as T; + } + + private async timedRequest( + url: string, + body: object, + timeoutMs: number, + signal?: AbortSignal, + ): Promise { + const controller = new AbortController(); + const abortRequest = (): void => controller.abort(); + if (signal?.aborted) { + abortRequest(); + } else { + signal?.addEventListener('abort', abortRequest, { once: true }); + } + const timeout = setTimeout(abortRequest, timeoutMs); + timeout.unref?.(); + try { + return await this.request(url, body, controller.signal); + } finally { + clearTimeout(timeout); + signal?.removeEventListener('abort', abortRequest); + } + } +} diff --git a/packages/code/tsconfig.json b/packages/code/tsconfig.json new file mode 100644 index 00000000..6c7ea362 --- /dev/null +++ b/packages/code/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "declaration": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src", + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/service/Dockerfile b/service/Dockerfile index 762790dc..00680111 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -18,6 +18,7 @@ COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY service/scripts ./scripts COPY shared /shared +COPY packages/code/src /packages/code/src COPY service/tsconfig.json ./ RUN bun build ./src/file-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' RUN bun build ./src/api-server.ts --minify --outdir .build-api --target bun --external '@opentelemetry/*' @@ -66,6 +67,7 @@ ENV NODE_ENV=development COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src COPY service/tsconfig.json ./ EXPOSE 3000 9230 CMD ["bun", "run", "--watch", "src/file-server.ts"] diff --git a/service/Dockerfile.api b/service/Dockerfile.api index 419bdc95..f1fdf9c8 100644 --- a/service/Dockerfile.api +++ b/service/Dockerfile.api @@ -18,6 +18,7 @@ RUN cd /temp/prod && bun install --frozen-lockfile --production FROM base AS builder COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src +COPY packages/code/src /packages/code/src COPY service/scripts ./scripts COPY shared /shared COPY service/tsconfig.json ./ @@ -46,6 +47,7 @@ FROM base AS development ENV NODE_ENV=development COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src +COPY packages/code/src /packages/code/src COPY shared /shared COPY service/tsconfig.json ./ EXPOSE 3112 9230 diff --git a/service/Dockerfile.local b/service/Dockerfile.local index f932deee..cbb7af13 100644 --- a/service/Dockerfile.local +++ b/service/Dockerfile.local @@ -17,6 +17,7 @@ FROM base AS builder COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src COPY service/tsconfig.json ./ RUN bun build ./src/local-api.ts --minify --outdir .build --target bun --external '@opentelemetry/*' @@ -35,5 +36,6 @@ ENV NODE_ENV=development COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src COPY service/tsconfig.json ./ CMD ["bun", "run", "--watch", "src/local-api.ts"] diff --git a/service/Dockerfile.node b/service/Dockerfile.node index 46f4c319..8a6006e4 100644 --- a/service/Dockerfile.node +++ b/service/Dockerfile.node @@ -8,6 +8,7 @@ RUN npm ci FROM base AS builder COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src COPY service/tsconfig.json ./ RUN npx tsc -p tsconfig.json @@ -26,6 +27,7 @@ FROM base AS development ENV NODE_ENV=development COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src COPY service/tsconfig.json ./ RUN npm install -g ts-node typescript EXPOSE 3000 9230 diff --git a/service/Dockerfile.service b/service/Dockerfile.service index 53931e71..3a89dc15 100644 --- a/service/Dockerfile.service +++ b/service/Dockerfile.service @@ -17,6 +17,7 @@ FROM base AS builder COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY shared /usr/src/shared +COPY packages/code/src /usr/src/packages/code/src COPY service/tsconfig.json ./ RUN bun build ./src/service-api.ts --outdir .build --target bun --external '@opentelemetry/*' diff --git a/service/Dockerfile.worker b/service/Dockerfile.worker index 9d1a4322..e99c16c4 100644 --- a/service/Dockerfile.worker +++ b/service/Dockerfile.worker @@ -19,6 +19,7 @@ RUN cd /temp/prod && bun install --frozen-lockfile --production FROM base AS builder COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src +COPY packages/code/src /packages/code/src COPY shared /shared COPY service/tsconfig.json ./ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' @@ -43,6 +44,7 @@ FROM base AS development ENV NODE_ENV=development COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src +COPY packages/code/src /packages/code/src COPY shared /shared COPY service/tsconfig.json ./ EXPOSE 3113 9230 diff --git a/service/rollup.config.js b/service/rollup.config.js index 2400f72d..0e7a8d8b 100644 --- a/service/rollup.config.js +++ b/service/rollup.config.js @@ -38,7 +38,11 @@ export default { commonjs(), typescript({ tsconfig: './tsconfig.esm.json', - include: ['src/**/*.ts', '../shared/telemetry-core.ts'], + include: [ + 'src/**/*.ts', + '../shared/telemetry-core.ts', + '../packages/code/src/protocol.ts', + ], sourceMap: true, declaration: false, declarationMap: false, diff --git a/service/src/api-server.ts b/service/src/api-server.ts index 78689826..8578460b 100644 --- a/service/src/api-server.ts +++ b/service/src/api-server.ts @@ -18,6 +18,7 @@ import { requestErrorLogger, requestNotFoundLogger } from './middleware/request- import { localAuth } from './auth/local'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; +import bridgeRouter from './bridge/router'; import { connection } from './queue'; import { metricsHandler } from './metrics'; import { httpMetricsMiddleware } from './middleware/httpMetrics'; @@ -51,6 +52,7 @@ app.get('/v1/health', async (_, res) => { } }); +v1.use('/bridge', bridgeRouter); v1.use(isLocalMode ? localAuth : apiKeyAuth); v1.use(serviceRouter); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts new file mode 100644 index 00000000..35960401 --- /dev/null +++ b/service/src/bridge/router.ts @@ -0,0 +1,381 @@ +import { timingSafeEqual } from 'crypto'; + +import { Router } from 'express'; +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import type { BridgeWorkerRegistration } from '../../../packages/code/src/protocol'; +import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; + +import { + BRIDGE_PROTOCOL_VERSION, + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, +} from '../../../packages/code/src/protocol'; +import { connection } from '../queue'; +import { env } from '../config'; +import { BridgeStoreError, RedisBridgeStore } from './store'; + +const INCARNATION_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; +const MAX_LEASE_WAIT_MS = 30_000; + +export const bridgeStore = new RedisBridgeStore(connection); + +function sameToken(left: string, right: string): boolean { + const leftBuffer = Buffer.from(left); + const rightBuffer = Buffer.from(right); + return ( + leftBuffer.length === rightBuffer.length && + timingSafeEqual(leftBuffer, rightBuffer) + ); +} + +function bridgeAuth(req: Request, res: Response, next: NextFunction): void { + if (!env.BRIDGE_TOKEN) { + res.status(503).json({ error: 'Code bridge is not configured' }); + return; + } + const token = + req + .header('Authorization') + ?.match(/^Bearer\s+(.+)$/i)?.[1] + ?.trim() ?? ''; + if (!token || !sameToken(token, env.BRIDGE_TOKEN)) { + res.status(401).json({ error: 'Invalid code bridge worker token' }); + return; + } + next(); +} + +function validWorkerId(value: string): boolean { + return isValidBridgeWorkerId(value); +} + +function validIncarnationId(value: unknown): value is string { + return typeof value === 'string' && INCARNATION_ID_PATTERN.test(value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function asyncRoute( + handler: (req: Request, res: Response) => Promise, +): RequestHandler { + return (req, res, next) => { + void handler(req, res).catch(next); + }; +} + +function sendStoreError(error: BridgeStoreError, res: Response): void { + const status = + error.code === 'ASSIGNMENT_NOT_FOUND' + ? 404 + : error.code === 'WORKER_BUSY' + ? 503 + : 409; + res.status(status).json({ error: error.message, code: error.code }); +} + +function isSettlement(value: unknown): value is CodeBridgeSettlement { + if (!isRecord(value)) return false; + if ( + value.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + typeof value.generation !== 'number' || + !Number.isSafeInteger(value.generation) || + value.generation < 1 || + typeof value.leaseToken !== 'string' || + value.leaseToken.length < 32 || + !validIncarnationId(value.incarnationId) + ) { + return false; + } + if (value.status === 'rejected') { + return typeof value.error === 'string' && value.error.length <= 4096; + } + return ( + value.status === 'fulfilled' && + typeof value.result === 'object' && + value.result !== null + ); +} + +const router = Router(); +router.use(bridgeAuth); + +router.post( + '/workers/register', + asyncRoute(async (req, res) => { + const registration = req.body as unknown; + if ( + !isRecord(registration) || + registration.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + typeof registration.workerId !== 'string' || + !validWorkerId(registration.workerId) || + !validIncarnationId(registration.incarnationId) || + !isValidBridgeWorkerCapabilities(registration.capabilities) + ) { + res.status(400).json({ error: 'Invalid bridge worker registration' }); + return; + } + if ( + env.BRIDGE_WORKER_ID && + registration.workerId !== env.BRIDGE_WORKER_ID + ) { + res.status(403).json({ + error: 'Worker is not authorized for this Code API deployment', + }); + return; + } + try { + await bridgeStore.register( + registration as unknown as BridgeWorkerRegistration, + ); + } catch (error) { + if (error instanceof BridgeStoreError) { + sendStoreError(error, res); + return; + } + throw error; + } + res.json({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: registration.workerId, + incarnationId: registration.incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }), +); + +router.post( + '/workers/:workerId/workspaces/reset', + asyncRoute(async (req, res) => { + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !validIncarnationId(body.incarnationId) || + typeof body.runtimeSessionId !== 'string' || + body.runtimeSessionId.trim().length === 0 || + body.runtimeSessionId.length > 512 || + body.confirmDiscarded !== true + ) { + res.status(400).json({ + error: 'Workspace reset requires confirmation of local discard', + }); + return; + } + if (env.BRIDGE_WORKER_ID && workerId !== env.BRIDGE_WORKER_ID) { + res.status(403).json({ + error: 'Worker is not authorized for this Code API deployment', + }); + return; + } + try { + const resetController = new AbortController(); + const abortReset = (): void => resetController.abort(); + req.once('aborted', abortReset); + res.once('close', abortReset); + try { + await bridgeStore.resetWorkspace( + workerId, + body.incarnationId, + body.runtimeSessionId, + resetController.signal, + ); + if (!resetController.signal.aborted) { + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, reset: true }); + } + } finally { + req.off('aborted', abortReset); + res.off('close', abortReset); + } + } catch (error) { + if (error instanceof BridgeStoreError) { + sendStoreError(error, res); + return; + } + throw error; + } + }), +); + +router.post( + '/workers/:workerId/lease', + asyncRoute(async (req, res) => { + const requestStartedAtMs = Date.now(); + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + const requestedWait = Number(body.waitMs ?? 25_000); + if ( + !validWorkerId(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !validIncarnationId(body.incarnationId) || + !Number.isFinite(requestedWait) || + requestedWait < 0 + ) { + res.status(400).json({ error: 'Invalid bridge lease request' }); + return; + } + if (env.BRIDGE_WORKER_ID && workerId !== env.BRIDGE_WORKER_ID) { + res.status(403).json({ + error: 'Worker is not authorized for this Code API deployment', + }); + return; + } + try { + const leaseController = new AbortController(); + const abortLease = (): void => leaseController.abort(); + req.once('aborted', abortLease); + res.once('close', abortLease); + let assignment: CodeBridgeAssignment | undefined; + try { + assignment = await bridgeStore.lease( + workerId, + body.incarnationId, + Math.min(requestedWait, MAX_LEASE_WAIT_MS), + leaseController.signal, + ); + if (leaseController.signal.aborted) { + if (assignment != null) await bridgeStore.returnLease(assignment); + return; + } + res.json({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + serverElapsedMs: Math.max(0, Date.now() - requestStartedAtMs), + assignment, + }); + } finally { + req.off('aborted', abortLease); + res.off('close', abortLease); + } + } catch (error) { + if (error instanceof BridgeStoreError) { + sendStoreError(error, res); + return; + } + throw error; + } + }), +); + +router.post( + '/workers/:workerId/assignments/:assignmentId/ack', + asyncRoute(async (req, res) => { + const body = isRecord(req.body) ? req.body : {}; + if ( + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !validIncarnationId(body.incarnationId) || + !Number.isSafeInteger(body.generation) || + Number(body.generation) < 1 || + typeof body.leaseToken !== 'string' || + body.leaseToken.length < 32 + ) { + res.status(400).json({ error: 'Invalid bridge lease acknowledgement' }); + return; + } + try { + const acknowledgementController = new AbortController(); + const abortAcknowledgement = (): void => + acknowledgementController.abort(); + req.once('aborted', abortAcknowledgement); + res.once('close', abortAcknowledgement); + try { + await bridgeStore.acknowledgeLease( + req.params.workerId, + body.incarnationId, + req.params.assignmentId, + Number(body.generation), + body.leaseToken, + acknowledgementController.signal, + ); + if (!acknowledgementController.signal.aborted) { + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, accepted: true }); + } + } finally { + req.off('aborted', abortAcknowledgement); + res.off('close', abortAcknowledgement); + } + } catch (error) { + if (error instanceof BridgeStoreError) { + sendStoreError(error, res); + return; + } + throw error; + } + }), +); + +router.post( + '/workers/:workerId/assignments/:assignmentId/settle', + asyncRoute(async (req, res) => { + const settlement = req.body as unknown; + if (!isSettlement(settlement)) { + res.status(400).json({ error: 'Invalid bridge settlement' }); + return; + } + try { + const settlementController = new AbortController(); + const abortSettlement = (): void => settlementController.abort(); + req.once('aborted', abortSettlement); + res.once('close', abortSettlement); + try { + await bridgeStore.settle( + req.params.workerId, + req.params.assignmentId, + settlement, + settlementController.signal, + ); + if (!settlementController.signal.aborted) { + res.json({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + accepted: true, + }); + } + } finally { + req.off('aborted', abortSettlement); + res.off('close', abortSettlement); + } + } catch (error) { + if (error instanceof BridgeStoreError) { + sendStoreError(error, res); + return; + } + throw error; + } + }), +); + +router.post( + '/workers/:workerId/assignments/:assignmentId/cancellation', + asyncRoute(async (req, res) => { + const body = isRecord(req.body) ? req.body : {}; + if ( + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !validIncarnationId(body.incarnationId) + ) { + res.status(400).json({ error: 'Invalid bridge cancellation request' }); + return; + } + const cancellationController = new AbortController(); + const abortCancellation = (): void => cancellationController.abort(); + req.once('aborted', abortCancellation); + res.once('close', abortCancellation); + try { + const cancelled = await bridgeStore.cancelled( + req.params.workerId, + body.incarnationId, + req.params.assignmentId, + cancellationController.signal, + ); + if (!cancellationController.signal.aborted) { + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, cancelled }); + } + } finally { + req.off('aborted', abortCancellation); + res.off('close', abortCancellation); + } + }), +); + +export default router; diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts new file mode 100644 index 00000000..24637a20 --- /dev/null +++ b/service/src/bridge/store.test.ts @@ -0,0 +1,1813 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { getEventListeners } from 'node:events'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import type * as t from '../types'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const store = new RedisBridgeStore(redis); +const incarnationId = 'incarnation-00000001'; +const redisEval = redis.eval.bind(redis); +const redisDel = redis.del.bind(redis); +const redisLpop = redis.lpop.bind(redis); +const redisGet = redis.get.bind(redis); + +afterEach(async () => { + redis.eval = redisEval as Redis['eval']; + redis.del = redisDel as Redis['del']; + redis.lpop = redisLpop as Redis['lpop']; + redis.get = redisGet as Redis['get']; + await redis.flushall(); +}); + +describe('RedisBridgeStore', () => { + test('delivers and settles one fenced stateful assignment', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: { 'X-Execution-Manifest': 'signed' }, + runtimeSessionId: 'rt-user-1', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease('vm-1', incarnationId, 1_000); + expect(assignment).toBeDefined(); + expect(assignment?.runtimeSessionId).toBe('rt-user-1'); + expect(assignment?.remainingMs).toBeGreaterThan(0); + expect(assignment?.remainingMs).toBeLessThanOrEqual(5_000); + + await store.settle('vm-1', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-1', + files: [], + }, + }); + + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { session_id: 'run-1' }, + }); + }); + + test('redelivers a lease claim until the worker acknowledges it', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'claim-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const completion = store.dispatch({ + workerId: 'claim-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const first = await store.lease('claim-worker', incarnationId, 1_000); + const redelivered = await store.lease( + 'claim-worker', + incarnationId, + 1_000, + ); + expect(redelivered?.assignmentId).toBe(first?.assignmentId); + + await store.acknowledgeLease( + 'claim-worker', + incarnationId, + first?.assignmentId ?? '', + first?.generation ?? 0, + first?.leaseToken ?? '', + ); + await store.settle('claim-worker', first?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: first?.generation ?? 0, + leaseToken: first?.leaseToken ?? '', + incarnationId, + status: 'rejected', + error: 'test complete', + }); + await expect(completion).resolves.toMatchObject({ status: 'rejected' }); + }); + + test('performs one immediate lease poll when wait is zero', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'nonblocking-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const completion = store.dispatch({ + workerId: 'nonblocking-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + for (let attempt = 0; attempt < 100; attempt += 1) { + if ( + ( + await redis.keys( + 'codeapi:bridge:v1:assignment:*', + ) + ).length > 0 + ) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 1)); + } + + const assignment = await store.lease( + 'nonblocking-worker', + incarnationId, + 0, + ); + expect(assignment).toBeDefined(); + await store.settle('nonblocking-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'rejected', + error: 'test complete', + }); + await expect(completion).resolves.toMatchObject({ status: 'rejected' }); + }); + + test('bounds a stalled Redis lease claim', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.eval = (() => new Promise(() => undefined)) as Redis['eval']; + + await expect( + timedStore.lease('stalled-worker', incarnationId, 0), + ).rejects.toThrow('Bridge lease claim timed out'); + }); + + test('bounds a stalled Redis worker registration', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.eval = (() => new Promise(() => undefined)) as Redis['eval']; + + await expect( + timedStore.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'stalled-registration-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).rejects.toThrow('Bridge worker registration timed out'); + }); + + test('bounds stalled Redis reads during cancellation polling', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.get = (() => new Promise(() => undefined)) as Redis['get']; + + await expect( + timedStore.cancelled( + 'stalled-cancellation-worker', + incarnationId, + 'assignment-stalled-cancellation', + ), + ).rejects.toThrow('Bridge cancellation assignment read timed out'); + }); + + test('bounds stalled Redis reads during lease acknowledgement', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.get = (() => new Promise(() => undefined)) as Redis['get']; + + await expect( + timedStore.acknowledgeLease( + 'stalled-ack-worker', + incarnationId, + 'assignment-stalled-ack', + 1, + 'lease-token-that-is-long-enough-for-testing', + ), + ).rejects.toThrow('Bridge acknowledgement assignment read timed out'); + }); + + test('bounds stalled Redis reads during settlement', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.get = (() => new Promise(() => undefined)) as Redis['get']; + + await expect( + timedStore.settle('stalled-settlement-worker', 'assignment-stalled', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + incarnationId, + status: 'rejected', + error: 'test', + }), + ).rejects.toThrow('Bridge settlement existing read timed out'); + }); + + test('bounds a stalled Redis workspace reset', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.eval = (() => new Promise(() => undefined)) as Redis['eval']; + + await expect( + timedStore.resetWorkspace( + 'stalled-reset-worker', + incarnationId, + 'rt-stalled-reset', + ), + ).rejects.toThrow('Bridge workspace reset timed out'); + }); + + test('encodes worker IDs so Redis key families cannot collide', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'foo', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'foo', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease('foo', incarnationId, 1_000); + + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'foo:lock', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + expect( + await redis.get('codeapi:bridge:v1:worker:foo:lock'), + ).toBe(assignment?.assignmentId ?? null); + expect( + await redis.get('codeapi:bridge:v1:worker:foo%3Alock'), + ).not.toBeNull(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('retains cancellation through the assignment lifetime', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'cancel-ttl-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'cancel-ttl-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 120_000, + signal: controller.signal, + }); + const assignment = await store.lease( + 'cancel-ttl-worker', + incarnationId, + 1_000, + ); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + + expect( + await redis.ttl( + `codeapi:bridge:v1:assignment:${assignment?.assignmentId}:cancelled`, + ), + ).toBeGreaterThan(30); + }); + + test('bounds a stalled quarantine command', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 10); + redis.eval = (() => new Promise(() => undefined)) as Redis['eval']; + + await expect( + timedStore.quarantine('stalled-worker', incarnationId, 'rt-user-1'), + ).rejects.toThrow('Bridge worker quarantine timed out'); + }); + + test('rejects dispatch to an offline worker', async () => { + const controller = new AbortController(); + await expect( + store.dispatch({ + workerId: 'offline', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: controller.signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_OFFLINE' }); + }); + + test('does not fence a workspace when dispatch is already aborted', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + controller.abort(); + await expect( + store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-aborted', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + expect( + await redis.keys( + 'codeapi:bridge:v1:worker:vm-1:workspace:*:quarantined', + ), + ).toHaveLength(0); + }); + + test('does not fence a workspace when dispatch aborts during lock acquisition', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + redis.eval = (async (...args: Parameters) => { + const result = await redisEval(...args); + if (String(args[0]).includes("EXISTS', KEYS[1]) == 1")) { + controller.abort(); + } + return result; + }) as Redis['eval']; + + await expect( + store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-aborted-lock', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + expect( + await redis.keys( + 'codeapi:bridge:v1:worker:vm-1:workspace:*:quarantined', + ), + ).toHaveLength(0); + expect(await redis.exists('codeapi:bridge:v1:worker:vm-1:lock')).toBe(0); + }); + + test('clears a workspace fence when a queued assignment expires undelivered', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-expired-queue', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + const queue = + 'codeapi:bridge:v1:worker:vm-1:incarnation:' + + `${incarnationId}:assignments`; + const assignmentId = await redis.lindex(queue, 0); + const assignmentKey = `codeapi:bridge:v1:assignment:${assignmentId}`; + const rawAssignment = await redis.get(assignmentKey); + const assignment = JSON.parse(rawAssignment ?? '{}') as Record< + string, + unknown + >; + assignment.expiresAt = new Date(0).toISOString(); + await redis.set(assignmentKey, JSON.stringify(assignment), 'EX', 30); + + await expect( + store.lease('vm-1', incarnationId, 100), + ).resolves.toBeUndefined(); + expect( + await redis.keys( + 'codeapi:bridge:v1:worker:vm-1:workspace:*:quarantined', + ), + ).toHaveLength(0); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('preserves a workspace fence when an acknowledged lease expires', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'ack-expired-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'ack-expired-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-ack-expired', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease( + 'ack-expired-worker', + incarnationId, + 1_000, + ); + await store.acknowledgeLease( + 'ack-expired-worker', + incarnationId, + assignment?.assignmentId ?? '', + assignment?.generation ?? 0, + assignment?.leaseToken ?? '', + ); + const storedKey = `codeapi:bridge:v1:assignment:${assignment?.assignmentId}`; + const stored = JSON.parse( + (await redis.get(storedKey)) ?? '{}', + ) as Record; + stored.expiresAt = new Date(0).toISOString(); + await redis.set(storedKey, JSON.stringify(stored), 'EX', 30); + + await expect( + store.lease('ack-expired-worker', incarnationId, 0), + ).resolves.toBeUndefined(); + expect( + await redis.keys( + 'codeapi:bridge:v1:worker:ack-expired-worker:workspace:*:quarantined', + ), + ).toHaveLength(1); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('clears a workspace fence when dispatch cancels before lease', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-cancelled-queue', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + expect( + await redis.keys( + 'codeapi:bridge:v1:worker:vm-1:workspace:*:quarantined', + ), + ).toHaveLength(0); + expect( + await redis.llen( + `codeapi:bridge:v1:worker:vm-1:incarnation:${incarnationId}:assignments`, + ), + ).toBe(0); + }); + + test('returns a popped assignment when its lease request is aborted', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const dispatchController = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: dispatchController.signal, + }); + const leaseController = new AbortController(); + redis.eval = (async (...args: Parameters) => { + const result = await redisEval(...args); + if ( + String(args[0]).includes( + "local claimed = redis.call('GET', KEYS[2])", + ) && + result != null + ) { + leaseController.abort(); + } + return result; + }) as Redis['eval']; + + await expect( + store.lease('vm-1', incarnationId, 1_000, leaseController.signal), + ).resolves.toBeUndefined(); + redis.eval = redisEval as Redis['eval']; + + const recovered = await store.lease('vm-1', incarnationId, 1_000); + expect(recovered).toBeDefined(); + expect(recovered?.workerId).toBe('vm-1'); + dispatchController.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('restores queue expiry when returning a lease', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'returned-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'returned-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease( + 'returned-worker', + incarnationId, + 1_000, + ); + await store.returnLease(assignment!); + + expect( + await redis.ttl( + `codeapi:bridge:v1:worker:returned-worker:incarnation:${incarnationId}:assignments`, + ), + ).toBeGreaterThan(0); + expect( + await store.lease('returned-worker', incarnationId, 1_000), + ).toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('returns a popped assignment after a transient Redis read failure', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + let failAssignmentRead = true; + redis.get = (async (key: string) => { + if ( + failAssignmentRead && + key.includes(':assignment:') && + !key.endsWith(':settlement') + ) { + failAssignmentRead = false; + throw new Error('redis read failed'); + } + return await redisGet(key); + }) as Redis['get']; + + await expect(store.lease('vm-1', incarnationId, 1_000)).rejects.toThrow( + 'redis read failed', + ); + redis.get = redisGet as Redis['get']; + const recovered = await store.lease('vm-1', incarnationId, 1_000); + expect(recovered).toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('rejects a stale lease token', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease('vm-1', incarnationId, 1_000); + + await expect( + store.settle('vm-1', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: 'stale-token-that-is-long-enough-to-pass-validation', + incarnationId, + status: 'rejected', + error: 'unused', + }), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('fences a replaced worker incarnation', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_FENCED' }); + }); + + test('a stale incarnation poll cannot consume replacement work', async () => { + const replacementIncarnationId = 'incarnation-00000002'; + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'restarted-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const stalePoll = store.lease('restarted-worker', incarnationId, 100); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'restarted-worker', + incarnationId: replacementIncarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'restarted-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + + await expect(stalePoll).resolves.toBeUndefined(); + await expect( + store.lease('restarted-worker', replacementIncarnationId, 1_000), + ).resolves.toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('dispatch retries atomically against a replacement incarnation', async () => { + const workerId = 'racing-worker'; + const replacementIncarnationId = 'incarnation-00000002'; + const capabilities = { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [] as string[], + }; + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities, + }); + const originalEval = redis.eval.bind(redis); + let replaced = false; + redis.eval = (async (...args: Parameters) => { + if (!replaced && String(args[0]).includes("redis.call('RPUSH'")) { + replaced = true; + const replacement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId: replacementIncarnationId, + capabilities, + }; + await redis.set( + `codeapi:bridge:v1:worker:${workerId}`, + JSON.stringify(replacement), + 'EX', + 60, + ); + await redis.set( + `codeapi:bridge:v1:worker:${workerId}:incarnation`, + replacementIncarnationId, + 'EX', + 60, + ); + } + return originalEval(...args); + }) as Redis['eval']; + const controller = new AbortController(); + const completion = store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + + const assignment = await store.lease( + workerId, + replacementIncarnationId, + 1_000, + ); + expect(assignment?.incarnationId).toBe(replacementIncarnationId); + redis.eval = originalEval as Redis['eval']; + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('defers worker replacement while an assignment is active', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'busy-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'busy-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease('busy-worker', incarnationId, 1_000); + expect(assignment).toBeDefined(); + + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'busy-worker', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_BUSY' }); + + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'busy-worker', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).resolves.toBeUndefined(); + }); + + test('recovers only the assignment owner after registration expiry', async () => { + const workerId = 'expired-registration-worker'; + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease(workerId, incarnationId, 1_000); + expect(assignment).toBeDefined(); + await redis.del( + `codeapi:bridge:v1:worker:${workerId}`, + `codeapi:bridge:v1:worker:${workerId}:incarnation`, + ); + + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_BUSY' }); + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).resolves.toBeUndefined(); + + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('removes abort listeners after each settlement poll delay', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'listener-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'listener-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 350, + signal: controller.signal, + }); + + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0); + }); + + test('bounds a stalled Redis settlement poll command', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 20); + await timedStore.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'stalled-redis-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + redis.get = ((key: string) => { + if (key.endsWith(':settlement')) { + return new Promise(() => {}); + } + return redisGet(key); + }) as Redis['get']; + const controller = new AbortController(); + + await expect( + timedStore.dispatch({ + workerId: 'stalled-redis-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }), + ).rejects.toThrow('Bridge settlement poll timed out'); + }); + + test('bounds a stalled Redis dispatch preparation command', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 20); + await timedStore.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'stalled-preparation-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + redis.get = (() => new Promise(() => {})) as Redis['get']; + + await expect( + timedStore.dispatch({ + workerId: 'stalled-preparation-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('Bridge worker registration read timed out'); + }); + + test('keeps assignment state through deadlines longer than ten minutes', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'long-running-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'long-running-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 15 * 60_000, + signal: controller.signal, + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + const [assignmentKey] = await redis.keys('codeapi:bridge:v1:assignment:*'); + + expect(await redis.ttl(assignmentKey)).toBeGreaterThan(10 * 60); + expect( + await redis.pttl('codeapi:bridge:v1:worker:long-running-worker:lock'), + ).toBeGreaterThan(10 * 60_000); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('observes a settlement accepted during the final poll delay', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'deadline-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const deadlineAtMs = Date.now() + 500; + const completion = store.dispatch({ + workerId: 'deadline-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-deadline', + deadlineAtMs, + signal: controller.signal, + }); + const assignment = await store.lease( + 'deadline-worker', + incarnationId, + 1_000, + ); + expect(assignment).toBeDefined(); + await new Promise((resolve) => + setTimeout(resolve, Math.max(0, deadlineAtMs - Date.now() - 30)), + ); + await store.settle('deadline-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-deadline', + files: [], + }, + }); + + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { session_id: 'run-deadline' }, + }); + }); + + test('preserves a committed result across transient cleanup failures', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'cleanup-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'cleanup-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-cleanup', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease( + 'cleanup-worker', + incarnationId, + 1_000, + ); + let cleanupAttempts = 0; + redis.eval = (async (...args: Parameters) => { + if (String(args[0]).includes("local queued = redis.call('LREM'")) { + cleanupAttempts += 1; + if (cleanupAttempts === 1) { + throw new Error('transient cleanup failure'); + } + } + return await redisEval(...args); + }) as Redis['eval']; + await store.settle('cleanup-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-cleanup', + files: [], + }, + }); + + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { session_id: 'run-cleanup' }, + }); + expect(cleanupAttempts).toBeGreaterThanOrEqual(2); + redis.eval = redisEval as Redis['eval']; + }); + + test('holds a durable workspace marker until finalization commits', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'commit-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + let releaseFinalizer!: () => void; + const finalizerGate = new Promise((resolve) => { + releaseFinalizer = resolve; + }); + let finalizerStarted!: () => void; + const started = new Promise((resolve) => { + finalizerStarted = resolve; + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'commit-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-commit', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + finalize: async (settlement) => { + finalizerStarted(); + await finalizerGate; + return settlement; + }, + }); + const assignment = await store.lease( + 'commit-worker', + incarnationId, + 1_000, + ); + const settlement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled' as const, + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-commit', + files: [], + }, + }; + await store.settle( + 'commit-worker', + assignment?.assignmentId ?? '', + settlement, + ); + await started; + const [pendingMarker] = await redis.keys( + 'codeapi:bridge:v1:worker:commit-worker:workspace:*:quarantined', + ); + expect(pendingMarker).toBeDefined(); + expect(await redis.get(pendingMarker)).toBe( + assignment?.assignmentId ?? null, + ); + + releaseFinalizer(); + await expect(completion).resolves.toMatchObject({ status: 'fulfilled' }); + expect(await redis.exists(pendingMarker)).toBe(0); + await expect( + store.settle( + 'commit-worker', + assignment?.assignmentId ?? '', + settlement, + ), + ).resolves.toBeUndefined(); + expect(await redis.exists(pendingMarker)).toBe(0); + }); + + test('bounds a stalled Redis workspace commit command', async () => { + const timedStore = new RedisBridgeStore(redis, 60, 20); + await timedStore.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'stalled-commit-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + let releaseFinalizer!: () => void; + const finalizerGate = new Promise((resolve) => { + releaseFinalizer = resolve; + }); + let finalizerStarted!: () => void; + const started = new Promise((resolve) => { + finalizerStarted = resolve; + }); + const completion = timedStore.dispatch({ + workerId: 'stalled-commit-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-stalled-commit', + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + finalize: async (settlement) => { + finalizerStarted(); + await finalizerGate; + return settlement; + }, + }); + const assignment = await timedStore.lease( + 'stalled-commit-worker', + incarnationId, + 1_000, + ); + await timedStore.acknowledgeLease( + 'stalled-commit-worker', + incarnationId, + assignment?.assignmentId ?? '', + assignment?.generation ?? 0, + assignment?.leaseToken ?? '', + ); + await timedStore.settle( + 'stalled-commit-worker', + assignment?.assignmentId ?? '', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-stalled-commit', + files: [], + }, + }, + ); + await started; + redis.eval = ((...args: Parameters) => { + if ( + Number(args[1]) === 1 && + String(args[0]).includes("return redis.call('DEL', KEYS[1])") + ) { + return new Promise(() => {}); + } + return redisEval(...args); + }) as Redis['eval']; + releaseFinalizer(); + + await expect(completion).rejects.toThrow( + 'Bridge workspace commit timed out', + ); + }); + + test('keeps an in-flight workspace fenced when execution never settles', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lost-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'lost-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-lost', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease( + 'lost-worker', + incarnationId, + 1_000, + ); + expect(assignment).toBeDefined(); + await store.acknowledgeLease( + 'lost-worker', + incarnationId, + assignment?.assignmentId ?? '', + assignment?.generation ?? 0, + assignment?.leaseToken ?? '', + ); + const [marker] = await redis.keys( + 'codeapi:bridge:v1:worker:lost-worker:workspace:*:quarantined', + ); + expect(await redis.get(marker)).toBe(assignment?.assignmentId ?? null); + await expect( + store.resetWorkspace('lost-worker', incarnationId, 'rt-lost'), + ).rejects.toMatchObject({ code: 'WORKER_BUSY' }); + + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lost-worker', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_BUSY' }); + expect( + await redis.get('codeapi:bridge:v1:worker:lost-worker:lock'), + ).toBe(assignment?.assignmentId ?? null); + await redis.del( + 'codeapi:bridge:v1:worker:lost-worker:lock', + 'codeapi:bridge:v1:worker:lost-worker:lock:incarnation', + ); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lost-worker', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + await expect( + store.dispatch({ + workerId: 'lost-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-lost', + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + + await store.resetWorkspace( + 'lost-worker', + 'incarnation-00000002', + 'rt-lost', + ); + const recoveredController = new AbortController(); + const recoveredCompletion = store.dispatch({ + workerId: 'lost-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-lost', + deadlineAtMs: Date.now() + 5_000, + signal: recoveredController.signal, + }); + const recoveredAssignment = await store.lease( + 'lost-worker', + 'incarnation-00000002', + 1_000, + ); + expect(recoveredAssignment).toBeDefined(); + recoveredController.abort(); + await expect(recoveredCompletion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('clears an in-flight workspace marker after a definite rejection', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'rejected-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'rejected-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-rejected', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + const assignment = await store.lease( + 'rejected-worker', + incarnationId, + 1_000, + ); + const [marker] = await redis.keys( + 'codeapi:bridge:v1:worker:rejected-worker:workspace:*:quarantined', + ); + expect(await redis.get(marker)).toBe(assignment?.assignmentId ?? null); + await store.settle( + 'rejected-worker', + assignment?.assignmentId ?? '', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'rejected', + error: 'sandbox rejected before execution', + }, + ); + + await expect(completion).resolves.toMatchObject({ status: 'rejected' }); + expect(await redis.exists(marker)).toBe(0); + }); + + test('accepts a late clean rejection and recovers its workspace fence', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'late-rejection-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const completion = store.dispatch({ + workerId: 'late-rejection-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-late-rejection', + deadlineAtMs: Date.now() + 200, + signal: new AbortController().signal, + }); + const assignment = await store.lease( + 'late-rejection-worker', + incarnationId, + 1_000, + ); + await store.acknowledgeLease( + 'late-rejection-worker', + incarnationId, + assignment?.assignmentId ?? '', + assignment?.generation ?? 0, + assignment?.leaseToken ?? '', + ); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + + await store.settle( + 'late-rejection-worker', + assignment?.assignmentId ?? '', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'rejected', + error: 'syntax_error', + }, + ); + expect( + await redis.keys( + 'codeapi:bridge:v1:worker:late-rejection-worker:workspace:*:quarantined', + ), + ).toHaveLength(0); + }); + + test('atomically rejects a fulfillment committed after its deadline', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'late-fulfillment-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const deadlineAtMs = Date.now() + 250; + redis.eval = (async (...args: Parameters) => { + const script = String(args[0]); + if (script.includes("redis.call('RPUSH', KEYS[3], ARGV[4])")) { + await new Promise((resolve) => setTimeout(resolve, 75)); + } + if (script.includes("local existing = redis.call('GET', KEYS[2])")) { + await new Promise((resolve) => + setTimeout(resolve, Math.max(0, deadlineAtMs - Date.now() + 25)), + ); + } + return redisEval(...args); + }) as Redis['eval']; + const completion = store.dispatch({ + workerId: 'late-fulfillment-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-late-fulfillment', + deadlineAtMs, + signal: new AbortController().signal, + }); + const assignment = await store.lease( + 'late-fulfillment-worker', + incarnationId, + 1_000, + ); + await store.acknowledgeLease( + 'late-fulfillment-worker', + incarnationId, + assignment?.assignmentId ?? '', + assignment?.generation ?? 0, + assignment?.leaseToken ?? '', + ); + await expect( + store.settle('late-fulfillment-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-late-fulfillment', + files: [], + }, + }), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('releases the worker lock when generation allocation fails', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const originalIncr = redis.incr.bind(redis); + let failOnce = true; + redis.incr = (async (...args: Parameters) => { + if (failOnce) { + failOnce = false; + throw new Error('incr failed'); + } + return originalIncr(...args); + }) as Redis['incr']; + const controller = new AbortController(); + await expect( + store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: controller.signal, + }), + ).rejects.toThrow('incr failed'); + redis.incr = originalIncr as Redis['incr']; + + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: controller.signal, + }); + const assignment = await store.lease('vm-1', incarnationId, 500); + expect(assignment).toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('quarantines a workspace when result finalization fails', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-user-1', + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + finalize: async () => { + await expect( + store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-user-1', + deadlineAtMs: Date.now() + 1_000, + signal: controller.signal, + }), + ).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + throw new Error('restore failed'); + }, + }); + const assignment = await store.lease('vm-1', incarnationId, 1_000); + await store.settle('vm-1', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-1', + files: [], + }, + }); + + await expect(completion).rejects.toThrow('restore failed'); + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_QUARANTINED' }); + + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + await expect( + store.dispatch({ + workerId: 'vm-1', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + runtimeSessionId: 'rt-user-1', + deadlineAtMs: Date.now() + 1_000, + signal: controller.signal, + }), + ).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + }); + + test('does not quarantine a stateless worker when finalization fails', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'stateless-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'stateless-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + finalize: async () => { + throw new Error('restore failed'); + }, + }); + const assignment = await store.lease( + 'stateless-worker', + incarnationId, + 1_000, + ); + await store.settle('stateless-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + language: 'bash', + version: '5.2.0', + session_id: 'run-1', + files: [], + }, + }); + + await expect(completion).rejects.toThrow('restore failed'); + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'stateless-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts new file mode 100644 index 00000000..9b2dce13 --- /dev/null +++ b/service/src/bridge/store.ts @@ -0,0 +1,1300 @@ +import { createHash, randomBytes } from 'crypto'; + +import type Redis from 'ioredis'; +import type * as t from '../types'; +import type { + BridgeAssignment, + BridgeSettlement, + BridgeWorkerRegistration, +} from '../../../packages/code/src/protocol'; + +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; + +const PREFIX = 'codeapi:bridge:v1'; +const POLL_INTERVAL_MS = 100; +const DEFAULT_WORKER_TTL_SECONDS = 60; +const DEFAULT_REDIS_COMMAND_TIMEOUT_MS = 1_000; + +export type CodeBridgeAssignment = BridgeAssignment; +export type CodeBridgeSettlement = BridgeSettlement< + t.ExecuteResponse & { + session_id: string; + files?: t.FileRefs; + run?: t.ExecuteResponse['run']; + } +>; + +export class BridgeStoreError extends Error { + constructor( + public readonly code: + | 'WORKER_OFFLINE' + | 'WORKER_BUSY' + | 'ASSIGNMENT_EXPIRED' + | 'ASSIGNMENT_FENCED' + | 'ASSIGNMENT_NOT_FOUND' + | 'WORKER_FENCED' + | 'WORKER_QUARANTINED' + | 'WORKSPACE_QUARANTINED' + | 'WORKER_MISMATCH', + message: string, + ) { + super(message); + this.name = 'BridgeStoreError'; + } +} + +interface StoredAssignment extends CodeBridgeAssignment { + leaseTokenHash: string; +} + +function workerKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; +} + +function workerIncarnationKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`; +} + +function incarnationFenceKey(workerId: string, incarnationId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:fenced`; +} + +function quarantineKey(workerId: string, incarnationId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:quarantined`; +} + +function workspaceQuarantineKey( + workerId: string, + runtimeSessionId: string, +): string { + const sessionHash = createHash('sha256') + .update(runtimeSessionId) + .digest('hex'); + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:workspace:${sessionHash}:quarantined`; +} + +function queueKey(workerId: string, incarnationId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:assignments`; +} + +function leaseClaimKey(workerId: string, incarnationId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:lease-claim`; +} + +function leaseAckKey(workerId: string, incarnationId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:lease-ack`; +} + +function generationKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:generation`; +} + +function lockKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:lock`; +} + +function lockIncarnationKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:lock:incarnation`; +} + +function assignmentKey(assignmentId: string): string { + return `${PREFIX}:assignment:${assignmentId}`; +} + +function settlementKey(assignmentId: string): string { + return `${PREFIX}:assignment:${assignmentId}:settlement`; +} + +function assignmentDeadlineKey(assignmentId: string): string { + return `${PREFIX}:assignment:${assignmentId}:deadline`; +} + +function cancellationKey(assignmentId: string): string { + return `${PREFIX}:assignment:${assignmentId}:cancelled`; +} + +function tokenHash(token: string): string { + return createHash('sha256').update(token).digest('hex'); +} + +function assignmentTtlSeconds(deadlineAtMs: number): number { + return Math.max(1, Math.ceil((deadlineAtMs - Date.now()) / 1000) + 30); +} + +async function delay(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted === true) return; + await new Promise((resolve) => { + const onAbort = (): void => { + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +function signalAborted(signal?: AbortSignal): boolean { + return signal?.aborted === true; +} + +async function boundedCommand( + command: Promise, + timeoutMs: number, + label: string, + signal?: AbortSignal, +): Promise { + void command.catch(() => undefined); + return await new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + callback(); + }; + const onAbort = (): void => + finish(() => + reject( + signal?.reason instanceof Error + ? signal.reason + : new Error(`${label} aborted`), + ), + ); + const timer = setTimeout( + () => finish(() => reject(new Error(`${label} timed out`))), + timeoutMs, + ); + timer.unref?.(); + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) onAbort(); + command.then( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), + ); + }); +} + +export class RedisBridgeStore { + constructor( + private readonly redis: Redis, + private readonly workerTtlSeconds = DEFAULT_WORKER_TTL_SECONDS, + private readonly redisCommandTimeoutMs = DEFAULT_REDIS_COMMAND_TIMEOUT_MS, + ) {} + + private async dispatchCommand( + command: () => Promise, + args: { deadlineAtMs: number; signal: AbortSignal }, + label: string, + ): Promise { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + try { + return await boundedCommand( + command(), + Math.max( + 1, + Math.min(this.redisCommandTimeoutMs, args.deadlineAtMs - Date.now()), + ), + label, + args.signal, + ); + } catch (error) { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + throw error; + } + } + + private async leaseCommand( + command: Promise, + signal: AbortSignal | undefined, + label: string, + ): Promise { + return await boundedCommand( + command, + this.redisCommandTimeoutMs, + label, + signal, + ); + } + + async register(registration: BridgeWorkerRegistration): Promise { + const script = [ + 'if redis.call(\'EXISTS\', KEYS[3]) == 1 then return -2 end', + 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return -1 end', + 'local current = redis.call(\'GET\', KEYS[4])', + 'if not current and redis.call(\'EXISTS\', KEYS[5]) == 1 then', + ' local owner = redis.call(\'GET\', KEYS[6])', + ' if owner ~= ARGV[1] then return -3 end', + 'end', + 'if current then', + ' if current ~= ARGV[1] then', + ' if redis.call(\'EXISTS\', KEYS[5]) == 1 then return -3 end', + ' redis.call(\'SET\', ARGV[4] .. current .. \':fenced\', \"1\")', + ' end', + 'end', + 'redis.call(\'SET\', KEYS[1], ARGV[2], \"EX\", ARGV[3])', + 'redis.call(\'SET\', KEYS[4], ARGV[1], \"EX\", ARGV[3])', + 'return 1', + ].join('\n'); + const result = Number( + await boundedCommand( + this.redis.eval( + script, + 6, + workerKey(registration.workerId), + incarnationFenceKey(registration.workerId, registration.incarnationId), + quarantineKey(registration.workerId, registration.incarnationId), + workerIncarnationKey(registration.workerId), + lockKey(registration.workerId), + lockIncarnationKey(registration.workerId), + registration.incarnationId, + JSON.stringify(registration), + String(this.workerTtlSeconds), + `${PREFIX}:worker:${encodeURIComponent(registration.workerId)}:incarnation:`, + ), + this.redisCommandTimeoutMs, + 'Bridge worker registration', + ), + ); + if (result === -2) { + throw new BridgeStoreError( + 'WORKER_QUARANTINED', + 'Bridge worker incarnation is quarantined', + ); + } + if (result === -1) { + throw new BridgeStoreError( + 'WORKER_FENCED', + 'Bridge worker incarnation was replaced', + ); + } + if (result === -3) { + throw new BridgeStoreError( + 'WORKER_BUSY', + 'Bridge worker cannot be replaced during an active assignment', + ); + } + } + + async dispatch(args: { + workerId: string; + body: t.PayloadBody; + headers: Record; + runtimeSessionId?: string; + deadlineAtMs: number; + signal: AbortSignal; + finalize?: ( + settlement: CodeBridgeSettlement, + ) => Promise; + }): Promise { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + let registration = await this.dispatchCommand( + () => this.registration(args.workerId), + args, + 'Bridge worker registration read', + ); + if (registration == null) { + throw new BridgeStoreError( + 'WORKER_OFFLINE', + `Bridge worker ${args.workerId} is offline`, + ); + } + if ( + args.runtimeSessionId !== undefined && + registration.capabilities.statefulWorkspace !== true + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} does not provide a stateful workspace`, + ); + } + if ( + args.runtimeSessionId !== undefined && + (await this.dispatchCommand( + () => + this.redis.exists( + workspaceQuarantineKey(args.workerId, args.runtimeSessionId ?? ''), + ), + args, + 'Bridge workspace fence read', + )) === 1 + ) { + throw new BridgeStoreError( + 'WORKSPACE_QUARANTINED', + 'Bridge workspace is quarantined after an incomplete result commit', + ); + } + + const assignmentId = randomBytes(18).toString('base64url'); + const leaseToken = randomBytes(32).toString('base64url'); + const ttlSeconds = assignmentTtlSeconds(args.deadlineAtMs); + const lockIncarnationId = registration.incarnationId; + let assignment: StoredAssignment | undefined; + let resultCommitted = false; + try { + const locked = await this.dispatchCommand( + () => + this.acquireLock( + args.workerId, + assignmentId, + lockIncarnationId, + ttlSeconds, + ), + args, + 'Bridge assignment lock acquisition', + ); + if (!locked) { + throw new BridgeStoreError( + 'WORKER_BUSY', + `Bridge worker ${args.workerId} is busy`, + ); + } + this.assertDispatchActive(args.signal, args.deadlineAtMs); + const generation = await this.dispatchCommand( + () => this.redis.incr(generationKey(args.workerId)), + args, + 'Bridge assignment generation allocation', + ); + assignment = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + assignmentId, + workerId: args.workerId, + incarnationId: registration.incarnationId, + generation, + leaseToken, + leaseTokenHash: tokenHash(leaseToken), + expiresAt: new Date(args.deadlineAtMs).toISOString(), + runtimeSessionId: args.runtimeSessionId, + request: { + body: args.body, + headers: args.headers, + }, + }; + let queued = false; + for (let attempt = 0; attempt < 8 && !queued; attempt += 1) { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + assignment.incarnationId = registration.incarnationId; + queued = await this.dispatchCommand( + () => this.enqueueForActiveIncarnation(assignment!, ttlSeconds), + args, + 'Bridge assignment enqueue', + ); + if (queued) break; + const replacement = await this.dispatchCommand( + () => this.registration(args.workerId), + args, + 'Bridge replacement registration read', + ); + if (replacement == null) { + throw new BridgeStoreError( + 'WORKER_OFFLINE', + `Bridge worker ${args.workerId} went offline during dispatch`, + ); + } + if ( + args.runtimeSessionId !== undefined && + replacement.capabilities.statefulWorkspace !== true + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} does not provide a stateful workspace`, + ); + } + registration = replacement; + } + if (!queued) { + throw new BridgeStoreError( + 'WORKER_OFFLINE', + `Bridge worker ${args.workerId} changed incarnation repeatedly during dispatch`, + ); + } + const settlement = await this.waitForSettlement( + assignment, + args.deadlineAtMs, + args.signal, + ); + try { + const result = + args.finalize == null + ? settlement + : await args.finalize(settlement); + await this.commitPendingWorkspace( + assignment, + settlement, + args.deadlineAtMs, + args.signal, + ); + resultCommitted = true; + return result; + } catch (error) { + if (args.runtimeSessionId !== undefined) { + await this.quarantine( + args.workerId, + assignment.incarnationId, + args.runtimeSessionId, + ); + } + throw error; + } + } finally { + if (resultCommitted) { + try { + await this.cleanupWithRetry(args.workerId, assignmentId, assignment); + } catch { + // The lock and assignment have deadline-derived TTLs. Preserve the + // already committed result rather than turning cleanup availability + // into a client-visible failure that could prompt duplicate work. + } + } else { + await this.cleanupDispatch(args.workerId, assignmentId, assignment); + } + } + } + + async lease( + workerId: string, + incarnationId: string, + waitMs: number, + signal?: AbortSignal, + ): Promise { + const deadline = Date.now() + waitMs; + let firstPoll = true; + while ( + !signalAborted(signal) && + (firstPoll || Date.now() < deadline) + ) { + firstPoll = false; + let assignmentId: string | null; + try { + assignmentId = await this.leaseCommand( + this.claimOrPopLease(workerId, incarnationId), + signal, + 'Bridge lease claim', + ); + } catch (error) { + if (signalAborted(signal)) return undefined; + throw error; + } + if (assignmentId == null) { + await delay( + Math.min(POLL_INTERVAL_MS, Math.max(0, deadline - Date.now())), + signal, + ); + continue; + } + try { + const assignment = await this.leaseCommand( + this.readAssignment(assignmentId), + signal, + 'Bridge lease assignment read', + ); + if ( + assignment == null || + assignment.workerId !== workerId || + assignment.incarnationId !== incarnationId + ) { + await this.leaseCommand( + this.discardLeaseClaim(workerId, incarnationId, assignmentId), + signal, + 'Bridge lease claim discard', + ); + continue; + } + if (signalAborted(signal)) { + await this.returnLease(assignment); + return undefined; + } + const registration = await this.leaseCommand( + this.registration(workerId), + signal, + 'Bridge lease registration read', + ); + if (registration?.incarnationId !== incarnationId) { + throw new BridgeStoreError( + 'WORKER_FENCED', + 'Bridge worker incarnation was replaced', + ); + } + if (Date.parse(assignment.expiresAt) <= Date.now()) { + const acknowledged = + (await this.leaseCommand( + this.redis.get(leaseAckKey(workerId, incarnationId)), + signal, + 'Bridge lease acknowledgement read', + )) === assignmentId; + if (!acknowledged) { + await this.leaseCommand( + this.clearUndeliveredWorkspaceFence(assignment), + signal, + 'Bridge undelivered workspace recovery', + ); + } + await this.leaseCommand( + this.discardLeaseClaim(workerId, incarnationId, assignmentId), + signal, + 'Bridge expired lease discard', + ); + continue; + } + if (signalAborted(signal)) { + await this.returnLease(assignment); + return undefined; + } + const { leaseTokenHash: _leaseTokenHash, ...wireAssignment } = assignment; + return { + ...wireAssignment, + remainingMs: Math.max( + 0, + Date.parse(assignment.expiresAt) - Date.now(), + ), + }; + } catch (error) { + await this.returnLeaseByIdWithRetry( + workerId, + incarnationId, + assignmentId, + ); + if (signalAborted(signal)) return undefined; + throw error; + } + } + return undefined; + } + + async acknowledgeLease( + workerId: string, + incarnationId: string, + assignmentId: string, + generation: number, + leaseToken: string, + signal?: AbortSignal, + ): Promise { + const assignment = await this.leaseCommand( + this.readAssignment(assignmentId), + signal, + 'Bridge acknowledgement assignment read', + ); + const registration = await this.leaseCommand( + this.registration(workerId), + signal, + 'Bridge acknowledgement registration read', + ); + if ( + assignment == null || + assignment.workerId !== workerId || + assignment.incarnationId !== incarnationId || + registration?.incarnationId !== incarnationId || + assignment.generation !== generation || + tokenHash(leaseToken) !== assignment.leaseTokenHash + ) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Bridge assignment lease acknowledgement is stale', + ); + } + const ttlSeconds = assignmentTtlSeconds(Date.parse(assignment.expiresAt)); + const acknowledged = Number( + await this.leaseCommand( + this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", + "redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[2])", + 'return 1', + ].join('\n'), + 2, + leaseClaimKey(workerId, incarnationId), + leaseAckKey(workerId, incarnationId), + assignmentId, + String(ttlSeconds), + ), + signal, + 'Bridge lease acknowledgement', + ), + ); + if (acknowledged !== 1) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Bridge assignment is not the active lease claim', + ); + } + } + + private async claimOrPopLease( + workerId: string, + incarnationId: string, + ): Promise { + const result = await this.redis.eval( + [ + "local claimed = redis.call('GET', KEYS[2])", + 'if claimed then return claimed end', + "local ttl = redis.call('TTL', KEYS[1])", + "local assignment = redis.call('LPOP', KEYS[1])", + 'if not assignment then return nil end', + "redis.call('SET', KEYS[2], assignment, 'EX', math.max(1, ttl))", + 'return assignment', + ].join('\n'), + 2, + queueKey(workerId, incarnationId), + leaseClaimKey(workerId, incarnationId), + ); + return result == null ? null : String(result); + } + + private async discardLeaseClaim( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + await this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) == ARGV[1] then", + " return redis.call('DEL', KEYS[1], KEYS[2])", + 'end', + 'return 0', + ].join('\n'), + 2, + leaseClaimKey(workerId, incarnationId), + leaseAckKey(workerId, incarnationId), + assignmentId, + ); + } + + async returnLease(assignment: CodeBridgeAssignment): Promise { + await this.returnLeaseById( + assignment.workerId, + assignment.incarnationId, + assignment.assignmentId, + ); + } + + private async returnLeaseById( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + await boundedCommand( + this.redis.eval( + [ + "if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end", + "if redis.call('GET', KEYS[3]) ~= ARGV[1] then return 0 end", + "local ttl = redis.call('TTL', KEYS[1])", + "redis.call('DEL', KEYS[3], KEYS[4])", + "redis.call('LREM', KEYS[2], 0, ARGV[1])", + "redis.call('LPUSH', KEYS[2], ARGV[1])", + "if ttl > 0 then redis.call('EXPIRE', KEYS[2], ttl) end", + 'return 1', + ].join('\n'), + 4, + assignmentKey(assignmentId), + queueKey(workerId, incarnationId), + leaseClaimKey(workerId, incarnationId), + leaseAckKey(workerId, incarnationId), + assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge lease return', + ); + } + + private async returnLeaseByIdWithRetry( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + await this.returnLeaseById(workerId, incarnationId, assignmentId); + return; + } catch (error) { + lastError = error; + await delay(25); + } + } + throw lastError; + } + + private async clearUndeliveredWorkspaceFence( + assignment: StoredAssignment, + ): Promise { + if (assignment.runtimeSessionId === undefined) return; + await this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) == ARGV[1] then", + " return redis.call('DEL', KEYS[1])", + 'end', + 'return 0', + ].join('\n'), + 1, + workspaceQuarantineKey( + assignment.workerId, + assignment.runtimeSessionId, + ), + assignment.assignmentId, + ); + } + + async settle( + workerId: string, + assignmentId: string, + settlement: CodeBridgeSettlement, + signal?: AbortSignal, + ): Promise { + const serializedSettlement = JSON.stringify(settlement); + const existingSettlement = await this.leaseCommand( + this.redis.get(settlementKey(assignmentId)), + signal, + 'Bridge settlement existing read', + ); + if (existingSettlement === serializedSettlement) return; + if (existingSettlement != null) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Bridge assignment was already settled with a different result', + ); + } + const assignment = await this.leaseCommand( + this.readAssignment(assignmentId), + signal, + 'Bridge settlement assignment read', + ); + if (assignment == null) { + throw new BridgeStoreError( + 'ASSIGNMENT_NOT_FOUND', + 'Bridge assignment was not found', + ); + } + if (assignment.workerId !== workerId) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + 'Bridge assignment belongs to another worker', + ); + } + const registration = await this.leaseCommand( + this.registration(workerId), + signal, + 'Bridge settlement registration read', + ); + if ( + settlement.incarnationId !== assignment.incarnationId || + registration?.incarnationId !== settlement.incarnationId || + settlement.generation !== assignment.generation || + tokenHash(settlement.leaseToken) !== assignment.leaseTokenHash + ) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Bridge assignment lease is stale', + ); + } + if ( + settlement.status !== 'rejected' && + Date.parse(assignment.expiresAt) <= Date.now() + ) { + throw new BridgeStoreError( + 'ASSIGNMENT_EXPIRED', + 'Bridge assignment has expired', + ); + } + const ttlSeconds = assignmentTtlSeconds(Date.parse(assignment.expiresAt)); + const settlementKeys = [ + assignmentKey(assignmentId), + settlementKey(assignmentId), + leaseClaimKey(workerId, assignment.incarnationId), + leaseAckKey(workerId, assignment.incarnationId), + assignmentDeadlineKey(assignmentId), + ]; + if (assignment.runtimeSessionId !== undefined) { + settlementKeys.push( + workspaceQuarantineKey(workerId, assignment.runtimeSessionId), + ); + } + const script = [ + 'local existing = redis.call(\'GET\', KEYS[2])', + 'if existing then', + ' if existing == ARGV[1] then return 2 end', + ' return -1', + 'end', + 'if redis.call(\'EXISTS\', KEYS[1]) == 0 then return 0 end', + 'if #KEYS == 6 and redis.call(\'GET\', KEYS[6]) ~= ARGV[3] then return -2 end', + 'if ARGV[4] ~= "rejected" and redis.call(\'EXISTS\', KEYS[5]) == 0 then return -3 end', + 'redis.call(\'SET\', KEYS[2], ARGV[1], \"EX\", ARGV[2])', + 'if redis.call(\'GET\', KEYS[3]) == ARGV[3] then redis.call(\'DEL\', KEYS[3], KEYS[4]) end', + 'if #KEYS == 6 and ARGV[4] == \"rejected\" then redis.call(\'DEL\', KEYS[6]) end', + 'return 1', + ].join('\n'); + const accepted = Number( + await this.leaseCommand( + this.redis.eval( + script, + settlementKeys.length, + ...settlementKeys, + serializedSettlement, + String(ttlSeconds), + assignmentId, + settlement.status, + ), + signal, + 'Bridge settlement commit', + ), + ); + if (accepted === -1) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Bridge assignment was already settled with a different result', + ); + } + if (accepted === -2) { + throw new BridgeStoreError( + 'WORKSPACE_QUARANTINED', + 'Bridge workspace in-flight marker was lost before settlement', + ); + } + if (accepted === -3) { + throw new BridgeStoreError( + 'ASSIGNMENT_EXPIRED', + 'Bridge assignment expired before settlement was committed', + ); + } + if (accepted !== 1 && accepted !== 2) { + throw new BridgeStoreError( + 'ASSIGNMENT_EXPIRED', + 'Bridge assignment closed before settlement was committed', + ); + } + } + + async cancelled( + workerId: string, + incarnationId: string, + assignmentId: string, + signal?: AbortSignal, + ): Promise { + const assignment = await this.leaseCommand( + this.readAssignment(assignmentId), + signal, + 'Bridge cancellation assignment read', + ); + const registration = await this.leaseCommand( + this.registration(workerId), + signal, + 'Bridge cancellation registration read', + ); + if ( + assignment == null || + assignment.workerId !== workerId || + assignment.incarnationId !== incarnationId || + registration?.incarnationId !== incarnationId + ) { + return true; + } + return ( + (await this.leaseCommand( + this.redis.exists(cancellationKey(assignmentId)), + signal, + 'Bridge cancellation marker read', + )) === 1 + ); + } + + async quarantine( + workerId: string, + incarnationId: string, + runtimeSessionId?: string, + ): Promise { + const script = [ + 'redis.call(\'SET\', KEYS[2], \"1\")', + 'if #KEYS == 4 then redis.call(\'SET\', KEYS[4], \"1\") end', + 'local current = redis.call(\'GET\', KEYS[3])', + 'if current == ARGV[1] then', + ' return redis.call(\'DEL\', KEYS[1], KEYS[3])', + 'end', + 'return 0', + ].join('\n'); + const keys = [ + workerKey(workerId), + quarantineKey(workerId, incarnationId), + workerIncarnationKey(workerId), + ]; + if (runtimeSessionId !== undefined) { + keys.push(workspaceQuarantineKey(workerId, runtimeSessionId)); + } + await boundedCommand( + this.redis.eval( + script, + keys.length, + ...keys, + incarnationId, + ), + this.redisCommandTimeoutMs, + 'Bridge worker quarantine', + ); + } + + async resetWorkspace( + workerId: string, + incarnationId: string, + runtimeSessionId: string, + signal?: AbortSignal, + ): Promise { + const result = Number( + await this.leaseCommand( + this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end", + "if redis.call('EXISTS', KEYS[2]) == 1 then return -2 end", + "redis.call('DEL', KEYS[3])", + 'return 1', + ].join('\n'), + 3, + workerIncarnationKey(workerId), + lockKey(workerId), + workspaceQuarantineKey(workerId, runtimeSessionId), + incarnationId, + ), + signal, + 'Bridge workspace reset', + ), + ); + if (result === -1) { + throw new BridgeStoreError( + 'WORKER_FENCED', + 'Only the active bridge worker incarnation can reset a workspace', + ); + } + if (result === -2) { + throw new BridgeStoreError( + 'WORKER_BUSY', + 'Bridge workspace cannot be reset while worker execution is active', + ); + } + } + + private async registration( + workerId: string, + ): Promise { + const raw = await this.redis.get(workerKey(workerId)); + return raw == null ? undefined : (JSON.parse(raw) as BridgeWorkerRegistration); + } + + private assertDispatchActive( + signal: AbortSignal, + deadlineAtMs: number, + ): void { + if (signal.aborted || Date.now() >= deadlineAtMs) { + throw new BridgeStoreError( + 'ASSIGNMENT_EXPIRED', + 'Bridge assignment ended before it could be delivered', + ); + } + } + + private async readAssignment( + assignmentId: string, + ): Promise { + const raw = await this.redis.get(assignmentKey(assignmentId)); + return raw == null ? undefined : (JSON.parse(raw) as StoredAssignment); + } + + private async waitForSettlement( + assignment: StoredAssignment, + deadlineAtMs: number, + signal: AbortSignal, + ): Promise { + while (!signal.aborted && Date.now() < deadlineAtMs) { + const raw = await boundedCommand( + this.redis.get(settlementKey(assignment.assignmentId)), + Math.max( + 1, + Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), + ), + 'Bridge settlement poll', + signal, + ); + if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement; + await delay(POLL_INTERVAL_MS, signal); + } + const closeKeys = [ + assignmentKey(assignment.assignmentId), + settlementKey(assignment.assignmentId), + ]; + if (assignment.runtimeSessionId !== undefined) { + closeKeys.push( + workspaceQuarantineKey( + assignment.workerId, + assignment.runtimeSessionId, + ), + ); + } + const closeScript = [ + 'local settlement = redis.call(\'GET\', KEYS[2])', + 'if settlement then return settlement end', + 'if #KEYS == 3 and redis.call(\'GET\', KEYS[3]) == ARGV[1] then return nil end', + 'redis.call(\'DEL\', KEYS[1])', + 'return nil', + ].join('\n'); + const finalSettlement = await boundedCommand( + this.redis.eval( + closeScript, + closeKeys.length, + ...closeKeys, + assignment.assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge settlement close', + ); + if (finalSettlement != null) { + return JSON.parse(String(finalSettlement)) as CodeBridgeSettlement; + } + throw new BridgeStoreError( + 'ASSIGNMENT_EXPIRED', + 'Bridge assignment exceeded its deadline', + ); + } + + private async cancel( + assignmentId: string, + assignment?: StoredAssignment, + ): Promise { + const ttlSeconds = + assignment == null + ? 30 + : assignmentTtlSeconds(Date.parse(assignment.expiresAt)); + await boundedCommand( + this.redis.set( + cancellationKey(assignmentId), + '1', + 'EX', + ttlSeconds, + ), + this.redisCommandTimeoutMs, + 'Bridge assignment cancellation', + ); + } + + private async enqueueForActiveIncarnation( + assignment: StoredAssignment, + ttlSeconds: number, + ): Promise { + const script = [ + 'if redis.call(\'GET\', KEYS[1]) ~= ARGV[1] then return 0 end', + 'if #KEYS == 6 and redis.call(\'EXISTS\', KEYS[6]) == 1 then return -1 end', + 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', + 'redis.call(\'RPUSH\', KEYS[3], ARGV[4])', + 'redis.call(\'EXPIRE\', KEYS[3], ARGV[3])', + 'redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])', + 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', + 'if #KEYS == 6 then redis.call(\'SET\', KEYS[6], ARGV[4]) end', + 'return 1', + ].join('\n'); + const keys = [ + workerIncarnationKey(assignment.workerId), + assignmentKey(assignment.assignmentId), + queueKey(assignment.workerId, assignment.incarnationId), + lockIncarnationKey(assignment.workerId), + assignmentDeadlineKey(assignment.assignmentId), + ]; + if (assignment.runtimeSessionId !== undefined) { + keys.push( + workspaceQuarantineKey( + assignment.workerId, + assignment.runtimeSessionId, + ), + ); + } + const result = await this.redis.eval( + script, + keys.length, + ...keys, + assignment.incarnationId, + JSON.stringify(assignment), + String(ttlSeconds), + assignment.assignmentId, + String(ttlSeconds * 1000), + String(Date.parse(assignment.expiresAt)), + ); + if (Number(result) === -1) { + throw new BridgeStoreError( + 'WORKSPACE_QUARANTINED', + 'Bridge workspace already has incomplete stateful work', + ); + } + return Number(result) === 1; + } + + private async acquireLock( + workerId: string, + assignmentId: string, + incarnationId: string, + ttlSeconds: number, + ): Promise { + const script = [ + 'if redis.call(\'EXISTS\', KEYS[1]) == 1 then return 0 end', + 'redis.call(\'SET\', KEYS[1], ARGV[1], \"PX\", ARGV[3])', + 'redis.call(\'SET\', KEYS[2], ARGV[2], \"PX\", ARGV[3])', + 'return 1', + ].join('\n'); + const result = await this.redis.eval( + script, + 2, + lockKey(workerId), + lockIncarnationKey(workerId), + assignmentId, + incarnationId, + String(ttlSeconds * 1000), + ); + return Number(result) === 1; + } + + private async cleanupDispatch( + workerId: string, + assignmentId: string, + assignment: StoredAssignment | undefined, + ): Promise { + await Promise.all([ + this.cancel(assignmentId, assignment), + assignment == null + ? boundedCommand( + this.releaseLock(workerId, assignmentId), + this.redisCommandTimeoutMs, + 'Bridge assignment lock release', + ) + : this.cleanup(assignment), + ]); + } + + private async commitPendingWorkspace( + assignment: StoredAssignment, + settlement: CodeBridgeSettlement, + deadlineAtMs: number, + signal: AbortSignal, + ): Promise { + if ( + assignment.runtimeSessionId === undefined || + settlement.status !== 'fulfilled' + ) { + return; + } + const runtimeSessionId = assignment.runtimeSessionId; + const script = [ + 'if redis.call(\'GET\', KEYS[1]) == ARGV[1] then', + ' return redis.call(\'DEL\', KEYS[1])', + 'end', + 'return 0', + ].join('\n'); + const committed = Number( + await boundedCommand( + this.redis.eval( + script, + 1, + workspaceQuarantineKey( + assignment.workerId, + runtimeSessionId, + ), + assignment.assignmentId, + ), + Math.max( + 1, + Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), + ), + 'Bridge workspace commit', + signal, + ), + ); + if (committed !== 1) { + throw new BridgeStoreError( + 'WORKSPACE_QUARANTINED', + 'Bridge workspace commit marker was lost before finalization completed', + ); + } + } + + private async cleanupWithRetry( + workerId: string, + assignmentId: string, + assignment: StoredAssignment | undefined, + ): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + await this.cleanupDispatch(workerId, assignmentId, assignment); + return; + } catch (error) { + lastError = error; + await delay(25); + } + } + throw lastError; + } + + private async cleanup(assignment: StoredAssignment): Promise { + const keys = [ + assignmentKey(assignment.assignmentId), + queueKey(assignment.workerId, assignment.incarnationId), + leaseClaimKey(assignment.workerId, assignment.incarnationId), + leaseAckKey(assignment.workerId, assignment.incarnationId), + assignment.runtimeSessionId === undefined + ? `${assignmentKey(assignment.assignmentId)}:no-workspace` + : workspaceQuarantineKey( + assignment.workerId, + assignment.runtimeSessionId, + ), + ]; + const cleanupScript = [ + "local queued = redis.call('LREM', KEYS[2], 0, ARGV[1])", + "local claimed = redis.call('GET', KEYS[3]) == ARGV[1]", + "local acknowledged = redis.call('GET', KEYS[4]) == ARGV[1]", + 'if ARGV[2] == "1" and (queued > 0 or (claimed and not acknowledged)) and redis.call(\'GET\', KEYS[5]) == ARGV[1] then', + " redis.call('DEL', KEYS[5])", + 'end', + 'if claimed and not acknowledged then', + " redis.call('DEL', KEYS[3], KEYS[4])", + 'end', + 'if queued == 0 and acknowledged and ARGV[2] == "1" and redis.call(\'GET\', KEYS[5]) == ARGV[1] then', + ' return -1', + 'end', + "return redis.call('DEL', KEYS[1], KEYS[3], KEYS[4])", + ].join('\n'); + const cleanupResult = Number( + await boundedCommand( + this.redis.eval( + cleanupScript, + keys.length, + ...keys, + assignment.assignmentId, + assignment.runtimeSessionId === undefined ? '0' : '1', + ), + this.redisCommandTimeoutMs, + 'Bridge assignment cleanup', + ), + ); + if (cleanupResult !== -1) { + await boundedCommand( + this.releaseLock(assignment.workerId, assignment.assignmentId), + this.redisCommandTimeoutMs, + 'Bridge assignment lock release', + ); + } + } + + private async releaseLock( + workerId: string, + assignmentId: string, + ): Promise { + const script = [ + 'if redis.call(\'GET\', KEYS[1]) == ARGV[1] then', + ' return redis.call(\'DEL\', KEYS[1], KEYS[2])', + 'end', + 'return 0', + ].join('\n'); + await this.redis.eval( + script, + 2, + lockKey(workerId), + lockIncarnationKey(workerId), + assignmentId, + ); + } +} diff --git a/service/src/config.test.ts b/service/src/config.test.ts index 2f88a87a..22878ea7 100644 --- a/service/src/config.test.ts +++ b/service/src/config.test.ts @@ -14,6 +14,7 @@ describe('sandbox execution configuration', () => { test('accepts every supported backend and session mode', () => { expect(resolveSandboxBackend('http')).toBe('http'); expect(resolveSandboxBackend('lambda-microvm')).toBe('lambda-microvm'); + expect(resolveSandboxBackend('remote-bridge')).toBe('remote-bridge'); expect(resolveRuntimeSessionMode('stateless')).toBe('stateless'); expect(resolveRuntimeSessionMode('affinity')).toBe('affinity'); expect(resolveRuntimeSessionMode('strict')).toBe('strict'); @@ -21,7 +22,7 @@ describe('sandbox execution configuration', () => { test('rejects unknown values instead of silently changing execution semantics', () => { expect(() => resolveSandboxBackend('lambda_microvm')).toThrow( - 'CODEAPI_SANDBOX_BACKEND must be one of: http, lambda-microvm', + 'CODEAPI_SANDBOX_BACKEND must be one of: http, lambda-microvm, remote-bridge', ); expect(() => resolveSandboxBackend('')).toThrow('CODEAPI_SANDBOX_BACKEND'); expect(() => resolveSandboxBackend(' ')).toThrow('CODEAPI_SANDBOX_BACKEND'); diff --git a/service/src/config.ts b/service/src/config.ts index ecf35661..dbc6dfe8 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -243,12 +243,12 @@ function configuredChoice( export function resolveSandboxBackend( raw: string | undefined, -): 'http' | 'lambda-microvm' { +): 'http' | 'lambda-microvm' | 'remote-bridge' { return configuredChoice( raw, 'CODEAPI_SANDBOX_BACKEND', 'http', - ['http', 'lambda-microvm'], + ['http', 'lambda-microvm', 'remote-bridge'], ); } @@ -350,8 +350,13 @@ export const env = { * - `http` (default): POST signed execute requests to SANDBOX_ENDPOINT * (current Kubernetes/libkrun sandbox-runner). * - `lambda-microvm`: AWS Lambda MicroVM backend. + * - `remote-bridge`: dispatch to an outbound-connected @librechat/code worker. */ SANDBOX_BACKEND: sandboxBackend, + /** Outbound worker selected by the remote-bridge backend. */ + BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', + /** Enrollment and lease credential shared only with the configured worker. */ + BRIDGE_TOKEN: process.env.CODEAPI_BRIDGE_TOKEN ?? '', /** * Runtime session affinity for stateful sandbox backends. * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. diff --git a/service/src/lifecycle.ts b/service/src/lifecycle.ts index 8fc0adc2..c5687610 100644 --- a/service/src/lifecycle.ts +++ b/service/src/lifecycle.ts @@ -4,6 +4,7 @@ import { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, connection } from import { validateStartupAuthConfig } from './auth/startup'; import { env } from './config'; import { + validateApiBridgePolicy, validateApiHardenedConfig, validateExecutionProfilePolicy, validateSandboxBackendPolicy, @@ -89,9 +90,12 @@ function setupQueueListeners(queue: Queue, name: string): void { export async function startupApiOnly(): Promise { logger.info('Starting API service (no workers)...'); validateApiHardenedConfig(); + validateApiBridgePolicy(); validateExecutionProfilePolicy({ requireBackendMatch: false }); /* No validateSandboxBackendPolicy() here: an API-only pod authenticates and * enqueues jobs, it never constructs the Lambda backend or checkpoint store. + * Bridge credentials are validated separately above because this process + * exposes the public registration, lease, and settlement routes. * Validating that policy would force worker-only config (LAMBDA_MICROVM_* and * the MINIO_* checkpoint creds) into API pods just to boot. The worker and * combined startups own that validation. */ diff --git a/service/src/local-api.ts b/service/src/local-api.ts index 701270d7..aafa9f03 100644 --- a/service/src/local-api.ts +++ b/service/src/local-api.ts @@ -10,6 +10,7 @@ import express, { json, Router } from 'express'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; +import bridgeRouter from './bridge/router'; import { requestErrorLogger, requestNotFoundLogger } from './middleware/request-error-logger'; import { executionProfileMiddleware } from './middleware/execution-profile'; import { localAuth } from './auth/local'; @@ -20,7 +21,11 @@ import './workers'; import { env } from './config'; import logger from './logger'; import { shutdownTelemetry, traceHttpRequest } from './telemetry'; -import { validateExecutionProfilePolicy } from './secure-startup'; +import { + validateApiBridgePolicy, + validateExecutionProfilePolicy, + validateSandboxBackendPolicy, +} from './secure-startup'; import { configureExecutionProfileMetrics } from './metrics'; const app = express(); @@ -45,6 +50,7 @@ app.get('/v1/health', async (_, res) => { } }); +v1.use('/bridge', bridgeRouter); v1.use(localAuth); v1.use(serviceRouter); v1.use(programmaticRouter); @@ -56,7 +62,9 @@ app.use(requestErrorLogger); async function localStartup(): Promise { logger.info('Starting local development server...'); logger.info('⚠️ LOCAL MODE - No authentication required'); + validateApiBridgePolicy(); validateExecutionProfilePolicy(); + validateSandboxBackendPolicy(); configureExecutionProfileMetrics({ profile: env.EXECUTION_PROFILE, sandboxBackend: env.SANDBOX_BACKEND, diff --git a/service/src/runtime-session/job-policy.test.ts b/service/src/runtime-session/job-policy.test.ts index 96725f9f..22615f7b 100644 --- a/service/src/runtime-session/job-policy.test.ts +++ b/service/src/runtime-session/job-policy.test.ts @@ -92,6 +92,19 @@ describe('resolveRuntimeSessionForJob', () => { })).toThrow('http/affinity worker cannot honor queued affinity runtime session'); }); + test('allows a remote bridge worker to honor a stateful job', () => { + expect(resolveRuntimeSessionForJob({ + workerBackend: 'remote-bridge', + workerMode: 'strict', + runtimeSessionMode: 'strict', + runtimeSessionId: 'rt_attached', + isSynthetic: false, + })).toEqual({ + runtimeSessionId: 'rt_attached', + runtimeSessionMode: 'strict', + }); + }); + test('rejects contradictory or invalid producer decisions', () => { expect(() => resolveRuntimeSessionForJob({ ...LAMBDA_WORKER, diff --git a/service/src/runtime-session/job-policy.ts b/service/src/runtime-session/job-policy.ts index bdadeddb..8c473c60 100644 --- a/service/src/runtime-session/job-policy.ts +++ b/service/src/runtime-session/job-policy.ts @@ -10,7 +10,7 @@ export type RuntimeSessionJobDecision = { runtimeSessionMode: RuntimeSessionMode; }; -type SandboxBackendName = 'http' | 'lambda-microvm'; +type SandboxBackendName = 'http' | 'lambda-microvm' | 'remote-bridge'; function isRuntimeSessionMode(value: unknown): value is RuntimeSessionMode { return value === 'stateless' || value === 'affinity' || value === 'strict'; @@ -70,7 +70,10 @@ export function resolveRuntimeSessionForJob(args: { if (runtimeSessionId === undefined) { throw new Error(`${runtimeSessionMode} queued job requires a runtimeSessionId`); } - if (args.workerMode === 'stateless' || args.workerBackend !== 'lambda-microvm') { + if ( + args.workerMode === 'stateless' + || (args.workerBackend !== 'lambda-microvm' && args.workerBackend !== 'remote-bridge') + ) { throw new Error( `${args.workerBackend}/${args.workerMode} worker cannot honor queued ` + `${runtimeSessionMode} runtime session`, diff --git a/service/src/sandbox-backend/index.test.ts b/service/src/sandbox-backend/index.test.ts index e9f2ac7d..44378011 100644 --- a/service/src/sandbox-backend/index.test.ts +++ b/service/src/sandbox-backend/index.test.ts @@ -26,6 +26,12 @@ describe('getSandboxBackend', () => { expect(backend.name).toBe('lambda-microvm'); }); + test('selects the outbound remote bridge backend when configured', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + const backend = getSandboxBackend(); + expect(backend.name).toBe('remote-bridge'); + }); + test('does not load Lambda-only modules for the HTTP backend', async () => { const serviceRoot = path.resolve(import.meta.dir, '../..'); const probe = Bun.spawn([ diff --git a/service/src/sandbox-backend/index.ts b/service/src/sandbox-backend/index.ts index 3ebf796a..e5192513 100644 --- a/service/src/sandbox-backend/index.ts +++ b/service/src/sandbox-backend/index.ts @@ -14,6 +14,25 @@ export { HttpSandboxBackend } from './http'; let backend: SandboxBackend | undefined; +class LazyRemoteBridgeSandboxBackend implements SandboxBackend { + readonly name = 'remote-bridge' as const; + private backendPromise: Promise | undefined; + + private load(): Promise { + this.backendPromise ??= import('./remote-bridge').then( + ({ RemoteBridgeSandboxBackend }) => new RemoteBridgeSandboxBackend(), + ); + return this.backendPromise; + } + + async execute( + req: SandboxTransportRequest, + ctx: SandboxExecuteContext, + ): Promise { + return (await this.load()).execute(req, ctx); + } +} + class LazyLambdaMicrovmSandboxBackend implements SandboxBackend { readonly name = 'lambda-microvm' as const; private backendPromise: Promise | undefined; @@ -72,6 +91,9 @@ class LazyLambdaMicrovmSandboxBackend implements SandboxBackend { } function createBackend(): SandboxBackend { + if (env.SANDBOX_BACKEND === 'remote-bridge') { + return new LazyRemoteBridgeSandboxBackend(); + } if (env.SANDBOX_BACKEND === 'lambda-microvm') { /* Loading the concrete backend also loads its session registry and * checkpoint code. Defer the whole graph so the default HTTP worker does diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts new file mode 100644 index 00000000..a18788a8 --- /dev/null +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -0,0 +1,82 @@ +import type { + SandboxBackend, + SandboxExecuteContext, + SandboxRawResponse, + SandboxTransportRequest, +} from './types'; +import type { RedisBridgeStore } from '../bridge/store'; + +import { env } from '../config'; +import { bridgeStore } from '../bridge/router'; +import { BridgeStoreError } from '../bridge/store'; +import { SandboxBackendError } from './types'; + +export class RemoteBridgeSandboxBackend implements SandboxBackend { + readonly name = 'remote-bridge' as const; + + constructor( + private readonly store: RedisBridgeStore = bridgeStore, + private readonly workerId: string = env.BRIDGE_WORKER_ID, + ) {} + + async execute( + req: SandboxTransportRequest, + ctx: SandboxExecuteContext, + ): Promise { + if (!this.workerId) { + throw new SandboxBackendError( + 'BRIDGE_WORKER_OFFLINE', + 'No bridge worker is configured', + ); + } + const sessionResultFinalizer = ctx.sessionResultFinalizer; + try { + const settlement = await this.store.dispatch({ + workerId: this.workerId, + body: req.body, + headers: req.headers, + runtimeSessionId: ctx.runtimeSessionId, + deadlineAtMs: ctx.deadlineAtMs ?? Date.now() + env.JOB_TIMEOUT, + signal: ctx.signal, + finalize: sessionResultFinalizer + ? async (settlement) => { + if (settlement.status === 'rejected') return settlement; + return { + ...settlement, + result: await sessionResultFinalizer(settlement.result), + }; + } + : undefined, + }); + if (settlement.status === 'rejected') { + throw new SandboxBackendError( + 'BRIDGE_EXECUTION_FAILED', + settlement.error, + ); + } + return settlement.result as SandboxRawResponse; + } catch (error) { + if (!(error instanceof BridgeStoreError)) throw error; + if (error.code === 'WORKER_BUSY') { + throw new SandboxBackendError( + 'BRIDGE_WORKER_BUSY', + error.message, + error, + ); + } + if (error.code === 'ASSIGNMENT_EXPIRED') { + throw new SandboxBackendError( + 'BRIDGE_DEADLINE_EXCEEDED', + error.message, + error, + ); + } + throw new SandboxBackendError( + 'BRIDGE_WORKER_OFFLINE', + error.message, + error, + true, + ); + } + } +} diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index 75d7f1af..15bb6942 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -56,13 +56,17 @@ export type SandboxRawResponse = t.ExecuteResponse & { }; export interface SandboxBackend { - readonly name: 'http' | 'lambda-microvm'; + readonly name: 'http' | 'lambda-microvm' | 'remote-bridge'; execute(req: SandboxTransportRequest, ctx: SandboxExecuteContext): Promise; shutdown?(): Promise; } export type SandboxBackendErrorCode = | 'RUNTIME_SESSION_BUSY' + | 'BRIDGE_WORKER_OFFLINE' + | 'BRIDGE_WORKER_BUSY' + | 'BRIDGE_EXECUTION_FAILED' + | 'BRIDGE_DEADLINE_EXCEEDED' | 'MICROVM_LAUNCH_FAILED' | 'MICROVM_LAUNCH_THROTTLED' | 'MICROVM_UNHEALTHY' diff --git a/service/src/secure-startup.test.ts b/service/src/secure-startup.test.ts index 4aa603e4..a365f355 100644 --- a/service/src/secure-startup.test.ts +++ b/service/src/secure-startup.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { env } from './config'; import { + validateApiBridgePolicy, validateApiHardenedConfig, validateEgressGatewayHardenedConfig, validateExecutionProfilePolicy, @@ -14,6 +15,8 @@ const saved = { executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: env.SANDBOX_BACKEND, + bridgeWorkerId: env.BRIDGE_WORKER_ID, + bridgeToken: env.BRIDGE_TOKEN, ptcMode: env.PTC_MODE, runtimeSessionMode: env.RUNTIME_SESSION_MODE, lambdaImageArn: env.LAMBDA_MICROVM_IMAGE_ARN, @@ -52,6 +55,8 @@ function restore(): void { env.EXECUTION_PROFILE = saved.executionProfile; env.EXECUTION_PROFILE_SOURCE = saved.executionProfileSource; env.SANDBOX_BACKEND = saved.sandboxBackend; + env.BRIDGE_WORKER_ID = saved.bridgeWorkerId; + env.BRIDGE_TOKEN = saved.bridgeToken; env.PTC_MODE = saved.ptcMode; env.RUNTIME_SESSION_MODE = saved.runtimeSessionMode; env.LAMBDA_MICROVM_IMAGE_ARN = saved.lambdaImageArn; @@ -279,12 +284,98 @@ describe('sandbox backend policy', () => { expect(() => validateSandboxBackendPolicy()).not.toThrow(); }); - test('stateful runtime session modes require the lambda backend', () => { + test('stateful runtime session modes require a stateful backend', () => { env.SANDBOX_BACKEND = 'http'; env.RUNTIME_SESSION_MODE = 'affinity'; - expect(() => validateSandboxBackendPolicy()).toThrow('requires the lambda-microvm backend'); + expect(() => validateSandboxBackendPolicy()).toThrow( + 'requires the lambda-microvm or remote-bridge backend', + ); env.RUNTIME_SESSION_MODE = 'strict'; - expect(() => validateSandboxBackendPolicy()).toThrow('requires the lambda-microvm backend'); + expect(() => validateSandboxBackendPolicy()).toThrow( + 'requires the lambda-microvm or remote-bridge backend', + ); + }); + + test('accepts a configured remote bridge and fails closed on missing enrollment', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.RUNTIME_SESSION_MODE = 'strict'; + env.PTC_MODE = 'replay'; + env.BRIDGE_WORKER_ID = ''; + env.BRIDGE_TOKEN = ''; + expect(() => validateSandboxBackendPolicy()).toThrow('CODEAPI_BRIDGE_WORKER_ID'); + + env.BRIDGE_WORKER_ID = 'engineering-vm'; + expect(() => validateSandboxBackendPolicy()).toThrow('CODEAPI_BRIDGE_TOKEN'); + + env.BRIDGE_TOKEN = 'development-bridge-token'; + expect(() => validateSandboxBackendPolicy()).not.toThrow(); + }); + + test('remote bridge requires replay PTC and a strong token in hardened mode', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.RUNTIME_SESSION_MODE = 'affinity'; + env.BRIDGE_WORKER_ID = 'engineering-vm'; + env.BRIDGE_TOKEN = 'development-bridge-token'; + env.PTC_MODE = 'blocking'; + expect(() => validateSandboxBackendPolicy()).toThrow( + 'PTC replay is the only supported PTC mode', + ); + + env.PTC_MODE = 'replay'; + env.HARDENED_SANDBOX_MODE = true; + expect(() => validateSandboxBackendPolicy()).toThrow('at least 32 bytes'); + + env.BRIDGE_TOKEN = 'strong-remote-bridge-token-32-bytes'; + expect(() => validateSandboxBackendPolicy()).not.toThrow(); + }); + + test('API bridge policy requires a strong token in hardened mode', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.BRIDGE_WORKER_ID = 'engineering-vm'; + env.BRIDGE_TOKEN = 'short-token'; + env.PTC_MODE = 'replay'; + env.HARDENED_SANDBOX_MODE = true; + + expect(() => validateApiBridgePolicy()).toThrow('at least 32 bytes'); + + env.BRIDGE_TOKEN = 'strong-remote-bridge-token-32-bytes'; + expect(() => validateApiBridgePolicy()).not.toThrow(); + }); + + test('API bridge policy rejects worker IDs the router cannot accept', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.BRIDGE_WORKER_ID = 'engineering/vm'; + env.BRIDGE_TOKEN = 'development-bridge-token'; + env.PTC_MODE = 'replay'; + + expect(() => validateApiBridgePolicy()).toThrow( + 'must match the bridge worker ID format', + ); + }); + + test('API bridge policy rejects whitespace-padded tokens', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.BRIDGE_WORKER_ID = 'engineering-vm'; + env.BRIDGE_TOKEN = ' padded-development-bridge-token '; + env.PTC_MODE = 'replay'; + + expect(() => validateApiBridgePolicy()).toThrow( + 'must not contain surrounding whitespace', + ); + }); + + test('remote bridge requires a positive finite job timeout', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.BRIDGE_WORKER_ID = 'engineering-vm'; + env.BRIDGE_TOKEN = 'development-bridge-token'; + env.PTC_MODE = 'replay'; + + env.JOB_TIMEOUT = -1; + expect(() => validateApiBridgePolicy()).toThrow('JOB_TIMEOUT'); + env.JOB_TIMEOUT = Number.POSITIVE_INFINITY; + expect(() => validateApiBridgePolicy()).toThrow('JOB_TIMEOUT'); + env.JOB_TIMEOUT = 300_000; + expect(() => validateApiBridgePolicy()).not.toThrow(); }); test('rejects blocking PTC on the lambda backend', () => { diff --git a/service/src/secure-startup.ts b/service/src/secure-startup.ts index a82dabf0..0bcf02c1 100644 --- a/service/src/secure-startup.ts +++ b/service/src/secure-startup.ts @@ -4,6 +4,7 @@ import { lambdaMicrovmNumericConfigError, } from './config'; import { INTERNAL_SERVICE_TOKEN_ENV } from './internal-service-auth'; +import { isValidBridgeWorkerId } from '../../packages/code/src/protocol'; export class SecureStartupConfigError extends Error { constructor(message: string) { @@ -53,6 +54,33 @@ export function validateApiHardenedConfig(): void { requireValue(INTERNAL_SERVICE_TOKEN_ENV, process.env[INTERNAL_SERVICE_TOKEN_ENV]); } +/** Validate bridge credentials in every process that exposes bridge routes. */ +export function validateApiBridgePolicy(): void { + if (env.SANDBOX_BACKEND !== 'remote-bridge') return; + requireSafeWholeNumber('JOB_TIMEOUT', env.JOB_TIMEOUT, 1); + requireValue('CODEAPI_BRIDGE_WORKER_ID', env.BRIDGE_WORKER_ID); + if (!isValidBridgeWorkerId(env.BRIDGE_WORKER_ID ?? '')) { + throw new SecureStartupConfigError( + 'CODEAPI_BRIDGE_WORKER_ID must match the bridge worker ID format', + ); + } + if (env.BRIDGE_TOKEN !== env.BRIDGE_TOKEN.trim()) { + throw new SecureStartupConfigError( + 'CODEAPI_BRIDGE_TOKEN must not contain surrounding whitespace', + ); + } + if (env.HARDENED_SANDBOX_MODE) { + requireStrongSecret('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); + } else { + requireValue('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); + } + if (env.PTC_MODE === 'blocking') { + throw new SecureStartupConfigError( + 'PTC replay is the only supported PTC mode for the remote-bridge backend (unset PTC_MODE=blocking)', + ); + } +} + export function validateWorkerHardenedConfig(): void { if (!env.HARDENED_SANDBOX_MODE) return; rejectValue('CODEAPI_EGRESS_GRANT_SECRET', process.env.CODEAPI_EGRESS_GRANT_SECRET); @@ -93,11 +121,15 @@ export function validateExecutionProfilePolicy(options: { if ( env.RUNTIME_SESSION_MODE === 'stateless' - || (requireBackendMatch && env.SANDBOX_BACKEND !== 'lambda-microvm') + || ( + requireBackendMatch + && env.SANDBOX_BACKEND !== 'lambda-microvm' + && env.SANDBOX_BACKEND !== 'remote-bridge' + ) ) { throw new SecureStartupConfigError( 'CODEAPI_EXECUTION_PROFILE=stateful requires ' - + (requireBackendMatch ? 'CODEAPI_SANDBOX_BACKEND=lambda-microvm and ' : '') + + (requireBackendMatch ? 'CODEAPI_SANDBOX_BACKEND=lambda-microvm or remote-bridge and ' : '') + 'CODEAPI_RUNTIME_SESSION_MODE=affinity or strict', ); } @@ -111,9 +143,13 @@ export function validateSandboxBackendPolicy(): void { if (env.RUNTIME_SESSION_MODE !== 'stateless' && env.SANDBOX_BACKEND === 'http') { throw new SecureStartupConfigError( `CODEAPI_RUNTIME_SESSION_MODE=${env.RUNTIME_SESSION_MODE} requires ` - + 'the lambda-microvm backend; use stateless mode with the http backend', + + 'the lambda-microvm or remote-bridge backend; use stateless mode with the http backend', ); } + if (env.SANDBOX_BACKEND === 'remote-bridge') { + validateApiBridgePolicy(); + return; + } if (env.SANDBOX_BACKEND !== 'lambda-microvm') return; const numericConfigError = lambdaMicrovmNumericConfigError(env); diff --git a/service/src/service-api.ts b/service/src/service-api.ts index ec15b1ac..6a0a8331 100644 --- a/service/src/service-api.ts +++ b/service/src/service-api.ts @@ -5,6 +5,7 @@ import { requestErrorLogger, requestNotFoundLogger } from './middleware/request- import { executionProfileMiddleware } from './middleware/execution-profile'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; +import bridgeRouter from './bridge/router'; import { connection } from './queue'; import { env } from './config'; import logger from './logger'; @@ -28,6 +29,7 @@ app.get('/v1/health', async (_, res) => { } }); +v1.use('/bridge', bridgeRouter); v1.use(apiKeyAuth); v1.use(serviceRouter); diff --git a/service/src/utils.test.ts b/service/src/utils.test.ts index f1952dfc..879736c5 100644 --- a/service/src/utils.test.ts +++ b/service/src/utils.test.ts @@ -145,6 +145,44 @@ describe('sandbox error formatting', () => { }); }); + test('maps remote bridge failures without exposing worker details', () => { + const cases = [ + ['BRIDGE_WORKER_OFFLINE', 503, 'Remote code worker is unavailable'], + ['BRIDGE_WORKER_BUSY', 409, 'Remote code worker is busy'], + ['BRIDGE_EXECUTION_FAILED', 502, 'Remote code execution failed'], + [ + 'BRIDGE_DEADLINE_EXCEEDED', + 504, + 'Remote code execution deadline exceeded', + ], + ] as const; + for (const [code, status, message] of cases) { + const failure = publicExecutionFailure( + new Error(`${code}: worker vm-private failed at redis.internal`), + ); + expect(failure).toEqual({ + status, + body: { error: code.toLowerCase(), message }, + }); + expect(JSON.stringify(failure)).not.toContain('vm-private'); + expect(JSON.stringify(failure)).not.toContain('redis.internal'); + } + }); + + test('maps multiline remote bridge failures without exposing details', () => { + const failure = publicExecutionFailure( + new Error('BRIDGE_EXECUTION_FAILED: first line\nprivate second line'), + ); + expect(failure).toEqual({ + status: 502, + body: { + error: 'bridge_execution_failed', + message: 'Remote code execution failed', + }, + }); + expect(JSON.stringify(failure)).not.toContain('private second line'); + }); + test('maps a recycled dirty session to a retryable public failure', () => { const failure = publicExecutionFailure( new Error('MICROVM_UNHEALTHY: Runtime session rt_private workspace was dirty and has been recycled'), diff --git a/service/src/utils.ts b/service/src/utils.ts index 3078d6e8..e47a7d4c 100644 --- a/service/src/utils.ts +++ b/service/src/utils.ts @@ -128,15 +128,19 @@ export function publicExecutionFailure(error: unknown): { status: number; body: } /* Typed worker failures cross BullMQ as `: `. Runtime-session - * and MicroVM codes describe sandbox availability; SESSION_INPUT_* codes + * MicroVM, and bridge codes describe sandbox availability; SESSION_INPUT_* codes * describe the caller's declared input set or its upstream object source. */ const backendMatch = message.match( - /^(RUNTIME_SESSION_BUSY|MICROVM_[A-Z_]+|SESSION_INPUT_[A-Z_]+):\s*(.+)$/, + /^(RUNTIME_SESSION_BUSY|MICROVM_[A-Z_]+|BRIDGE_[A-Z_]+|SESSION_INPUT_[A-Z_]+):/, ); if (backendMatch) { const code = backendMatch[1]; const statuses: Record = { RUNTIME_SESSION_BUSY: 409, + BRIDGE_WORKER_OFFLINE: 503, + BRIDGE_WORKER_BUSY: 409, + BRIDGE_EXECUTION_FAILED: 502, + BRIDGE_DEADLINE_EXCEEDED: 504, SESSION_INPUT_TOO_LARGE: 413, SESSION_INPUT_UNAVAILABLE: 422, SESSION_INPUT_SOURCE_FAILED: 502, @@ -147,6 +151,10 @@ export function publicExecutionFailure(error: unknown): { status: number; body: const status = statuses[code] ?? (sessionInputFailure ? 500 : 503); const publicMessages: Record = { RUNTIME_SESSION_BUSY: 'Runtime session is busy', + BRIDGE_WORKER_OFFLINE: 'Remote code worker is unavailable', + BRIDGE_WORKER_BUSY: 'Remote code worker is busy', + BRIDGE_EXECUTION_FAILED: 'Remote code execution failed', + BRIDGE_DEADLINE_EXCEEDED: 'Remote code execution deadline exceeded', MICROVM_LAUNCH_FAILED: 'Sandbox launch failed', MICROVM_LAUNCH_THROTTLED: 'Sandbox capacity is temporarily unavailable', MICROVM_UNHEALTHY: 'Sandbox runtime is unavailable', diff --git a/service/tsconfig.json b/service/tsconfig.json index c3e88635..dcddf82d 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -13,7 +13,11 @@ "@/*": ["src/*"] } }, - "include": ["src/**/*.ts", "../shared/telemetry-core.ts"], + "include": [ + "src/**/*.ts", + "../shared/telemetry-core.ts", + "../packages/code/src/protocol.ts" + ], "exclude": [ "node_modules", "**/*.spec.ts", From 89afb4e6603e3c7f9c34a9c1b39484c3f673d172 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 1 Sep 2026 12:34:07 -0400 Subject: [PATCH 004/116] feat: add secure code worker pairing (#67) * feat: add outbound stateful code bridge * test: cover remote bridge startup policy * fix: harden remote bridge lifecycle fencing * feat: add secure code worker pairing * fix: harden paired worker lifecycle * fix: require paired auth on hardened APIs * fix: harden bridge pairing startup policy * fix: preserve bridge fencing through pairing * fix: distinguish assignment settlement conflicts * fix: harden remote bridge assignment lifecycle * fix: close remote bridge commit races * fix: surface shutdown workspace quarantine * fix: fail closed across bridge lifecycle gaps * fix: persist stateful settlement commit barriers * fix: bound bridge control-plane timing * fix: recover abandoned bridge leases * fix: fence stateful bridge workspaces * fix: recover bridge lease read failures * fix: add safe workspace fence recovery * fix: bound bridge liveness timers * fix: bound bridge cleanup recovery * fix: preserve bridge deadline fencing * fix: acknowledge bridge lease delivery * fix: close bridge deadline gaps * fix: harden bridge recovery edges * fix: preserve bridge rejection recovery * fix: fence bridge lease recovery * fix: bound bridge control lifetimes * fix: retain bridge recovery ownership * fix: isolate bridge control state * fix: bound bridge workspace reset * fix: validate bridge deployment inputs * fix: close bridge registration and deadline races * fix: anchor bridge lease freshness * fix: preserve bridge response status * fix: harden paired bridge lifecycle * fix: sustain paired credentials in flight * fix: fence credential refresh deadlines * fix: fence sandbox start at deadline * fix: complete paired worker revocation * fix: make worker revocation atomic * fix(bridge): fence queued registration after revoke --- .env.example | 1 + docs/adr/001-stateful-code-environments.md | 86 ++ docs/remote-bridge/README.md | 36 +- packages/code/README.md | 42 +- packages/code/src/cli.ts | 189 +++-- packages/code/src/identity.test.ts | 35 + packages/code/src/identity.ts | 66 ++ packages/code/src/index.ts | 3 + packages/code/src/pairing.test.ts | 36 + packages/code/src/pairing.ts | 73 ++ packages/code/src/protocol.ts | 14 + packages/code/src/storage.test.ts | 41 + packages/code/src/storage.ts | 66 ++ packages/code/src/worker.test.ts | 846 ++++++++++++++++++- packages/code/src/worker.ts | 301 ++++++- service/rollup.config.js | 1 + service/src/api-server.ts | 2 +- service/src/bridge/index.ts | 16 + service/src/bridge/pairing.test.ts | 349 ++++++++ service/src/bridge/pairing.ts | 376 +++++++++ service/src/bridge/router.test.ts | 249 ++++++ service/src/bridge/router.ts | 265 +++++- service/src/bridge/store.ts | 22 +- service/src/config.test.ts | 7 + service/src/config.ts | 14 + service/src/lifecycle.ts | 1 + service/src/local-api.ts | 2 +- service/src/sandbox-backend/remote-bridge.ts | 2 +- service/src/secure-startup.test.ts | 50 +- service/src/secure-startup.ts | 35 +- service/src/service-api.ts | 2 +- service/tsconfig.json | 3 +- 32 files changed, 3082 insertions(+), 149 deletions(-) create mode 100644 docs/adr/001-stateful-code-environments.md create mode 100644 packages/code/src/identity.test.ts create mode 100644 packages/code/src/identity.ts create mode 100644 packages/code/src/pairing.test.ts create mode 100644 packages/code/src/pairing.ts create mode 100644 packages/code/src/storage.test.ts create mode 100644 packages/code/src/storage.ts create mode 100644 service/src/bridge/index.ts create mode 100644 service/src/bridge/pairing.test.ts create mode 100644 service/src/bridge/pairing.ts create mode 100644 service/src/bridge/router.test.ts diff --git a/.env.example b/.env.example index 3bd113e5..98c97260 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,7 @@ SANDBOX_OUTPUT_MAX_SIZE=65536 # CODEAPI_RUNTIME_SESSION_MODE=affinity # CODEAPI_BRIDGE_WORKER_ID=my-vm # CODEAPI_BRIDGE_TOKEN=replace-with-a-strong-random-secret +# CODEAPI_BRIDGE_AUTH_MODE=paired # Service Configuration PYTHON_CONCURRENCY=5 diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md new file mode 100644 index 00000000..8b9830df --- /dev/null +++ b/docs/adr/001-stateful-code-environments.md @@ -0,0 +1,86 @@ +# ADR 001: Stateful code environments use an outbound Code API bridge + +- Status: Accepted for alpha +- Date: 2026-08-30 + +## Context + +LibreChat needs coding agents to reuse a workspace across conversation turns +while allowing the environment owner to choose the VM. Internet-facing +LibreChat instances cannot safely require inbound access to that VM, forward +end-user tokens to it, or treat an MCP connection as a sandbox boundary. + +The first alpha demonstrated a stable runtime-session ID, a single fenced +worker lease, and workspace persistence across turns. Its static shared worker +token was sufficient to prove execution flow but is not an acceptable hardened +enrollment mechanism. + +## Decision + +The product concept is a **stateful code environment**. Code API remains its +broker and policy boundary, and `remote-bridge` is a Code API sandbox backend. +The `@librechat/code` worker connects outbound from the chosen VM and forwards +assignments only to a loopback or private sandbox endpoint. + +Hardened workers enroll through a one-time pairing code: + +1. An administrator creates a code scoped to the configured worker ID. +2. The CLI generates an Ed25519 keypair locally and redeems the code with only + its public key. +3. Code API returns a fifteen-minute credential bound to that public key. +4. Every worker request signs the method, path, body digest, timestamp, nonce, + and credential. +5. Code API rejects stale timestamps and replayed nonces and supports rotation + and immediate revocation. + +Static bearer authentication remains a non-hardened compatibility mode. + +## Ownership and state + +The alpha environment is deployment/operator owned and configured with one +worker ID. A future LibreChat control plane may persist deployment-, tenant-, +or user-owned environment records and issue the same pairing operation through +RBAC-protected APIs without changing the worker execution protocol. + +Workspace state belongs to the stable runtime session, not to a transient +assignment lease. For `remote-bridge`, that state currently survives turns on +the same worker and backing disk. It is not yet checkpointed or portable across +worker replacement; the UI and operator documentation must not imply otherwise. + +## Security invariants + +- The VM requires no inbound internet listener. +- Code API, not the worker, authenticates LibreChat users and normalizes work. +- A stolen short-lived credential is insufficient without the worker private + key; a stolen private key is insufficient after credential expiry or + revocation. +- Pairing codes and credentials are stored by digest where lookup permits. +- One configured worker has at most one active fenced assignment. +- Sandbox isolation and default-deny egress remain mandatory; pairing secures + the transport identity but does not make the host a sandbox. +- A compromised worker can lie about advertised capabilities. Capability + labels and policy digests are audit signals until enforcement is coupled to + an attested sandbox or trusted host policy. + +## Consequences + +- `@librechat/code` owns the provider-neutral protocol, identity handling, and + worker CLI; Code API owns enrollment, scheduling, and execution policy. +- LibreChat owns environment persistence, ownership, RBAC, and user experience. +- The Agents SDK keeps only its adapter until a second concrete consumer proves + which coding-tool abstractions are genuinely provider neutral. +- MCP may expose environment operations later, but it is not the worker + transport or isolation boundary. +- Multi-worker directories, checkpoint/restore, owner-scoped quotas, and + enforced network capability profiles remain follow-up decisions. + +## Alternatives rejected + +- **Inbound SSH/HTTP to the VM:** expands attack surface and complicates NAT and + firewall operation. +- **MCP as the worker protocol:** conflates tool discovery with leases, + cancellation, fencing, and sandbox policy. +- **Put the runtime in the Agents SDK:** couples provider-neutral execution to + one agent integration and makes non-agent consumers depend on agent internals. +- **Long-lived shared bearer token:** easy to bootstrap, but replayable and not + bound to a worker-held key. diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index dd22e73b..3a9f12fc 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -12,7 +12,7 @@ LibreChat -> Code API -> Redis assignment Code API remains the public authentication, policy, manifest, timeout, and result-normalization boundary. The bridge worker has a separate operator -credential and never accepts end-user bearer tokens directly. +identity and never accepts end-user bearer tokens directly. ## Code API configuration @@ -23,7 +23,8 @@ CODEAPI_SANDBOX_BACKEND=remote-bridge CODEAPI_EXECUTION_PROFILE=stateful CODEAPI_RUNTIME_SESSION_MODE=affinity CODEAPI_BRIDGE_WORKER_ID=my-vm -CODEAPI_BRIDGE_TOKEN= +CODEAPI_BRIDGE_TOKEN= +CODEAPI_BRIDGE_AUTH_MODE=paired ``` Use `strict` instead of `affinity` if every request must include a runtime @@ -31,8 +32,21 @@ session hint. In hardened mode, startup requires the bridge token to be at least 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a remote execution cannot retain an open Code API process across tool callbacks. -Start the CLI beside a sandbox using the same worker ID and secret; see -[`@librechat/code`](../../packages/code/README.md). +Create a single-use pairing code with the administrator secret: + +```bash +curl -fsS https://code.example.com/v1/bridge/pairings \ + -H "Authorization: Bearer $CODEAPI_BRIDGE_TOKEN" \ + -H 'Content-Type: application/json' \ + --data '{"workerId":"my-vm"}' +``` + +Redeem the returned code on the VM using +[`@librechat/code`](../../packages/code/README.md). The CLI generates its key +locally, proves possession on every request, and rotates its short-lived +credential before expiry. `CODEAPI_BRIDGE_AUTH_MODE=static` remains available +for non-hardened development compatibility only. + Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, persistent local runner per session. A single sandbox endpoint is stateless and @@ -63,6 +77,13 @@ execution. ## Lifecycle and fencing - Registration is ephemeral in Redis and must be refreshed by the worker. +- Pairing codes are stored hashed, expire after ten minutes, and are consumed + atomically on their first redemption attempt. +- Worker credentials expire after fifteen minutes and are bound to an Ed25519 + public key. Exact-request signatures include the HTTP method, path, body + digest, timestamp, nonce, and credential. +- Accepted proof nonces cannot be replayed, credentials rotate before expiry, + and an administrator can revoke the active worker identity immediately. - Code API permits one active assignment per configured worker. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are @@ -101,9 +122,10 @@ For internet-facing LibreChat deployments, use the hardened microVM/NsJail stack, default-deny sandbox egress, signed execution manifests, least-privilege host credentials, resource limits, and host/network monitoring. Bind the local sandbox endpoint to loopback or a private container network. Rotate a leaked -bridge token immediately; the initial protocol intentionally uses a static -operator secret and supports one configured worker per Code API deployment. +administrator token immediately. Pairing secures worker transport identity; it +cannot attest that a compromised VM truthfully reports or enforces its sandbox +capabilities. -The next control-plane layer can add short-lived pairing credentials and a +The next control-plane layer can add owner-scoped environment records and a multi-worker directory without changing the execution protocol or moving code tools into the Agents SDK. diff --git a/packages/code/README.md b/packages/code/README.md index 2439ab25..883b2695 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -9,7 +9,37 @@ untrusted internet traffic). It connects outbound to Code API, long-polls for assignments, forwards them to the local sandbox, and returns fenced results. The VM does not need an inbound public port. -## Run +## Pair + +Hardened deployments use a one-time code instead of copying a long-lived +worker secret onto the VM. After an administrator creates a code, run: + +```bash +librechat-code pair https://code.example.com/v1 '' \ + --worker-id my-vm +``` + +The CLI generates an Ed25519 key locally and writes its paired identity to +`~/.config/librechat/code/my-vm.json` with owner-only permissions. The private +key never leaves the VM. Worker requests carry an exact-request signature, +timestamp, and one-time nonce; the short-lived credential rotates +automatically. + +Then start the worker without a shared secret: + +```bash +LIBRECHAT_CODE_WORKER_ID=my-vm \ +LIBRECHAT_CODE_SANDBOX_ENDPOINT=http://127.0.0.1:2000/api/v2 \ +librechat-code run +``` + +Use `--identity ` while pairing and +`LIBRECHAT_CODE_IDENTITY_FILE=` while running to override the identity +file location. + +## Static compatibility mode + +Non-hardened development deployments may still run with a static token: ```bash npm install -g @librechat/code @@ -18,7 +48,7 @@ LIBRECHAT_CODE_URL=https://code.example.com/v1 \ LIBRECHAT_CODE_WORKER_TOKEN='' \ LIBRECHAT_CODE_WORKER_ID=my-vm \ LIBRECHAT_CODE_SANDBOX_ENDPOINT=http://127.0.0.1:2000/api/v2 \ -librechat-code +librechat-code run ``` Optional environment variables: @@ -40,6 +70,10 @@ A single built-in sandbox runner binds itself to one runtime session and must not be advertised as stateful. Use the default stateless capability until a session-routing supervisor is configured. +Static worker authentication is rejected when Code API hardened mode is +enabled. Expose only the sandbox loopback endpoint to the CLI, and enforce +VM/container egress policy independently of the bridge transport. + The worker retries result settlement through the assignment deadline. If a stateful result remains ambiguous, it exits with a quarantine error instead of accepting another assignment. Reset or discard that session's local runner @@ -51,7 +85,3 @@ with `librechat-code reset-workspace `. The command uses the configured worker credentials, registers a fresh incarnation, and only clears the server fence when no assignment is active. Run it while the normal worker process is stopped, then restart the normal worker after the command exits. - -Use a unique worker ID and secret per Code API deployment, expose only the -sandbox loopback endpoint to the CLI, and enforce VM/container egress policy -independently of the bridge transport. diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index cb1b100d..7edd8a26 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,15 +1,22 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; + +import { pairBridgeWorker } from './pairing.js'; +import { + defaultBridgeIdentityPath, + loadBridgeIdentity, + saveBridgeIdentity, +} from './storage.js'; import { BridgeWorker } from './worker.js'; import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, } from './protocol.js'; -function required(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; +function required(name: string, value = process.env[name]): string { + const normalized = value?.trim(); + if (!normalized) throw new Error(`${name} is required`); + return normalized; } function list(value: string | undefined): string[] { @@ -21,72 +28,148 @@ function list(value: string | undefined): string[] { ); } -const controller = new AbortController(); -process.once('SIGINT', () => controller.abort()); -process.once('SIGTERM', () => controller.abort()); +function option(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + if (index >= 0) return args[index + 1]; + return args.find((value) => value.startsWith(`${name}=`))?.slice(name.length + 1); +} -const policy = process.env.LIBRECHAT_CODE_POLICY ?? 'default-deny'; -const statefulWorkspace = - process.env.LIBRECHAT_CODE_STATEFUL_WORKSPACE?.trim().toLowerCase() === 'true'; -const sandboxEndpoint = - process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? - 'http://127.0.0.1:2000/api/v2'; -if (statefulWorkspace && !sandboxEndpoint.includes('{runtimeSessionId}')) { - throw new Error( - 'LIBRECHAT_CODE_STATEFUL_WORKSPACE requires LIBRECHAT_CODE_SANDBOX_ENDPOINT to contain {runtimeSessionId}', +async function pair(args: string[]): Promise { + const codeApiUrl = required('instance URL', args[1]); + const code = required('one-time pairing code', args[2]); + const workerId = required( + '--worker-id or LIBRECHAT_CODE_WORKER_ID', + option(args, '--worker-id') ?? process.env.LIBRECHAT_CODE_WORKER_ID, ); -} -const workerId = required('LIBRECHAT_CODE_WORKER_ID'); -if (!isValidBridgeWorkerId(workerId)) { - throw new Error( - 'LIBRECHAT_CODE_WORKER_ID must match the bridge worker ID format', + const identityPath = + option(args, '--identity') ?? + process.env.LIBRECHAT_CODE_IDENTITY_FILE ?? + defaultBridgeIdentityPath(workerId); + const identity = await pairBridgeWorker({ codeApiUrl, workerId, code }); + await saveBridgeIdentity(identityPath, identity); + process.stdout.write( + `Paired worker ${workerId}. Identity saved to ${identityPath}\n`, ); } -const capabilities = { - statefulWorkspace, - sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? 'nsjail', - runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), - policyDigest: createHash('sha256').update(policy).digest('hex'), -}; -if (!isValidBridgeWorkerCapabilities(capabilities)) { - throw new Error( - 'LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid', +async function run(runtimeSessionId?: string): Promise { + const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); + const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); + const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); + const identityPath = + configuredIdentityPath ?? + (configuredWorkerId && !configuredToken + ? defaultBridgeIdentityPath(configuredWorkerId) + : undefined); + const pairedIdentity = identityPath + ? await loadBridgeIdentity(identityPath) + : undefined; + const workerId = required( + 'LIBRECHAT_CODE_WORKER_ID', + configuredWorkerId ?? pairedIdentity?.workerId, + ); + if (!isValidBridgeWorkerId(workerId)) { + throw new Error( + 'LIBRECHAT_CODE_WORKER_ID must match the bridge worker ID format', + ); + } + if (pairedIdentity && pairedIdentity.workerId !== workerId) { + throw new Error( + `Identity belongs to ${pairedIdentity.workerId}, not configured worker ${workerId}`, + ); + } + const codeApiUrl = required( + 'LIBRECHAT_CODE_URL', + process.env.LIBRECHAT_CODE_URL ?? pairedIdentity?.codeApiUrl, ); + const policy = process.env.LIBRECHAT_CODE_POLICY ?? 'default-deny'; + const statefulWorkspace = + process.env.LIBRECHAT_CODE_STATEFUL_WORKSPACE?.trim().toLowerCase() === + 'true'; + const sandboxEndpoint = + process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? + 'http://127.0.0.1:2000/api/v2'; + if (statefulWorkspace && !sandboxEndpoint.includes('{runtimeSessionId}')) { + throw new Error( + 'LIBRECHAT_CODE_STATEFUL_WORKSPACE requires LIBRECHAT_CODE_SANDBOX_ENDPOINT to contain {runtimeSessionId}', + ); + } + const workerIdentity = pairedIdentity + ? { + privateKey: pairedIdentity.privateKey, + credential: pairedIdentity.credential, + expiresAt: pairedIdentity.expiresAt, + } + : undefined; + const capabilities = { + statefulWorkspace, + sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? 'nsjail', + runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), + policyDigest: createHash('sha256').update(policy).digest('hex'), + }; + if (!isValidBridgeWorkerCapabilities(capabilities)) { + throw new Error( + 'LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid', + ); + } + const controller = new AbortController(); + process.once('SIGINT', () => controller.abort()); + process.once('SIGTERM', () => controller.abort()); + const worker = new BridgeWorker({ + codeApiUrl, + token: configuredToken, + identity: workerIdentity, + workerId, + sandboxEndpoint, + capabilities, + onIdentityChange: + pairedIdentity && identityPath + ? async (identity) => { + await saveBridgeIdentity(identityPath, { + ...pairedIdentity, + credential: identity.credential, + expiresAt: identity.expiresAt, + }); + } + : undefined, + onError: (error) => { + const message = + error instanceof Error ? error.message : 'unknown bridge error'; + process.stderr.write(`librechat-code: reconnecting after ${message}\n`); + }, + }); + if (runtimeSessionId !== undefined) { + await worker.refreshCredential(controller.signal); + await worker.register(controller.signal); + await worker.resetWorkspace(runtimeSessionId, controller.signal); + process.stdout.write( + `librechat-code: reset acknowledged for ${runtimeSessionId}\n`, + ); + return; + } + await worker.run(controller.signal); } -const worker = new BridgeWorker({ - codeApiUrl: required('LIBRECHAT_CODE_URL'), - token: required('LIBRECHAT_CODE_WORKER_TOKEN'), - workerId, - sandboxEndpoint, - capabilities, - onError: (error) => { - const message = error instanceof Error ? error.message : 'unknown bridge error'; - process.stderr.write(`librechat-code: reconnecting after ${message}\n`); - }, -}); async function main(): Promise { - const command = process.argv[2]; - if (command === 'reset-workspace') { - const runtimeSessionId = process.argv[3]?.trim(); + const args = process.argv.slice(2); + if (args[0] === 'pair') { + await pair(args); + return; + } + if (args[0] === 'reset-workspace') { + const runtimeSessionId = args[1]?.trim(); if (!runtimeSessionId) { throw new Error( 'Usage: librechat-code reset-workspace ', ); } - await worker.register(controller.signal); - await worker.resetWorkspace(runtimeSessionId, controller.signal); - process.stdout.write( - `librechat-code: reset acknowledged for ${runtimeSessionId}\n`, - ); + await run(runtimeSessionId); return; } - if (command != null) { - throw new Error(`Unknown command: ${command}`); + if (args[0] && args[0] !== 'run') { + throw new Error(`Unknown command: ${args[0]}`); } - await worker.run(controller.signal); + await run(); } - main().catch((error: Error) => { process.stderr.write(`librechat-code: ${error.message}\n`); process.exitCode = 1; diff --git a/packages/code/src/identity.test.ts b/packages/code/src/identity.test.ts new file mode 100644 index 00000000..4e92b91f --- /dev/null +++ b/packages/code/src/identity.test.ts @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + createBridgeIdentity, + signBridgeRequest, + verifyBridgeRequest, +} from './identity.js'; + +test('worker identity proves possession for the exact HTTP request', () => { + const identity = createBridgeIdentity(); + const request = { + credential: 'short-lived-credential', + method: 'POST', + path: '/v1/bridge/workers/vm-1/lease', + timestamp: new Date().toISOString(), + nonce: 'single-use-request-nonce', + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + + const signature = signBridgeRequest(identity.privateKey, request); + + assert.equal( + verifyBridgeRequest(identity.publicKey, request, signature), + true, + ); + assert.equal( + verifyBridgeRequest( + identity.publicKey, + { ...request, body: JSON.stringify({ protocolVersion: 1, waitMs: 0 }) }, + signature, + ), + false, + ); +}); diff --git a/packages/code/src/identity.ts b/packages/code/src/identity.ts new file mode 100644 index 00000000..ab11c865 --- /dev/null +++ b/packages/code/src/identity.ts @@ -0,0 +1,66 @@ +import { + createHash, + generateKeyPairSync, + sign, + verify, +} from 'node:crypto'; + +export interface BridgeIdentity { + publicKey: string; + privateKey: string; +} + +export interface BridgeRequestProofInput { + credential: string; + method: string; + path: string; + timestamp: string; + nonce: string; + body: string; +} + +export function createBridgeIdentity(): BridgeIdentity { + const { publicKey, privateKey } = generateKeyPairSync('ed25519', { + publicKeyEncoding: { type: 'spki', format: 'pem' }, + privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, + }); + return { publicKey, privateKey }; +} + +function canonicalBridgeRequest(input: BridgeRequestProofInput): string { + const bodyDigest = createHash('sha256').update(input.body).digest('hex'); + return [ + input.method.toUpperCase(), + input.path, + input.timestamp, + input.nonce, + bodyDigest, + input.credential, + ].join('\n'); +} + +export function signBridgeRequest( + privateKey: string, + input: BridgeRequestProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRequest(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRequest( + publicKey: string, + input: BridgeRequestProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRequest(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index c5eeaafc..c65b9f15 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -1,2 +1,5 @@ export * from './protocol.js'; +export * from './identity.js'; +export * from './pairing.js'; +export * from './storage.js'; export * from './worker.js'; diff --git a/packages/code/src/pairing.test.ts b/packages/code/src/pairing.test.ts new file mode 100644 index 00000000..2b564394 --- /dev/null +++ b/packages/code/src/pairing.test.ts @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { pairBridgeWorker } from './pairing.js'; + +test('pairing binds a generated worker key to a single-use code', async () => { + const fetchImpl: typeof fetch = async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + workerId: string; + code: string; + publicKey: string; + }; + assert.equal(body.workerId, 'vm-1'); + assert.equal(body.code, 'one-time-code'); + assert.match(body.publicKey, /BEGIN PUBLIC KEY/); + return Response.json({ + protocolVersion: 1, + workerId: body.workerId, + credential: 'issued-short-lived-credential-value', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + }); + }; + + const paired = await pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1/', + workerId: 'vm-1', + code: 'one-time-code', + fetchImpl, + }); + + assert.equal(paired.workerId, 'vm-1'); + assert.equal(paired.codeApiUrl, 'https://code.example/v1'); + assert.equal(paired.credential, 'issued-short-lived-credential-value'); + assert.match(paired.publicKey, /BEGIN PUBLIC KEY/); + assert.match(paired.privateKey, /BEGIN PRIVATE KEY/); +}); diff --git a/packages/code/src/pairing.ts b/packages/code/src/pairing.ts new file mode 100644 index 00000000..ed33fb18 --- /dev/null +++ b/packages/code/src/pairing.ts @@ -0,0 +1,73 @@ +import { createBridgeIdentity } from './identity.js'; +import { + BRIDGE_PROTOCOL_VERSION, + BridgeProtocolError, +} from './protocol.js'; + +import type { BridgeWorkerCredentialResponse } from './protocol.js'; + +export interface PairBridgeWorkerOptions { + codeApiUrl: string; + workerId: string; + code: string; + fetchImpl?: typeof fetch; +} + +export interface PairedBridgeWorkerIdentity + extends BridgeWorkerCredentialResponse { + codeApiUrl: string; + publicKey: string; + privateKey: string; +} + +function normalizedBaseUrl(value: string): string { + return value.replace(/\/+$/, ''); +} + +function errorMessage(value: object): string | undefined { + if ('error' in value && typeof value.error === 'string') return value.error; + return undefined; +} + +export async function pairBridgeWorker( + options: PairBridgeWorkerOptions, +): Promise { + const codeApiUrl = normalizedBaseUrl(options.codeApiUrl); + const identity = createBridgeIdentity(); + const response = await (options.fetchImpl ?? fetch)( + `${codeApiUrl}/bridge/pairings/redeem`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: options.workerId, + code: options.code, + publicKey: identity.publicKey, + }), + }, + ); + const payload = (await response.json()) as object; + if (!response.ok) { + throw new BridgeProtocolError( + errorMessage(payload) ?? `Bridge pairing failed with HTTP ${response.status}`, + response.status, + ); + } + const credential = payload as BridgeWorkerCredentialResponse; + if ( + credential.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + credential.workerId !== options.workerId || + typeof credential.credential !== 'string' || + credential.credential.length < 32 || + !Number.isFinite(Date.parse(credential.expiresAt)) + ) { + throw new BridgeProtocolError('Code API returned an invalid worker credential'); + } + return { + ...credential, + codeApiUrl, + publicKey: identity.publicKey, + privateKey: identity.privateKey, + }; +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 9a5d2ae0..b2027e1c 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -28,6 +28,20 @@ export interface BridgeWorkerRegistrationResponse { leaseTtlMs: number; } +export interface BridgePairingRedemption { + protocolVersion: BridgeProtocolVersion; + workerId: string; + code: string; + publicKey: string; +} + +export interface BridgeWorkerCredentialResponse { + protocolVersion: BridgeProtocolVersion; + workerId: string; + credential: string; + expiresAt: string; +} + export interface BridgeSandboxRequest { body: TBody; headers: Record; diff --git a/packages/code/src/storage.test.ts b/packages/code/src/storage.test.ts new file mode 100644 index 00000000..ddd782b0 --- /dev/null +++ b/packages/code/src/storage.test.ts @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { + defaultBridgeIdentityPath, + loadBridgeIdentity, + saveBridgeIdentity, +} from './storage.js'; + +test('default identity paths do not collide after worker ID sanitization', () => { + assert.notEqual( + defaultBridgeIdentityPath('vm:a'), + defaultBridgeIdentityPath('vm_a'), + ); +}); + +test('paired identity is persisted atomically with owner-only permissions', async () => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-')); + const path = join(directory, 'identity.json'); + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'issued-short-lived-credential-value', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + publicKey: 'public-key', + privateKey: 'private-key', + }; + + try { + await saveBridgeIdentity(path, identity); + + assert.deepEqual(await loadBridgeIdentity(path), identity); + assert.equal((await stat(path)).mode & 0o777, 0o600); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts new file mode 100644 index 00000000..a26c3eb0 --- /dev/null +++ b/packages/code/src/storage.ts @@ -0,0 +1,66 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { chmod, mkdir, open, readFile, rename, rm } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; + +import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; + +import type { PairedBridgeWorkerIdentity } from './pairing.js'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function isPairedIdentity(value: unknown): value is PairedBridgeWorkerIdentity { + if (!isRecord(value)) return false; + return ( + value.protocolVersion === BRIDGE_PROTOCOL_VERSION && + typeof value.workerId === 'string' && + typeof value.codeApiUrl === 'string' && + typeof value.credential === 'string' && + typeof value.expiresAt === 'string' && + Number.isFinite(Date.parse(value.expiresAt)) && + typeof value.publicKey === 'string' && + typeof value.privateKey === 'string' + ); +} + +export function defaultBridgeIdentityPath(workerId: string): string { + const readableName = workerId.replace(/[^A-Za-z0-9._-]/g, '_'); + const fileName = readableName === workerId + ? readableName + : `${readableName}-${createHash('sha256').update(workerId).digest('hex').slice(0, 16)}`; + return join(homedir(), '.config', 'librechat', 'code', `${fileName}.json`); +} + +export async function saveBridgeIdentity( + path: string, + identity: PairedBridgeWorkerIdentity, +): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const temporaryPath = `${path}.${randomBytes(8).toString('hex')}.tmp`; + try { + const file = await open(temporaryPath, 'wx', 0o600); + try { + await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); + await file.sync(); + } finally { + await file.close(); + } + await rename(temporaryPath, path); + await chmod(path, 0o600); + } catch (error) { + await rm(temporaryPath, { force: true }); + throw error; + } +} + +export async function loadBridgeIdentity( + path: string, +): Promise { + const identity = JSON.parse(await readFile(path, 'utf8')) as unknown; + if (!isPairedIdentity(identity)) { + throw new BridgeProtocolError(`Invalid bridge identity file: ${path}`); + } + return identity; +} diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index d633366e..fc88c6e3 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -1,9 +1,16 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; +import { createBridgeIdentity, verifyBridgeRequest } from './identity.js'; +import { + BridgeWorker, + BridgeWorkspaceQuarantinedError, + reconnectDelayMs, +} from './worker.js'; import type { BridgeAssignment } from './protocol.js'; +const incarnationId = 'incarnation-00000001'; + test('worker forwards a fenced assignment to the sandbox and settles the result', async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; const fetchImpl: typeof fetch = async (input, init) => { @@ -1663,3 +1670,840 @@ test('worker uses the server-relative lease budget despite VM clock skew', async }); assert.equal(settlementAttempted, true); }); + + +test('worker continues after an expired assignment settlement conflict', async () => { + const controller = new AbortController(); + let registrations = 0; + let leases = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + reconnectDelayMs: 0, + reconnectMaxDelayMs: 0, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/workers/register')) { + registrations += 1; + if (registrations === 2) controller.abort(); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + } + if (url.endsWith('/lease')) { + leases += 1; + return Response.json({ + protocolVersion: 1, + assignment: leases === 1 + ? { + protocolVersion: 1, + assignmentId: 'assignment-expired', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + } + : undefined, + }); + } + if (url.endsWith('/execute')) { + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/settle')) { + return Response.json( + { error: 'Bridge assignment has expired', code: 'ASSIGNMENT_EXPIRED' }, + { status: 409 }, + ); + } + return Response.json({ cancelled: false }); + }, + }); + + await worker.run(controller.signal); + + assert.equal(registrations, 2); +}); + +test('paired worker proves possession on bridge requests', async () => { + const key = createBridgeIdentity(); + let bridgeRequest: { url: string; init?: RequestInit } | undefined; + const fetchImpl: typeof fetch = async (input, init) => { + bridgeRequest = { url: String(input), init }; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'issued-short-lived-credential-value', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + }, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await worker.register(); + + assert.ok(bridgeRequest); + const headers = bridgeRequest.init?.headers as Record; + const body = String(bridgeRequest.init?.body); + assert.equal( + verifyBridgeRequest( + key.publicKey, + { + credential: 'issued-short-lived-credential-value', + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: headers['X-LibreChat-Code-Timestamp'], + nonce: headers['X-LibreChat-Code-Nonce'], + body, + }, + headers['X-LibreChat-Code-Signature'], + ), + true, + ); +}); + +test('paired worker rotates an expiring credential before registration', async () => { + const key = createBridgeIdentity(); + const requests: Array<{ url: string; init?: RequestInit }> = []; + let persistedCredential = ''; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + requests.push({ url, init }); + if (url.endsWith('/credentials/refresh')) { + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'rotated-short-lived-credential-value', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + }); + } + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'original-short-lived-credential-value', + expiresAt: new Date(Date.now() + 30_000).toISOString(), + }, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + onIdentityChange: (identity) => { + persistedCredential = identity.credential; + }, + }); + + await worker.refreshCredential(); + await worker.register(); + + assert.equal(persistedCredential, 'rotated-short-lived-credential-value'); + assert.equal( + (requests[1].init?.headers as Record).Authorization, + 'Bridge rotated-short-lived-credential-value', + ); +}); + +test('paired worker retries persistence before adopting a rotated credential', async () => { + const key = createBridgeIdentity(); + const identity = { + privateKey: key.privateKey, + credential: 'original-short-lived-credential-value', + expiresAt: new Date(Date.now() + 30_000).toISOString(), + }; + let persistenceAttempts = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async () => + Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'rotated-short-lived-credential-value', + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }), + onIdentityChange: () => { + persistenceAttempts += 1; + if (persistenceAttempts === 1) throw new Error('disk unavailable'); + }, + }); + + await assert.rejects(worker.refreshCredential(), /disk unavailable/); + assert.equal(identity.credential, 'original-short-lived-credential-value'); + await worker.refreshCredential(); + assert.equal(identity.credential, 'rotated-short-lived-credential-value'); + assert.equal(persistenceAttempts, 2); +}); + +test('paired worker refreshes before an assignment that outlives its credential', async () => { + const key = createBridgeIdentity(); + const requests: string[] = []; + const fetchImpl: typeof fetch = async (input) => { + const url = String(input); + requests.push(url); + if (url.endsWith('/credentials/refresh')) { + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'assignment-safe-rotated-credential-value', + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }); + } + if (url.endsWith('/execute')) { + return Response.json({ session_id: 'run-long', files: [] }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-too-short-for-assignment', + expiresAt: new Date(Date.now() + 30_000).toISOString(), + }, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-long', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'assignment-long-lease-token-value', + expiresAt: new Date(Date.now() + 5 * 60_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.match(requests[0], /credentials\/refresh$/); + assert.equal(requests[1], 'http://127.0.0.1:2000/api/v2/execute'); +}); + +test('paired worker rotates credentials throughout a long assignment', async () => { + const key = createBridgeIdentity(); + let refreshCount = 0; + const identity = { + privateKey: key.privateKey, + credential: 'credential-before-long-running-assignment', + expiresAt: new Date(Date.now() + 5).toISOString(), + }; + const fetchImpl: typeof fetch = async (input) => { + const url = String(input); + if (url.endsWith('/credentials/refresh')) { + refreshCount += 1; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: `rotated-long-assignment-credential-${refreshCount}`, + expiresAt: new Date(Date.now() + 30).toISOString(), + }); + } + if (url.endsWith('/execute')) { + await new Promise((resolve) => setTimeout(resolve, 55)); + return Response.json({ session_id: 'run-long-rotation', files: [] }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-credential-maintenance', + workerId: 'vm-1', + incarnationId, + generation: 5, + leaseToken: 'assignment-credential-maintenance-token', + expiresAt: new Date(Date.now() + 500).toISOString(), + remainingMs: 500, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.ok(refreshCount >= 2); + assert.match(identity.credential, /^rotated-long-assignment-credential-/); +}); + +test('paired worker cancels a stalled credential refresh after execution', async () => { + const key = createBridgeIdentity(); + let refreshStarted!: () => void; + const started = new Promise((resolve) => { + refreshStarted = resolve; + }); + let refreshAborted = false; + const identity = { + privateKey: key.privateKey, + credential: 'credential-before-stalled-refresh', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity, + credentialRefreshWindowMs: 100, + credentialRefreshTransportTimeoutMs: 10_000, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + const url = String(input); + if (url.endsWith('/credentials/refresh')) { + refreshStarted(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + refreshAborted = true; + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + } + if (url.endsWith('/execute')) { + await started; + return Response.json({ + session_id: 'run-stalled-refresh', + files: [], + }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + queueMicrotask(() => { + identity.expiresAt = new Date(Date.now() + 50).toISOString(); + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-stalled-refresh', + workerId: 'vm-1', + incarnationId, + generation: 6, + leaseToken: 'assignment-stalled-refresh-token', + expiresAt: new Date(Date.now() + 2_000).toISOString(), + remainingMs: 2_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(refreshAborted, true); +}); + +test('paired worker refreshes conservatively before server clock calibration', async () => { + const key = createBridgeIdentity(); + let refreshCount = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-idle-clock-skew-refresh', + expiresAt: new Date(Date.now() + 65_000).toISOString(), + }, + credentialRefreshWindowMs: 10_000, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async () => { + refreshCount += 1; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-idle-clock-skew-refresh', + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }); + }, + }); + + await worker.refreshCredential(); + + assert.equal(refreshCount, 1); +}); + +test('paired worker charges initial credential refresh against the assignment deadline', async () => { + const key = createBridgeIdentity(); + let sandboxStarted = false; + let rejected = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-deadline-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + credentialRefreshTransportTimeoutMs: 10_000, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + const url = String(input); + if (url.endsWith('/credentials/refresh')) { + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + if (url.endsWith('/execute')) { + sandboxStarted = true; + } + if (url.endsWith('/settle')) { + rejected = + JSON.parse(String(init?.body)).status === 'rejected'; + } + return Response.json({ + protocolVersion: 1, + accepted: true, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-deadline-refresh', + workerId: 'vm-1', + incarnationId, + generation: 7, + leaseToken: 'assignment-deadline-refresh-token', + expiresAt: new Date(Date.now() + 30).toISOString(), + remainingMs: 30, + runtimeSessionId: 'rt-deadline-refresh', + request: { body: { language: 'bash' }, headers: {} }, + }), + { name: 'AbortError' }, + ); + + assert.equal(sandboxStarted, false); + assert.equal(rejected, true); +}); + +test('paired worker rechecks the deadline after request serialization', async () => { + const key = createBridgeIdentity(); + let sandboxStarted = false; + let rejected = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: + 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-valid-during-serialization', + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + sandboxStarted = true; + } + if (url.endsWith('/settle')) { + rejected = + JSON.parse(String(init?.body)).status === 'rejected'; + } + return Response.json({ + protocolVersion: 1, + accepted: true, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }, + }); + const slowBody = { + get language(): string { + const blockedUntilMs = Date.now() + 25; + while (Date.now() < blockedUntilMs) { + // Deliberately consume the remaining synchronous request budget. + } + return 'bash'; + }, + }; + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-serialization-deadline', + workerId: 'vm-1', + incarnationId, + generation: 8, + leaseToken: 'assignment-serialization-deadline-token', + expiresAt: new Date(Date.now() + 10).toISOString(), + remainingMs: 10, + runtimeSessionId: 'rt-serialization-deadline', + request: { body: slowBody, headers: {} }, + }); + + assert.equal(sandboxStarted, false); + assert.equal(rejected, true); +}); + +test('paired worker keeps endpoint validation failures known-clean', async () => { + const key = createBridgeIdentity(); + let sandboxStarted = false; + let rejected = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-for-invalid-endpoint', + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) sandboxStarted = true; + if (url.endsWith('/settle')) { + rejected = + JSON.parse(String(init?.body)).status === 'rejected'; + } + return Response.json({ + protocolVersion: 1, + accepted: true, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-invalid-endpoint', + workerId: 'vm-1', + incarnationId, + generation: 9, + leaseToken: 'assignment-invalid-endpoint-token', + expiresAt: new Date(Date.now() + 500).toISOString(), + remainingMs: 500, + runtimeSessionId: 'rt-invalid-endpoint', + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(sandboxStarted, false); + assert.equal(rejected, true); +}); + +test('paired worker rechecks shutdown after persisting a refreshed identity', async () => { + const key = createBridgeIdentity(); + const controller = new AbortController(); + let sandboxStarted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-shutdown-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) sandboxStarted = true; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-persisted-during-shutdown', + expiresAt: new Date(Date.now() + 15 * 60_000).toISOString(), + }); + }, + onIdentityChange: () => { + controller.abort(); + }, + }); + + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'assignment-shutdown-refresh', + workerId: 'vm-1', + incarnationId, + generation: 10, + leaseToken: 'assignment-shutdown-refresh-token', + expiresAt: new Date(Date.now() + 500).toISOString(), + remainingMs: 500, + runtimeSessionId: 'rt-shutdown-refresh', + request: { body: { language: 'bash' }, headers: {} }, + }, + controller.signal, + ), + { name: 'AbortError' }, + ); + + assert.equal(sandboxStarted, false); +}); + +test('paired worker retries transient refresh failures before credential expiry', async () => { + const key = createBridgeIdentity(); + let refreshCount = 0; + const identity = { + privateKey: key.privateKey, + credential: 'credential-before-transient-refresh', + expiresAt: new Date(Date.now() + 40).toISOString(), + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity, + credentialRefreshWindowMs: 15, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/credentials/refresh')) { + refreshCount += 1; + if (refreshCount === 1) { + return Response.json( + { error: 'temporarily unavailable' }, + { status: 503 }, + ); + } + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-transient-refresh', + expiresAt: new Date(Date.now() + 500).toISOString(), + }); + } + if (url.endsWith('/execute')) { + await new Promise((resolve) => setTimeout(resolve, 70)); + return Response.json({ + session_id: 'run-transient-refresh', + files: [], + }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-transient-refresh', + workerId: 'vm-1', + incarnationId, + generation: 7, + leaseToken: 'assignment-transient-refresh-token', + expiresAt: new Date(Date.now() + 500).toISOString(), + remainingMs: 500, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(refreshCount, 2); + assert.equal(identity.credential, 'credential-after-transient-refresh'); +}); + +test('paired worker preserves its refresh margin when the server clock is ahead', async () => { + const key = createBridgeIdentity(); + const serverClockOffsetMs = 55; + let refreshCount = 0; + const identity = { + privateKey: key.privateKey, + credential: 'credential-before-clock-skew-refresh', + expiresAt: new Date(Date.now() + serverClockOffsetMs + 20).toISOString(), + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/credentials/refresh')) { + refreshCount += 1; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-clock-skew-refresh', + expiresAt: new Date( + Date.now() + serverClockOffsetMs + 500, + ).toISOString(), + }); + } + if (url.endsWith('/execute')) { + await new Promise((resolve) => setTimeout(resolve, 35)); + return Response.json({ + session_id: 'run-clock-skew-refresh', + files: [], + }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + const remainingMs = 500; + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-clock-skew-refresh', + workerId: 'vm-1', + incarnationId, + generation: 8, + leaseToken: 'assignment-clock-skew-refresh-token', + expiresAt: new Date( + Date.now() + serverClockOffsetMs + remainingMs, + ).toISOString(), + remainingMs, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(refreshCount, 1); + assert.equal(identity.credential, 'credential-after-clock-skew-refresh'); +}); + +test('worker shutdown interrupts reconnect backoff', async () => { + const controller = new AbortController(); + let failed!: () => void; + const failure = new Promise((resolve) => { + failed = resolve; + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async () => { + throw new Error('offline'); + }, + reconnectDelayMs: 30_000, + reconnectMaxDelayMs: 30_000, + onError: () => failed(), + }); + + const run = worker.run(controller.signal); + await failure; + controller.abort(); + await run; +}); + +test('reconnect delay uses bounded exponential jitter', () => { + assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 0), 500); + assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 1), 1_000); + assert.equal(reconnectDelayMs(10, 1_000, 30_000, () => 1), 30_000); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 5c1afb95..7c313fee 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -5,6 +5,7 @@ import { BridgeProtocolError, bridgeWorkerPath, } from './protocol.js'; +import { signBridgeRequest } from './identity.js'; import type { BridgeAssignment, @@ -12,12 +13,14 @@ import type { BridgeSettlement, BridgeSettlementResponse, BridgeWorkerCapabilities, + BridgeWorkerCredentialResponse, BridgeWorkerRegistrationResponse, } from './protocol.js'; export interface BridgeWorkerOptions { codeApiUrl: string; - token: string; + token?: string; + identity?: BridgeWorkerIdentity; workerId: string; sandboxEndpoint: string; capabilities: BridgeWorkerCapabilities; @@ -30,15 +33,29 @@ export interface BridgeWorkerOptions { cancellationTransportTimeoutMs?: number; rejectionAckGraceMs?: number; reconnectDelayMs?: number; + reconnectMaxDelayMs?: number; + reconnectRandom?: () => number; + credentialRefreshWindowMs?: number; + credentialRefreshTransportTimeoutMs?: number; fetchImpl?: typeof fetch; onError?: (error: unknown) => void; + onIdentityChange?: (identity: BridgeWorkerIdentity) => void | Promise; incarnationId?: string; } +export interface BridgeWorkerIdentity { + privateKey: string; + credential: string; + expiresAt: string; +} + const DEFAULT_LEASE_WAIT_MS = 25_000; const MAX_LEASE_WAIT_MS = 30_000; const DEFAULT_LEASE_TRANSPORT_GRACE_MS = 5_000; const DEFAULT_RECONNECT_DELAY_MS = 1_000; +const DEFAULT_RECONNECT_MAX_DELAY_MS = 30_000; +const CREDENTIAL_REFRESH_WINDOW_MS = 60_000; +const MAX_PROOF_CLOCK_SKEW_MS = 60_000; const DEFAULT_REGISTRATION_TTL_MS = 60_000; const DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS = 10_000; @@ -46,11 +63,22 @@ const DEFAULT_CANCELLATION_POLL_INTERVAL_MS = 500; const DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS = 2_000; const MIN_REGISTRATION_HEARTBEAT_MS = 25; const REGISTRATION_RETRY_DELAY_MS = 100; +const CREDENTIAL_REFRESH_RETRY_DELAY_MS = 100; const SETTLEMENT_RETRY_DELAY_MS = 100; const REJECTION_ACK_GRACE_MS = 30_000; const MAX_SETTLEMENT_ERROR_LENGTH = 4_096; const RUNTIME_SESSION_PLACEHOLDER = '{runtimeSessionId}'; +export function reconnectDelayMs( + attempt: number, + baseDelayMs = DEFAULT_RECONNECT_DELAY_MS, + maxDelayMs = DEFAULT_RECONNECT_MAX_DELAY_MS, + random: () => number = Math.random, +): number { + const cap = Math.min(maxDelayMs, baseDelayMs * 2 ** Math.max(0, attempt)); + return Math.floor(cap * (0.5 + Math.min(1, Math.max(0, random())) * 0.5)); +} + function normalizedBaseUrl(value: string): string { return value.replace(/\/+$/, ''); } @@ -60,6 +88,21 @@ function errorMessage(value: object): string | undefined { return undefined; } +async function abortableDelay(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted === true) return; + await new Promise((resolve) => { + const onAbort = (): void => { + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + function errorCode(value: object): string | undefined { if ('code' in value && typeof value.code === 'string') return value.code; return undefined; @@ -82,8 +125,14 @@ export class BridgeWorker { private readonly incarnationId: string; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; + private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; constructor(private readonly options: BridgeWorkerOptions) { + if (!options.token && !options.identity) { + throw new BridgeProtocolError( + 'Bridge worker requires a static token or paired identity', + ); + } this.fetchImpl = options.fetchImpl ?? fetch; this.codeApiUrl = normalizedBaseUrl(options.codeApiUrl); this.sandboxEndpoint = normalizedBaseUrl(options.sandboxEndpoint); @@ -132,6 +181,10 @@ export class BridgeWorker { 'Code API registered a different worker incarnation', ); } + const registeredAtMs = Date.parse(registration.registeredAt); + if (Number.isFinite(registeredAtMs)) { + this.serverClockOffsetMs = registeredAtMs - registrationStartedAtMs; + } this.registrationTtlMs = registration.leaseTtlMs; this.lastRegisteredAtMs = registrationStartedAtMs; return registration; @@ -288,10 +341,13 @@ export class BridgeWorker { } async run(signal?: AbortSignal): Promise { + let reconnectAttempt = 0; while (!signal?.aborted) { try { + await this.refreshCredential(signal); await this.register(signal); const assignment = await this.lease(signal); + reconnectAttempt = 0; if (!assignment) continue; await this.executeAndSettle(assignment, signal); } catch (error) { @@ -309,9 +365,110 @@ export class BridgeWorker { throw error; } this.options.onError?.(error); - const delay = - this.options.reconnectDelayMs ?? DEFAULT_RECONNECT_DELAY_MS; - await new Promise((resolve) => setTimeout(resolve, delay)); + const delay = reconnectDelayMs( + reconnectAttempt, + this.options.reconnectDelayMs, + this.options.reconnectMaxDelayMs, + this.options.reconnectRandom, + ); + reconnectAttempt += 1; + await abortableDelay(delay, signal); + } + } + } + + async refreshCredential( + signal?: AbortSignal, + validThroughMs = + Date.now() + + this.serverClockOffsetMs + + (this.options.credentialRefreshWindowMs ?? CREDENTIAL_REFRESH_WINDOW_MS), + transportTimeoutMs = Number.POSITIVE_INFINITY, + ): Promise { + const identity = this.options.identity; + if (identity == null) return; + if ( + Date.parse(identity.expiresAt) > validThroughMs + ) { + return; + } + const credential = await this.timedRequest( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}` + + '/credentials/refresh', + { protocolVersion: BRIDGE_PROTOCOL_VERSION }, + Math.max( + 1, + Math.min( + transportTimeoutMs, + this.options.credentialRefreshTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, + ), + ), + signal, + ); + if ( + credential.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + credential.workerId !== this.options.workerId || + typeof credential.credential !== 'string' || + credential.credential.length < 32 || + !Number.isFinite(Date.parse(credential.expiresAt)) || + Date.parse(credential.expiresAt) <= validThroughMs + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid rotated worker credential', + ); + } + const rotatedIdentity: BridgeWorkerIdentity = { + ...identity, + credential: credential.credential, + expiresAt: credential.expiresAt, + }; + await this.options.onIdentityChange?.(rotatedIdentity); + identity.credential = rotatedIdentity.credential; + identity.expiresAt = rotatedIdentity.expiresAt; + } + + private async maintainCredential( + assignment: BridgeAssignment, + stopSignal: AbortSignal, + serverClockOffsetMs: number, + ): Promise { + const identity = this.options.identity; + if (identity == null) return; + const refreshWindowMs = + this.options.credentialRefreshWindowMs ?? CREDENTIAL_REFRESH_WINDOW_MS; + const assignmentDeadlineMs = + Date.parse(assignment.expiresAt) - serverClockOffsetMs; + while (!stopSignal.aborted && Date.now() < assignmentDeadlineMs) { + const refreshAtMs = + Date.parse(identity.expiresAt) - serverClockOffsetMs - refreshWindowMs; + const waitMs = Math.max( + 0, + Math.min(refreshAtMs - Date.now(), assignmentDeadlineMs - Date.now()), + ); + await abortableDelay(waitMs, stopSignal); + if (stopSignal.aborted || Date.now() >= assignmentDeadlineMs) return; + try { + await this.refreshCredential( + stopSignal, + Date.now() + serverClockOffsetMs + refreshWindowMs, + ); + } catch (error) { + if (stopSignal.aborted) return; + const terminal = + error instanceof BridgeProtocolError && + (error.status === 401 || error.status === 403); + const credentialRemainingMs = + Date.parse(identity.expiresAt) - + (Date.now() + serverClockOffsetMs); + if (terminal || credentialRemainingMs <= 0) throw error; + await abortableDelay( + Math.min( + CREDENTIAL_REFRESH_RETRY_DELAY_MS, + Math.max(1, Math.floor(credentialRemainingMs / 2)), + ), + stopSignal, + ); } } } @@ -325,11 +482,56 @@ export class BridgeWorker { ? signal.reason : new DOMException('aborted', 'AbortError'); } + const serverClockOffsetMs = + Number.isSafeInteger(assignment.remainingMs) && + (assignment.remainingMs ?? -1) >= 0 + ? Date.parse(assignment.expiresAt) - + (Date.now() + (assignment.remainingMs ?? 0)) + : 0; + this.serverClockOffsetMs = serverClockOffsetMs; + const localDeadlineAtMs = + Date.now() + this.assignmentRemainingMs(assignment); + try { + await this.refreshCredential( + signal, + Date.now() + + serverClockOffsetMs + + (this.options.credentialRefreshWindowMs ?? + CREDENTIAL_REFRESH_WINDOW_MS), + Math.max(1, localDeadlineAtMs - Date.now()), + ); + } catch (error) { + if (!signal?.aborted) { + await this.rejectUnexecutedAssignment( + assignment, + 'Bridge credential refresh failed before sandbox execution', + ); + } + throw error; + } + const remainingAfterRefreshMs = localDeadlineAtMs - Date.now(); + if (remainingAfterRefreshMs <= 0) { + await this.rejectUnexecutedAssignment( + assignment, + 'Bridge assignment expired during credential refresh', + ); + throw new BridgeProtocolError( + 'Bridge assignment expired during credential refresh', + ); + } + if (signal != null && Boolean(signal.aborted)) { + throw signal.reason instanceof Error + ? signal.reason + : new DOMException('aborted', 'AbortError'); + } const executionController = new AbortController(); - const abortExecution = (): void => executionController.abort(); + const credentialController = new AbortController(); + const abortExecution = (): void => { + executionController.abort(); + credentialController.abort(); + }; signal?.addEventListener('abort', abortExecution, { once: true }); - const deadlineDelay = this.assignmentRemainingMs(assignment); - const localDeadlineAtMs = Date.now() + deadlineDelay; + const deadlineDelay = remainingAfterRefreshMs; const deadlineTimer = setTimeout( () => executionController.abort(), deadlineDelay, @@ -351,10 +553,23 @@ export class BridgeWorker { executionController, cancellationController.signal, ); + let credentialMaintenanceError: unknown; + let credentialMaintenance: Promise | undefined; let settlement: BridgeSettlement; let ambiguousSandboxError: unknown; let sandboxRejectedExecution = false; + let sandboxStarted = false; try { + credentialMaintenance = this.maintainCredential( + assignment, + credentialController.signal, + serverClockOffsetMs, + ).catch((error) => { + credentialMaintenanceError = error; + executionController.abort(); + }); + const sandboxExecuteUrl = + `${this.sandboxEndpointFor(assignment)}/execute`; const sandboxSessionId = this.sandboxSessionIdFor(assignment); const headers = { ...assignment.request.headers, @@ -362,15 +577,22 @@ export class BridgeWorker { ? { 'X-Runtime-Session-Id': sandboxSessionId } : {}), }; + const sandboxRequestBody = JSON.stringify(assignment.request.body); + if (Date.now() >= localDeadlineAtMs) { + throw new BridgeProtocolError( + 'Bridge assignment expired before sandbox execution', + ); + } + sandboxStarted = true; const response = await this.fetchImpl( - `${this.sandboxEndpointFor(assignment)}/execute`, + sandboxExecuteUrl, { method: 'POST', headers: { ...headers, 'Content-Type': 'application/json', }, - body: JSON.stringify(assignment.request.body), + body: sandboxRequestBody, signal: executionController.signal, }, ); @@ -380,6 +602,9 @@ export class BridgeWorker { } catch (error) { if (response.ok) throw error; } + if (credentialMaintenanceError != null) { + throw credentialMaintenanceError; + } if (!response.ok) { sandboxRejectedExecution = response.status >= 400 && @@ -405,6 +630,7 @@ export class BridgeWorker { } catch (error) { if ( assignment.runtimeSessionId != null && + sandboxStarted && !sandboxRejectedExecution ) { ambiguousSandboxError = error; @@ -436,7 +662,7 @@ export class BridgeWorker { const knownCleanStatefulRejection = assignment.runtimeSessionId != null && settlement.status === 'rejected' && - sandboxRejectedExecution; + (!sandboxStarted || sandboxRejectedExecution); if (knownCleanStatefulRejection) { heartbeatController.abort(); await heartbeat; @@ -470,6 +696,8 @@ export class BridgeWorker { } finally { heartbeatController.abort(); await heartbeat; + credentialController.abort(); + await credentialMaintenance; signal?.removeEventListener('abort', abortExecution); } } @@ -520,6 +748,10 @@ export class BridgeWorker { ); } + private async delay(ms: number, signal: AbortSignal): Promise { + await abortableDelay(ms, signal); + } + private async maintainRegistration( signal: AbortSignal, retryTransient = false, @@ -584,21 +816,6 @@ export class BridgeWorker { } } - private async delay(ms: number, signal: AbortSignal): Promise { - if (signal.aborted) return; - await new Promise((resolve) => { - const onAbort = (): void => { - clearTimeout(timer); - resolve(); - }; - const timer = setTimeout(() => { - signal.removeEventListener('abort', onAbort); - resolve(); - }, ms); - signal.addEventListener('abort', onAbort, { once: true }); - }); - } - private assignmentUrl(assignment: BridgeAssignment, action: string): string { return ( `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}` + @@ -743,13 +960,14 @@ export class BridgeWorker { body: object, signal?: AbortSignal, ): Promise { + const requestBody = JSON.stringify(body); const response = await this.fetchImpl(url, { method: 'POST', headers: { - Authorization: `Bearer ${this.options.token}`, + ...this.authorizationHeaders(url, requestBody), 'Content-Type': 'application/json', }, - body: JSON.stringify(body), + body: requestBody, signal, }); let payload: unknown; @@ -772,6 +990,35 @@ export class BridgeWorker { return payload as T; } + private authorizationHeaders( + url: string, + body: string, + ): Record { + const identity = this.options.identity; + if (identity == null) { + return { Authorization: `Bearer ${this.options.token}` }; + } + const timestamp = new Date().toISOString(); + const nonce = randomBytes(18).toString('base64url'); + const proof = { + credential: identity.credential, + method: 'POST', + path: new URL(url).pathname, + timestamp, + nonce, + body, + }; + return { + Authorization: `Bridge ${identity.credential}`, + 'X-LibreChat-Code-Timestamp': timestamp, + 'X-LibreChat-Code-Nonce': nonce, + 'X-LibreChat-Code-Signature': signBridgeRequest( + identity.privateKey, + proof, + ), + }; + } + private async timedRequest( url: string, body: object, diff --git a/service/rollup.config.js b/service/rollup.config.js index 0e7a8d8b..059c71d1 100644 --- a/service/rollup.config.js +++ b/service/rollup.config.js @@ -42,6 +42,7 @@ export default { 'src/**/*.ts', '../shared/telemetry-core.ts', '../packages/code/src/protocol.ts', + '../packages/code/src/identity.ts', ], sourceMap: true, declaration: false, diff --git a/service/src/api-server.ts b/service/src/api-server.ts index 8578460b..1e4634a4 100644 --- a/service/src/api-server.ts +++ b/service/src/api-server.ts @@ -18,7 +18,7 @@ import { requestErrorLogger, requestNotFoundLogger } from './middleware/request- import { localAuth } from './auth/local'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; -import bridgeRouter from './bridge/router'; +import bridgeRouter from './bridge'; import { connection } from './queue'; import { metricsHandler } from './metrics'; import { httpMetricsMiddleware } from './middleware/httpMetrics'; diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts new file mode 100644 index 00000000..dfba2a8b --- /dev/null +++ b/service/src/bridge/index.ts @@ -0,0 +1,16 @@ +import { connection } from '../queue'; +import { env } from '../config'; +import { RedisBridgePairingStore } from './pairing'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; + +export const bridgeStore = new RedisBridgeStore(connection); +export const bridgePairings = new RedisBridgePairingStore(connection); + +export default createBridgeRouter({ + store: bridgeStore, + pairings: bridgePairings, + authMode: env.BRIDGE_AUTH_MODE, + adminToken: env.BRIDGE_TOKEN, + configuredWorkerId: env.BRIDGE_WORKER_ID, +}); diff --git a/service/src/bridge/pairing.test.ts b/service/src/bridge/pairing.test.ts new file mode 100644 index 00000000..7b5ab124 --- /dev/null +++ b/service/src/bridge/pairing.test.ts @@ -0,0 +1,349 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; + +import { + createBridgeIdentity, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const pairings = new RedisBridgePairingStore(redis); +const store = new RedisBridgeStore(redis); + +afterEach(async () => { + await redis.flushall(); +}); + +describe('RedisBridgePairingStore', () => { + test('redeems a pairing code exactly once for the intended worker identity', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + + const credential = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + + expect(credential.workerId).toBe('vm-1'); + expect(credential.credential.length).toBeGreaterThanOrEqual(32); + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + }); + + test('authorizes a credential only with proof from its worker key', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const issued = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path: '/v1/bridge/workers/vm-1/lease', + timestamp: new Date().toISOString(), + nonce: 'request-nonce-1', + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + + await expect( + pairings.authorize({ + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('rejects replay of an already accepted worker proof', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const issued = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path: '/v1/bridge/workers/vm-1/lease', + timestamp: new Date().toISOString(), + nonce: 'single-use-nonce', + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + const request = { + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }; + + await pairings.authorize(request); + + await expect(pairings.authorize(request)).rejects.toMatchObject({ + code: 'PROOF_REPLAYED', + }); + }); + + test('rejects a correctly signed proof outside the clock window', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const issued = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path: '/v1/bridge/workers/vm-1/lease', + timestamp: new Date(Date.now() - 5 * 60_000).toISOString(), + nonce: 'stale-request-nonce', + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + + await expect( + pairings.authorize({ + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }), + ).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + }); + + test('revocation immediately invalidates the active worker credential', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const issued = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce: 'post-revocation-request', + body: JSON.stringify({ protocolVersion: 1, workerId: 'vm-1' }), + }; + await store.register({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }); + + await pairings.revoke('vm-1'); + + await expect( + pairings.authorize({ + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }), + ).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + expect(await redis.get('codeapi:bridge:v1:worker:vm-1')).toBeNull(); + expect( + await redis.get('codeapi:bridge:v1:worker:vm-1:incarnation'), + ).toBeNull(); + await expect( + store.register({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_FENCED' }); + }); + + test('revocation fences a registration authorized before the revoke', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const issued = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce: 'registration-revoke-race', + body: JSON.stringify({ protocolVersion: 1, workerId: 'vm-1' }), + }; + const authorization = await pairings.authorize({ + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }); + + await pairings.revoke('vm-1'); + + await expect( + store.register( + { + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000002', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }, + authorization, + ), + ).rejects.toMatchObject({ code: 'WORKER_FENCED' }); + expect(await redis.get('codeapi:bridge:v1:worker:vm-1')).toBeNull(); + expect( + await redis.get('codeapi:bridge:v1:worker:vm-1:incarnation'), + ).toBeNull(); + }); + + test('revocation invalidates pairing codes issued before the revoke', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + + await pairings.revoke('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + }); + + test('rotation keeps the prior same-identity credential usable for recovery', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const original = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + + const rotated = await pairings.rotate('vm-1'); + + const proofFor = ( + credential: string, + nonce: string, + ): Parameters[0] => { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/vm-1/lease', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + return { + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }; + }; + + await expect( + pairings.authorize(proofFor(original.credential, 'old-credential')), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + await expect( + pairings.authorize(proofFor(rotated.credential, 'new-credential')), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('recovers when a refresh response is lost after the server commits it', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + const original = await pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }); + const proofFor = ( + credential: string, + nonce: string, + ): Parameters[0] => { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/vm-1/credentials/refresh', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1 }), + }; + return { + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(identity.privateKey, proof), + }; + }; + + await pairings.rotate('vm-1'); + const retryAuthorization = await pairings.authorize( + proofFor(original.credential, 'refresh-response-lost'), + ); + const recovered = await pairings.rotate( + 'vm-1', + retryAuthorization.credentialId, + ); + + await expect( + pairings.authorize(proofFor(recovered.credential, 'refresh-recovered')), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('repairing a worker invalidates its previously paired credential', async () => { + const firstIdentity = createBridgeIdentity(); + const firstPairing = await pairings.issue('vm-1'); + const first = await pairings.redeem({ + workerId: 'vm-1', + code: firstPairing.code, + publicKey: firstIdentity.publicKey, + }); + const nextIdentity = createBridgeIdentity(); + const nextPairing = await pairings.issue('vm-1'); + await pairings.redeem({ + workerId: 'vm-1', + code: nextPairing.code, + publicKey: nextIdentity.publicKey, + }); + const proof = { + credential: first.credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce: 'superseded-pairing', + body: JSON.stringify({ protocolVersion: 1, workerId: 'vm-1' }), + }; + + await expect( + pairings.authorize({ + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(firstIdentity.privateKey, proof), + }), + ).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + }); +}); diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts new file mode 100644 index 00000000..c5da8fb4 --- /dev/null +++ b/service/src/bridge/pairing.ts @@ -0,0 +1,376 @@ +import { + createHash, + createPublicKey, + randomBytes, +} from 'crypto'; + +import type Redis from 'ioredis'; + +import { verifyBridgeRequest } from '../../../packages/code/src/identity'; + +const PREFIX = 'codeapi:bridge:v1'; +const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; +const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; +const PROOF_NONCE_TTL_SECONDS = 2 * 60; +const PROOF_CLOCK_SKEW_MS = 60_000; +const ROTATE_CREDENTIAL_SCRIPT = ` +local activeDigest = redis.call('GET', KEYS[1]) +local previous = redis.call('GET', KEYS[2]) +if not activeDigest or not previous then + return 0 +end +if activeDigest ~= ARGV[1] and redis.call('GET', KEYS[4]) ~= ARGV[5] then + return 0 +end +redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) +redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[4]) +redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) +return 1 +`; +const ISSUE_CREDENTIAL_SCRIPT = ` +local generation = redis.call('GET', KEYS[4]) or '0' +if generation ~= ARGV[5] then + return 0 +end +redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[4]) +redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[4]) +redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) +return 1 +`; +const REVOKE_WORKER_SCRIPT = ` +local activeDigest = redis.call('GET', KEYS[1]) +local activeIncarnation = redis.call('GET', KEYS[6]) +redis.call('INCR', KEYS[4]) +redis.call('DEL', KEYS[1], KEYS[2], KEYS[5], KEYS[6]) +if activeDigest then + redis.call('DEL', KEYS[3] .. activeDigest) +end +if activeIncarnation then + redis.call('SET', ARGV[1] .. activeIncarnation .. ':fenced', '1') +end +return 1 +`; + +interface StoredPairing { + workerId: string; + expiresAt: string; + generation: number; +} + +interface StoredCredential { + workerId: string; + identityId: string; + publicKey: string; + expiresAt: string; +} + +export interface BridgePairing { + workerId: string; + code: string; + expiresAt: string; +} + +export interface BridgeWorkerCredential { + workerId: string; + credential: string; + expiresAt: string; +} + +export class BridgePairingError extends Error { + constructor( + public readonly code: + | 'PAIRING_INVALID' + | 'PUBLIC_KEY_INVALID' + | 'CREDENTIAL_INVALID' + | 'PROOF_INVALID' + | 'PROOF_REPLAYED', + message: string, + ) { + super(message); + this.name = 'BridgePairingError'; + } +} + +function digest(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} + +function pairingKey(code: string): string { + return `${PREFIX}:pairing:${digest(code)}`; +} + +function credentialKey(credential: string): string { + return credentialDigestKey(digest(credential)); +} + +function credentialDigestKey(credentialDigest: string): string { + return `${PREFIX}:credential:${credentialDigest}`; +} + +function workerIdentityKey(workerId: string): string { + return `${PREFIX}:identity:${workerId}`; +} + +function workerStableIdentityKey(workerId: string): string { + return `${PREFIX}:stable-identity:${workerId}`; +} + +function workerPairingGenerationKey(workerId: string): string { + return `${PREFIX}:pairing-generation:${workerId}`; +} + +function proofNonceKey(credential: string, nonce: string): string { + return `${PREFIX}:proof:${digest(credential)}:${digest(nonce)}`; +} + +function validEd25519PublicKey(publicKey: string): boolean { + try { + return createPublicKey(publicKey).asymmetricKeyType === 'ed25519'; + } catch { + return false; + } +} + +export class RedisBridgePairingStore { + constructor( + private readonly redis: Redis, + private readonly pairingTtlSeconds = DEFAULT_PAIRING_TTL_SECONDS, + private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, + ) {} + + async issue(workerId: string): Promise { + const code = randomBytes(24).toString('base64url'); + const expiresAt = new Date( + Date.now() + this.pairingTtlSeconds * 1000, + ).toISOString(); + const generation = Number( + (await this.redis.get(workerPairingGenerationKey(workerId))) ?? '0', + ); + const pairing: StoredPairing = { workerId, expiresAt, generation }; + await this.redis.set( + pairingKey(code), + JSON.stringify(pairing), + 'EX', + this.pairingTtlSeconds, + ); + return { workerId, code, expiresAt }; + } + + async redeem(args: { + workerId: string; + code: string; + publicKey: string; + }): Promise { + const raw = await this.redis.getdel(pairingKey(args.code)); + if (raw == null) { + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code is invalid or expired', + ); + } + const pairing = JSON.parse(raw) as StoredPairing; + if (pairing.workerId !== args.workerId) { + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code does not authorize this worker', + ); + } + if (!validEd25519PublicKey(args.publicKey)) { + throw new BridgePairingError( + 'PUBLIC_KEY_INVALID', + 'Worker public key must be an Ed25519 key', + ); + } + + return await this.issueCredential( + args.workerId, + args.publicKey, + undefined, + undefined, + pairing.generation, + ); + } + + async authorize(args: { + workerId: string; + credential: string; + method: string; + path: string; + timestamp: string; + nonce: string; + body: string; + signature: string; + }): Promise<{ + workerId: string; + credentialId: string; + activeCredentialId: string; + identityId: string; + pairingGeneration: number; + }> { + const proofTime = Date.parse(args.timestamp); + if ( + !Number.isFinite(proofTime) || + Math.abs(Date.now() - proofTime) > PROOF_CLOCK_SKEW_MS + ) { + throw new BridgePairingError( + 'PROOF_INVALID', + 'Worker request proof is outside the accepted clock window', + ); + } + const credentialDigest = digest(args.credential); + const [raw, activeDigest, pairingGeneration] = await this.redis.mget( + credentialDigestKey(credentialDigest), + workerIdentityKey(args.workerId), + workerPairingGenerationKey(args.workerId), + ); + if (raw == null || activeDigest == null) { + throw new BridgePairingError( + 'CREDENTIAL_INVALID', + 'Worker credential is invalid or expired', + ); + } + const stored = JSON.parse(raw) as StoredCredential; + if (activeDigest !== credentialDigest) { + const activeRaw = await this.redis.get( + credentialDigestKey(activeDigest), + ); + const active = activeRaw == null + ? undefined + : JSON.parse(activeRaw) as StoredCredential; + if (active?.identityId !== stored.identityId) { + throw new BridgePairingError( + 'CREDENTIAL_INVALID', + 'Worker credential is invalid or expired', + ); + } + } + if (stored.workerId !== args.workerId) { + throw new BridgePairingError( + 'CREDENTIAL_INVALID', + 'Worker credential does not authorize this worker', + ); + } + if (!verifyBridgeRequest(stored.publicKey, args, args.signature)) { + throw new BridgePairingError( + 'PROOF_INVALID', + 'Worker request proof is invalid', + ); + } + const accepted = await this.redis.set( + proofNonceKey(args.credential, args.nonce), + '1', + 'EX', + PROOF_NONCE_TTL_SECONDS, + 'NX', + ); + if (accepted !== 'OK') { + throw new BridgePairingError( + 'PROOF_REPLAYED', + 'Worker request proof has already been used', + ); + } + return { + workerId: stored.workerId, + credentialId: credentialDigest, + activeCredentialId: activeDigest, + identityId: stored.identityId, + pairingGeneration: Number(pairingGeneration ?? '0'), + }; + } + + async revoke(workerId: string): Promise { + await this.redis.eval( + REVOKE_WORKER_SCRIPT, + 6, + workerIdentityKey(workerId), + workerStableIdentityKey(workerId), + `${PREFIX}:credential:`, + workerPairingGenerationKey(workerId), + `${PREFIX}:worker:${encodeURIComponent(workerId)}`, + `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, + `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, + ); + } + + async rotate( + workerId: string, + expectedCredentialId?: string, + ): Promise { + const identityKey = workerIdentityKey(workerId); + const previousDigest = expectedCredentialId ?? await this.redis.get(identityKey); + const previousRaw = + previousDigest == null + ? null + : await this.redis.get(credentialDigestKey(previousDigest)); + if (previousRaw == null || previousDigest == null) { + throw new BridgePairingError( + 'CREDENTIAL_INVALID', + 'Worker credential is invalid or expired', + ); + } + const previous = JSON.parse(previousRaw) as StoredCredential; + return await this.issueCredential( + workerId, + previous.publicKey, + previousDigest, + previous.identityId, + ); + } + + private async issueCredential( + workerId: string, + publicKey: string, + previousDigest?: string, + identityId = randomBytes(18).toString('base64url'), + pairingGeneration?: number, + ): Promise { + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date( + Date.now() + this.credentialTtlSeconds * 1000, + ).toISOString(); + const stored: StoredCredential = { workerId, identityId, publicKey, expiresAt }; + if (previousDigest !== undefined) { + const rotated = await this.redis.eval( + ROTATE_CREDENTIAL_SCRIPT, + 4, + workerIdentityKey(workerId), + credentialDigestKey(previousDigest), + credentialDigestKey(credentialDigest), + workerStableIdentityKey(workerId), + previousDigest, + credentialDigest, + JSON.stringify(stored), + String(this.credentialTtlSeconds), + identityId, + ); + if (rotated !== 1) { + throw new BridgePairingError( + 'CREDENTIAL_INVALID', + 'Worker credential is invalid or expired', + ); + } + return { workerId, credential, expiresAt }; + } + const issued = await this.redis.eval( + ISSUE_CREDENTIAL_SCRIPT, + 4, + credentialDigestKey(credentialDigest), + workerIdentityKey(workerId), + workerStableIdentityKey(workerId), + workerPairingGenerationKey(workerId), + JSON.stringify(stored), + credentialDigest, + identityId, + String(this.credentialTtlSeconds), + String(pairingGeneration ?? 0), + ); + if (issued !== 1) { + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code was revoked before redemption completed', + ); + } + return { workerId, credential, expiresAt }; + } +} diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts new file mode 100644 index 00000000..a4563399 --- /dev/null +++ b/service/src/bridge/router.test.ts @@ -0,0 +1,249 @@ +import { createServer, type Server } from 'http'; + +import { afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; + +import { + createBridgeIdentity, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgePairingStore } from './pairing'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +let server: Server | undefined; + +afterEach(async () => { + server?.close(); + server = undefined; + await redis.flushall(); +}); + +describe('paired bridge HTTP API', () => { + test('pairs a worker and accepts its proof-of-possession registration', async () => { + const app = express(); + const store = new RedisBridgeStore(redis); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'paired', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + const baseUrl = `http://127.0.0.1:${address.port}/v1/bridge`; + const pairingResponse = await fetch(`${baseUrl}/pairings`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ workerId: 'vm-1' }), + }); + const pairing = (await pairingResponse.json()) as { code: string }; + expect(pairingResponse.status).toBe(200); + + const identity = createBridgeIdentity(); + const redemptionResponse = await fetch(`${baseUrl}/pairings/redeem`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }), + }); + const issued = (await redemptionResponse.json()) as { + credential: string; + }; + expect(redemptionResponse.status).toBe(200); + + const path = '/v1/bridge/workers/register'; + const body = JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path, + timestamp: new Date().toISOString(), + nonce: 'http-registration-nonce', + body, + }; + const headers = { + Authorization: `Bridge ${issued.credential}`, + 'Content-Type': 'application/json', + 'X-LibreChat-Code-Timestamp': proof.timestamp, + 'X-LibreChat-Code-Nonce': proof.nonce, + 'X-LibreChat-Code-Signature': signBridgeRequest( + identity.privateKey, + proof, + ), + }; + const registrationUrl = `http://127.0.0.1:${address.port}${path}`; + const registrationResponse = await fetch(registrationUrl, { + method: 'POST', + headers, + body, + }); + + expect(registrationResponse.status).toBe(200); + await expect(registrationResponse.json()).resolves.toMatchObject({ + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + }); + + const crossDeploymentRevoke = await fetch( + `${baseUrl}/workers/another-deployments-worker/revoke`, + { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: '{}', + }, + ); + expect(crossDeploymentRevoke.status).toBe(400); + + const replayResponse = await fetch(registrationUrl, { + method: 'POST', + headers, + body, + }); + expect(replayResponse.status).toBe(401); + await expect(replayResponse.json()).resolves.toMatchObject({ + code: 'PROOF_REPLAYED', + }); + + const revokeResponse = await fetch(`${baseUrl}/workers/vm-1/revoke`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: '{}', + }); + expect(revokeResponse.status).toBe(200); + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_FENCED' }); + }); + + test('forwards pairing store failures to Express error middleware', async () => { + const app = express(); + const pairings = new RedisBridgePairingStore(redis); + pairings.issue = async () => { + throw new Error('pairing store unavailable'); + }; + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings, + authMode: 'paired', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + app.use( + ( + error: Error, + _req: express.Request, + res: express.Response, + _next: express.NextFunction, + ) => { + res.status(503).json({ error: error.message }); + }, + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + + const response = await fetch( + `http://127.0.0.1:${address.port}/v1/bridge/pairings`, + { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ workerId: 'vm-1' }), + }, + ); + + expect(response.status).toBe(503); + await expect(response.json()).resolves.toEqual({ + error: 'pairing store unavailable', + }); + }); + + test('does not treat a missing configured worker ID as a wildcard', async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings: new RedisBridgePairingStore(redis), + authMode: 'paired', + adminToken: 'strong-administrator-bootstrap-token', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + + const response = await fetch( + `http://127.0.0.1:${address.port}/v1/bridge/pairings`, + { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ workerId: 'vm-1' }), + }, + ); + + expect(response.status).toBe(400); + }); +}); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 35960401..8ba474ad 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -10,14 +10,21 @@ import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, } from '../../../packages/code/src/protocol'; -import { connection } from '../queue'; -import { env } from '../config'; +import { BridgePairingError, RedisBridgePairingStore } from './pairing'; import { BridgeStoreError, RedisBridgeStore } from './store'; const INCARNATION_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; const MAX_LEASE_WAIT_MS = 30_000; -export const bridgeStore = new RedisBridgeStore(connection); +export type BridgeAuthMode = 'static' | 'paired'; + +export interface BridgeRouterOptions { + store: RedisBridgeStore; + pairings: RedisBridgePairingStore; + authMode: BridgeAuthMode; + adminToken: string; + configuredWorkerId?: string; +} function sameToken(left: string, right: string): boolean { const leftBuffer = Buffer.from(left); @@ -28,23 +35,6 @@ function sameToken(left: string, right: string): boolean { ); } -function bridgeAuth(req: Request, res: Response, next: NextFunction): void { - if (!env.BRIDGE_TOKEN) { - res.status(503).json({ error: 'Code bridge is not configured' }); - return; - } - const token = - req - .header('Authorization') - ?.match(/^Bearer\s+(.+)$/i)?.[1] - ?.trim() ?? ''; - if (!token || !sameToken(token, env.BRIDGE_TOKEN)) { - res.status(401).json({ error: 'Invalid code bridge worker token' }); - return; - } - next(); -} - function validWorkerId(value: string): boolean { return isValidBridgeWorkerId(value); } @@ -98,11 +88,180 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement { ); } -const router = Router(); -router.use(bridgeAuth); +export function createBridgeRouter(options: BridgeRouterOptions): Router { + const router = Router(); + + const configuredWorker = (workerId: string): boolean => + options.configuredWorkerId != null && + options.configuredWorkerId !== '' && + workerId === options.configuredWorkerId; + + const bearerToken = (req: Request): string => + req + .header('Authorization') + ?.match(/^Bearer\s+(.+)$/i)?.[1] + ?.trim() ?? ''; + + const adminAuth = ( + req: Request, + res: Response, + next: NextFunction, + ): void => { + if (!options.adminToken) { + res.status(503).json({ error: 'Code bridge is not configured' }); + return; + } + const token = bearerToken(req); + if (!token || !sameToken(token, options.adminToken)) { + res.status(401).json({ error: 'Invalid code bridge administrator token' }); + return; + } + next(); + }; + + const staticWorkerAuth = ( + req: Request, + res: Response, + next: NextFunction, + ): void => { + const token = bearerToken(req); + if (!token || !sameToken(token, options.adminToken)) { + res.status(401).json({ error: 'Invalid code bridge worker token' }); + return; + } + next(); + }; + + const pairedWorkerAuth = ( + req: Request, + res: Response, + next: NextFunction, + ): void => { + const workerId = + req.params.workerId || + (isRecord(req.body) && typeof req.body.workerId === 'string' + ? req.body.workerId + : ''); + const credential = + req + .header('Authorization') + ?.match(/^Bridge\s+(.+)$/i)?.[1] + ?.trim() ?? ''; + const timestamp = req.header('X-LibreChat-Code-Timestamp') ?? ''; + const nonce = req.header('X-LibreChat-Code-Nonce') ?? ''; + const signature = req.header('X-LibreChat-Code-Signature') ?? ''; + if ( + !validWorkerId(workerId) || + !credential || + !timestamp || + !nonce || + !signature + ) { + res.status(401).json({ error: 'Invalid paired worker authorization' }); + return; + } + void options.pairings + .authorize({ + workerId, + credential, + method: req.method, + path: req.originalUrl.split('?')[0], + timestamp, + nonce, + body: JSON.stringify(req.body ?? {}), + signature, + }) + .then((authorization) => { + res.locals.bridgeWorkerAuthorization = authorization; + next(); + }) + .catch((error: unknown) => { + if (error instanceof BridgePairingError) { + res.status(401).json({ error: error.message, code: error.code }); + return; + } + next(error); + }); + }; + + const workerAuth = + options.authMode === 'paired' ? pairedWorkerAuth : staticWorkerAuth; + + router.post('/pairings', adminAuth, asyncRoute(async (req, res) => { + if (options.authMode !== 'paired') { + res.status(409).json({ error: 'Paired worker authentication is disabled' }); + return; + } + const workerId = isRecord(req.body) ? req.body.workerId : undefined; + if ( + typeof workerId !== 'string' || + !validWorkerId(workerId) || + !configuredWorker(workerId) + ) { + res.status(400).json({ error: 'Invalid bridge worker ID' }); + return; + } + const pairing = await options.pairings.issue(workerId); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...pairing }); + })); + + router.post('/pairings/redeem', asyncRoute(async (req, res) => { + if (options.authMode !== 'paired') { + res.status(409).json({ error: 'Paired worker authentication is disabled' }); + return; + } + const redemption = req.body as unknown; + if ( + !isRecord(redemption) || + redemption.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + typeof redemption.workerId !== 'string' || + !validWorkerId(redemption.workerId) || + !configuredWorker(redemption.workerId) || + typeof redemption.code !== 'string' || + redemption.code.length < 16 || + typeof redemption.publicKey !== 'string' || + redemption.publicKey.length > 4096 + ) { + res.status(400).json({ error: 'Invalid bridge pairing redemption' }); + return; + } + try { + const credential = await options.pairings.redeem({ + workerId: redemption.workerId, + code: redemption.code, + publicKey: redemption.publicKey, + }); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...credential }); + } catch (error) { + if (error instanceof BridgePairingError) { + const status = error.code === 'PUBLIC_KEY_INVALID' ? 400 : 401; + res.status(status).json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + + router.post( + '/workers/:workerId/revoke', + adminAuth, + asyncRoute(async (req, res) => { + if ( + !validWorkerId(req.params.workerId) || + !configuredWorker(req.params.workerId) + ) { + res.status(400).json({ error: 'Invalid bridge worker ID' }); + return; + } + await options.pairings.revoke(req.params.workerId); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, revoked: true }); + }), + ); + router.post( '/workers/register', + workerAuth, asyncRoute(async (req, res) => { const registration = req.body as unknown; if ( @@ -117,8 +276,7 @@ router.post( return; } if ( - env.BRIDGE_WORKER_ID && - registration.workerId !== env.BRIDGE_WORKER_ID + !configuredWorker(registration.workerId) ) { res.status(403).json({ error: 'Worker is not authorized for this Code API deployment', @@ -126,8 +284,16 @@ router.post( return; } try { - await bridgeStore.register( + await options.store.register( registration as unknown as BridgeWorkerRegistration, + options.authMode === 'paired' + ? ( + res.locals.bridgeWorkerAuthorization as { + identityId: string; + pairingGeneration: number; + } + ) + : undefined, ); } catch (error) { if (error instanceof BridgeStoreError) { @@ -148,6 +314,7 @@ router.post( router.post( '/workers/:workerId/workspaces/reset', + workerAuth, asyncRoute(async (req, res) => { const workerId = req.params.workerId; const body = isRecord(req.body) ? req.body : {}; @@ -165,7 +332,7 @@ router.post( }); return; } - if (env.BRIDGE_WORKER_ID && workerId !== env.BRIDGE_WORKER_ID) { + if (!configuredWorker(workerId)) { res.status(403).json({ error: 'Worker is not authorized for this Code API deployment', }); @@ -177,7 +344,7 @@ router.post( req.once('aborted', abortReset); res.once('close', abortReset); try { - await bridgeStore.resetWorkspace( + await options.store.resetWorkspace( workerId, body.incarnationId, body.runtimeSessionId, @@ -202,6 +369,7 @@ router.post( router.post( '/workers/:workerId/lease', + workerAuth, asyncRoute(async (req, res) => { const requestStartedAtMs = Date.now(); const workerId = req.params.workerId; @@ -217,7 +385,7 @@ router.post( res.status(400).json({ error: 'Invalid bridge lease request' }); return; } - if (env.BRIDGE_WORKER_ID && workerId !== env.BRIDGE_WORKER_ID) { + if (!configuredWorker(workerId)) { res.status(403).json({ error: 'Worker is not authorized for this Code API deployment', }); @@ -230,14 +398,14 @@ router.post( res.once('close', abortLease); let assignment: CodeBridgeAssignment | undefined; try { - assignment = await bridgeStore.lease( + assignment = await options.store.lease( workerId, body.incarnationId, Math.min(requestedWait, MAX_LEASE_WAIT_MS), leaseController.signal, ); if (leaseController.signal.aborted) { - if (assignment != null) await bridgeStore.returnLease(assignment); + if (assignment != null) await options.store.returnLease(assignment); return; } res.json({ @@ -261,6 +429,7 @@ router.post( router.post( '/workers/:workerId/assignments/:assignmentId/ack', + workerAuth, asyncRoute(async (req, res) => { const body = isRecord(req.body) ? req.body : {}; if ( @@ -281,7 +450,7 @@ router.post( req.once('aborted', abortAcknowledgement); res.once('close', abortAcknowledgement); try { - await bridgeStore.acknowledgeLease( + await options.store.acknowledgeLease( req.params.workerId, body.incarnationId, req.params.assignmentId, @@ -308,6 +477,7 @@ router.post( router.post( '/workers/:workerId/assignments/:assignmentId/settle', + workerAuth, asyncRoute(async (req, res) => { const settlement = req.body as unknown; if (!isSettlement(settlement)) { @@ -320,7 +490,7 @@ router.post( req.once('aborted', abortSettlement); res.once('close', abortSettlement); try { - await bridgeStore.settle( + await options.store.settle( req.params.workerId, req.params.assignmentId, settlement, @@ -348,6 +518,7 @@ router.post( router.post( '/workers/:workerId/assignments/:assignmentId/cancellation', + workerAuth, asyncRoute(async (req, res) => { const body = isRecord(req.body) ? req.body : {}; if ( @@ -362,7 +533,7 @@ router.post( req.once('aborted', abortCancellation); res.once('close', abortCancellation); try { - const cancelled = await bridgeStore.cancelled( + const cancelled = await options.store.cancelled( req.params.workerId, body.incarnationId, req.params.assignmentId, @@ -378,4 +549,30 @@ router.post( }), ); -export default router; + router.post( + '/workers/:workerId/credentials/refresh', + workerAuth, + asyncRoute(async (req, res) => { + try { + const credential = await options.pairings.rotate( + req.params.workerId, + ( + res.locals.bridgeWorkerAuthorization as + | { credentialId: string } + | undefined + )?.credentialId, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...credential }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(401).json({ error: error.message, code: error.code }); + return; + } + throw error; + } + }), + ); + + + return router; +} diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 9b2dce13..90eef488 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -220,8 +220,16 @@ export class RedisBridgeStore { ); } - async register(registration: BridgeWorkerRegistration): Promise { + async register( + registration: BridgeWorkerRegistration, + authorization?: { identityId: string; pairingGeneration: number }, + ): Promise { const script = [ + 'if ARGV[5] ~= "" then', + ' local pairingGeneration = redis.call(\'GET\', KEYS[7]) or "0"', + ' if pairingGeneration ~= ARGV[5] then return -4 end', + ' if redis.call(\'GET\', KEYS[8]) ~= ARGV[6] then return -4 end', + 'end', 'if redis.call(\'EXISTS\', KEYS[3]) == 1 then return -2 end', 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return -1 end', 'local current = redis.call(\'GET\', KEYS[4])', @@ -243,17 +251,21 @@ export class RedisBridgeStore { await boundedCommand( this.redis.eval( script, - 6, + 8, workerKey(registration.workerId), incarnationFenceKey(registration.workerId, registration.incarnationId), quarantineKey(registration.workerId, registration.incarnationId), workerIncarnationKey(registration.workerId), lockKey(registration.workerId), lockIncarnationKey(registration.workerId), + `${PREFIX}:pairing-generation:${registration.workerId}`, + `${PREFIX}:stable-identity:${registration.workerId}`, registration.incarnationId, JSON.stringify(registration), String(this.workerTtlSeconds), `${PREFIX}:worker:${encodeURIComponent(registration.workerId)}:incarnation:`, + authorization == null ? '' : String(authorization.pairingGeneration), + authorization?.identityId ?? '', ), this.redisCommandTimeoutMs, 'Bridge worker registration', @@ -277,6 +289,12 @@ export class RedisBridgeStore { 'Bridge worker cannot be replaced during an active assignment', ); } + if (result === -4) { + throw new BridgeStoreError( + 'WORKER_FENCED', + 'Bridge worker authorization was revoked before registration completed', + ); + } } async dispatch(args: { diff --git a/service/src/config.test.ts b/service/src/config.test.ts index 22878ea7..87658f98 100644 --- a/service/src/config.test.ts +++ b/service/src/config.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from 'bun:test'; import { parsePlanLimits, + resolveBridgeAuthMode, resolveRuntimeSessionMode, resolveSandboxBackend, } from './config'; @@ -9,6 +10,7 @@ describe('sandbox execution configuration', () => { test('defaults only unset backend and session mode values', () => { expect(resolveSandboxBackend(undefined)).toBe('http'); expect(resolveRuntimeSessionMode(undefined)).toBe('stateless'); + expect(resolveBridgeAuthMode(undefined)).toBe('static'); }); test('accepts every supported backend and session mode', () => { @@ -18,6 +20,8 @@ describe('sandbox execution configuration', () => { expect(resolveRuntimeSessionMode('stateless')).toBe('stateless'); expect(resolveRuntimeSessionMode('affinity')).toBe('affinity'); expect(resolveRuntimeSessionMode('strict')).toBe('strict'); + expect(resolveBridgeAuthMode('static')).toBe('static'); + expect(resolveBridgeAuthMode('paired')).toBe('paired'); }); test('rejects unknown values instead of silently changing execution semantics', () => { @@ -31,6 +35,9 @@ describe('sandbox execution configuration', () => { ); expect(() => resolveRuntimeSessionMode('')).toThrow('CODEAPI_RUNTIME_SESSION_MODE'); expect(() => resolveRuntimeSessionMode(' ')).toThrow('CODEAPI_RUNTIME_SESSION_MODE'); + expect(() => resolveBridgeAuthMode('token')).toThrow( + 'CODEAPI_BRIDGE_AUTH_MODE must be one of: static, paired', + ); }); }); diff --git a/service/src/config.ts b/service/src/config.ts index dbc6dfe8..605c701b 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -263,8 +263,20 @@ export function resolveRuntimeSessionMode( ); } +export function resolveBridgeAuthMode( + raw: string | undefined, +): 'static' | 'paired' { + return configuredChoice( + raw, + 'CODEAPI_BRIDGE_AUTH_MODE', + 'static', + ['static', 'paired'], + ); +} + const sandboxBackend = resolveSandboxBackend(process.env.CODEAPI_SANDBOX_BACKEND); const runtimeSessionMode = resolveRuntimeSessionMode(process.env.CODEAPI_RUNTIME_SESSION_MODE); +const bridgeAuthMode = resolveBridgeAuthMode(process.env.CODEAPI_BRIDGE_AUTH_MODE); export const env = { PORT: process.env.SERVICE_PORT ?? 3112, @@ -355,6 +367,8 @@ export const env = { SANDBOX_BACKEND: sandboxBackend, /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', + /** Static compatibility auth or short-lived proof-of-possession credentials. */ + BRIDGE_AUTH_MODE: bridgeAuthMode, /** Enrollment and lease credential shared only with the configured worker. */ BRIDGE_TOKEN: process.env.CODEAPI_BRIDGE_TOKEN ?? '', /** diff --git a/service/src/lifecycle.ts b/service/src/lifecycle.ts index c5687610..d7ef75da 100644 --- a/service/src/lifecycle.ts +++ b/service/src/lifecycle.ts @@ -155,6 +155,7 @@ async function gracefulStartup(): Promise { validateWorkerHardenedConfig(); validateExecutionProfilePolicy(); validateSandboxBackendPolicy(); + validateApiBridgePolicy(); await validateLifecycleAuthConfig(); configureProfileMetrics(); diff --git a/service/src/local-api.ts b/service/src/local-api.ts index aafa9f03..66896b63 100644 --- a/service/src/local-api.ts +++ b/service/src/local-api.ts @@ -10,7 +10,7 @@ import express, { json, Router } from 'express'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; -import bridgeRouter from './bridge/router'; +import bridgeRouter from './bridge'; import { requestErrorLogger, requestNotFoundLogger } from './middleware/request-error-logger'; import { executionProfileMiddleware } from './middleware/execution-profile'; import { localAuth } from './auth/local'; diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index a18788a8..a30ab748 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -7,7 +7,7 @@ import type { import type { RedisBridgeStore } from '../bridge/store'; import { env } from '../config'; -import { bridgeStore } from '../bridge/router'; +import { bridgeStore } from '../bridge'; import { BridgeStoreError } from '../bridge/store'; import { SandboxBackendError } from './types'; diff --git a/service/src/secure-startup.test.ts b/service/src/secure-startup.test.ts index a365f355..85f0e854 100644 --- a/service/src/secure-startup.test.ts +++ b/service/src/secure-startup.test.ts @@ -16,6 +16,7 @@ const saved = { executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: env.SANDBOX_BACKEND, bridgeWorkerId: env.BRIDGE_WORKER_ID, + bridgeAuthMode: env.BRIDGE_AUTH_MODE, bridgeToken: env.BRIDGE_TOKEN, ptcMode: env.PTC_MODE, runtimeSessionMode: env.RUNTIME_SESSION_MODE, @@ -56,6 +57,7 @@ function restore(): void { env.EXECUTION_PROFILE_SOURCE = saved.executionProfileSource; env.SANDBOX_BACKEND = saved.sandboxBackend; env.BRIDGE_WORKER_ID = saved.bridgeWorkerId; + env.BRIDGE_AUTH_MODE = saved.bridgeAuthMode; env.BRIDGE_TOKEN = saved.bridgeToken; env.PTC_MODE = saved.ptcMode; env.RUNTIME_SESSION_MODE = saved.runtimeSessionMode; @@ -311,7 +313,7 @@ describe('sandbox backend policy', () => { expect(() => validateSandboxBackendPolicy()).not.toThrow(); }); - test('remote bridge requires replay PTC and a strong token in hardened mode', () => { + test('hardened remote bridge requires replay PTC, paired auth, and a strong administrator token', () => { env.SANDBOX_BACKEND = 'remote-bridge'; env.RUNTIME_SESSION_MODE = 'affinity'; env.BRIDGE_WORKER_ID = 'engineering-vm'; @@ -326,15 +328,38 @@ describe('sandbox backend policy', () => { expect(() => validateSandboxBackendPolicy()).toThrow('at least 32 bytes'); env.BRIDGE_TOKEN = 'strong-remote-bridge-token-32-bytes'; + expect(() => validateSandboxBackendPolicy()).toThrow( + 'CODEAPI_BRIDGE_AUTH_MODE=paired', + ); + + env.BRIDGE_AUTH_MODE = 'paired'; expect(() => validateSandboxBackendPolicy()).not.toThrow(); }); + test('API-only hardened bridge validation rejects static worker auth', () => { + env.SANDBOX_BACKEND = 'http'; + env.HARDENED_SANDBOX_MODE = true; + env.BRIDGE_AUTH_MODE = 'static'; + env.BRIDGE_WORKER_ID = 'engineering-vm'; + env.BRIDGE_TOKEN = 'strong-remote-bridge-token-32-bytes'; + expect(() => validateApiBridgePolicy()).toThrow( + 'CODEAPI_BRIDGE_AUTH_MODE=paired', + ); + + env.BRIDGE_AUTH_MODE = 'paired'; + expect(() => validateApiBridgePolicy()).not.toThrow(); + + env.BRIDGE_TOKEN = 'guessable'; + expect(() => validateApiBridgePolicy()).toThrow('at least 32 bytes'); + }); + test('API bridge policy requires a strong token in hardened mode', () => { env.SANDBOX_BACKEND = 'remote-bridge'; env.BRIDGE_WORKER_ID = 'engineering-vm'; env.BRIDGE_TOKEN = 'short-token'; env.PTC_MODE = 'replay'; env.HARDENED_SANDBOX_MODE = true; + env.BRIDGE_AUTH_MODE = 'paired'; expect(() => validateApiBridgePolicy()).toThrow('at least 32 bytes'); @@ -364,6 +389,29 @@ describe('sandbox backend policy', () => { ); }); + test('paired API routes require a configured worker on every backend', () => { + env.SANDBOX_BACKEND = 'http'; + env.BRIDGE_AUTH_MODE = 'paired'; + env.BRIDGE_TOKEN = 'development-bridge-token'; + env.BRIDGE_WORKER_ID = ''; + + expect(() => validateApiBridgePolicy()).toThrow( + 'CODEAPI_BRIDGE_WORKER_ID', + ); + }); + + test('hardened API routes reject padded bridge tokens on HTTP backends', () => { + env.SANDBOX_BACKEND = 'http'; + env.HARDENED_SANDBOX_MODE = true; + env.BRIDGE_AUTH_MODE = 'paired'; + env.BRIDGE_WORKER_ID = 'engineering-vm'; + env.BRIDGE_TOKEN = ' strong-remote-bridge-token-32-bytes '; + + expect(() => validateApiBridgePolicy()).toThrow( + 'must not contain surrounding whitespace', + ); + }); + test('remote bridge requires a positive finite job timeout', () => { env.SANDBOX_BACKEND = 'remote-bridge'; env.BRIDGE_WORKER_ID = 'engineering-vm'; diff --git a/service/src/secure-startup.ts b/service/src/secure-startup.ts index 0bcf02c1..f6f13390 100644 --- a/service/src/secure-startup.ts +++ b/service/src/secure-startup.ts @@ -56,27 +56,36 @@ export function validateApiHardenedConfig(): void { /** Validate bridge credentials in every process that exposes bridge routes. */ export function validateApiBridgePolicy(): void { - if (env.SANDBOX_BACKEND !== 'remote-bridge') return; - requireSafeWholeNumber('JOB_TIMEOUT', env.JOB_TIMEOUT, 1); - requireValue('CODEAPI_BRIDGE_WORKER_ID', env.BRIDGE_WORKER_ID); - if (!isValidBridgeWorkerId(env.BRIDGE_WORKER_ID ?? '')) { - throw new SecureStartupConfigError( - 'CODEAPI_BRIDGE_WORKER_ID must match the bridge worker ID format', - ); - } if (env.BRIDGE_TOKEN !== env.BRIDGE_TOKEN.trim()) { throw new SecureStartupConfigError( 'CODEAPI_BRIDGE_TOKEN must not contain surrounding whitespace', ); } - if (env.HARDENED_SANDBOX_MODE) { - requireStrongSecret('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); - } else { + const bridgeEnabled = + env.SANDBOX_BACKEND === 'remote-bridge' || + env.BRIDGE_AUTH_MODE === 'paired'; + if (bridgeEnabled) { + requireValue('CODEAPI_BRIDGE_WORKER_ID', env.BRIDGE_WORKER_ID); + if (!isValidBridgeWorkerId(env.BRIDGE_WORKER_ID ?? '')) { + throw new SecureStartupConfigError( + 'CODEAPI_BRIDGE_WORKER_ID must match the bridge worker ID format', + ); + } requireValue('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); } - if (env.PTC_MODE === 'blocking') { + if (env.SANDBOX_BACKEND === 'remote-bridge') { + requireSafeWholeNumber('JOB_TIMEOUT', env.JOB_TIMEOUT, 1); + if (env.PTC_MODE === 'blocking') { + throw new SecureStartupConfigError( + 'PTC replay is the only supported PTC mode for the remote-bridge backend (unset PTC_MODE=blocking)', + ); + } + } + if (!env.HARDENED_SANDBOX_MODE) return; + requireStrongSecret('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); + if (env.BRIDGE_AUTH_MODE !== 'paired') { throw new SecureStartupConfigError( - 'PTC replay is the only supported PTC mode for the remote-bridge backend (unset PTC_MODE=blocking)', + 'Hardened API deployments require CODEAPI_BRIDGE_AUTH_MODE=paired because bridge routes are always exposed', ); } } diff --git a/service/src/service-api.ts b/service/src/service-api.ts index 6a0a8331..79db08d5 100644 --- a/service/src/service-api.ts +++ b/service/src/service-api.ts @@ -5,7 +5,7 @@ import { requestErrorLogger, requestNotFoundLogger } from './middleware/request- import { executionProfileMiddleware } from './middleware/execution-profile'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; -import bridgeRouter from './bridge/router'; +import bridgeRouter from './bridge'; import { connection } from './queue'; import { env } from './config'; import logger from './logger'; diff --git a/service/tsconfig.json b/service/tsconfig.json index dcddf82d..13f30872 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -16,7 +16,8 @@ "include": [ "src/**/*.ts", "../shared/telemetry-core.ts", - "../packages/code/src/protocol.ts" + "../packages/code/src/protocol.ts", + "../packages/code/src/identity.ts" ], "exclude": [ "node_modules", From 18dadd58345d71b9f056e9d2702f1fad86e81129 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 1 Sep 2026 13:11:45 -0400 Subject: [PATCH 005/116] =?UTF-8?q?=F0=9F=AA=AA=20feat:=20Add=20Principal-?= =?UTF-8?q?Bound=20Bridge=20Workers=20(#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add outbound stateful code bridge * test: cover remote bridge startup policy * fix: harden remote bridge lifecycle fencing * feat: add secure code worker pairing * fix: harden paired worker lifecycle * fix: require paired auth on hardened APIs * fix: harden bridge pairing startup policy * fix: preserve bridge fencing through pairing * fix: distinguish assignment settlement conflicts * feat: add principal-bound bridge workers * fix: authenticate principal worker routing * fix: fence bridge identity and backend routing * fix: fence bridge redemption and queue routing * fix: preserve legacy routing and assignment auth * fix: preserve long-lived bridge assignments * fix: persist replay queue backend * fix: fence bridge replay and credential rotation * fix: address principal worker review findings * fix: reconcile principal workers with bridge fencing * fix: fence principal worker lifecycle transitions * fix: fence bridge settlement ownership * fix: fence bridge leases to active principals --- README.md | 4 +- docs/remote-bridge/README.md | 54 +++- packages/code/src/worker.test.ts | 82 ++++++ packages/code/src/worker.ts | 12 +- service/src/auth/librechat-jwt.test.ts | 3 + service/src/auth/librechat-jwt.ts | 3 + service/src/auth/principal.ts | 1 + service/src/bridge/index.ts | 1 + service/src/bridge/pairing.test.ts | 226 +++++++++++++++ service/src/bridge/pairing.ts | 190 +++++++++---- service/src/bridge/router.test.ts | 165 +++++++++++ service/src/bridge/router.ts | 107 ++++++- service/src/bridge/selection.test.ts | 101 +++++++ service/src/bridge/selection.ts | 72 +++++ service/src/bridge/store.test.ts | 264 ++++++++++++++++++ service/src/bridge/store.ts | 136 ++++++++- service/src/config.ts | 2 + service/src/execution-profile.test.ts | 65 +++++ service/src/execution-profile.ts | 71 +++++ service/src/lifecycle.ts | 18 +- service/src/local-api.ts | 9 +- service/src/queue.ts | 71 ++++- .../src/sandbox-backend/remote-bridge.test.ts | 153 ++++++++++ service/src/sandbox-backend/remote-bridge.ts | 19 +- service/src/sandbox-backend/types.ts | 3 + service/src/secure-startup.test.ts | 33 +++ service/src/secure-startup.ts | 21 +- service/src/service/programmatic-router.ts | 103 +++++-- .../src/service/programmatic-state.test.ts | 44 ++- service/src/service/programmatic-state.ts | 28 ++ service/src/service/replay-state.ts | 13 + service/src/service/router.ts | 31 ++ service/src/types/service.ts | 6 +- service/src/utils.test.ts | 38 ++- service/src/utils.ts | 10 +- service/src/workers.ts | 13 +- 36 files changed, 2012 insertions(+), 160 deletions(-) create mode 100644 service/src/bridge/selection.test.ts create mode 100644 service/src/bridge/selection.ts create mode 100644 service/src/sandbox-backend/remote-bridge.test.ts diff --git a/README.md b/README.md index 716941d9..384e013a 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,9 @@ Set `CODEAPI_EXECUTION_PROFILE` consistently on an API deployment and its workers. The default profile keeps the existing `python-queue` and `other-queue`; the stateful profile uses `stateful-python-queue` and `stateful-other-queue`. This allows both deployments to share Redis without -cross-consuming jobs. +cross-consuming jobs. The `remote-bridge` backend additionally uses +`remote-bridge-python-queue` and `remote-bridge-other-queue`, fencing attached +worker jobs from Lambda consumers during rolling deployments. An existing Lambda MicroVM deployment upgraded from a pre-profile release may leave `CODEAPI_EXECUTION_PROFILE` unset for its first binary rollout. An diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 3a9f12fc..8d10f024 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -32,6 +32,23 @@ session hint. In hardened mode, startup requires the bridge token to be at least 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a remote execution cannot retain an open Code API process across tool callbacks. +To attach multiple principal-owned workers to one Code API deployment, enable +dynamic routing. A compatibility default worker is optional in this mode: + +```dotenv +CODEAPI_BRIDGE_DYNAMIC_WORKERS=true +CODEAPI_BRIDGE_AUTH_MODE=paired +# CODEAPI_BRIDGE_WORKER_ID=my-default-vm +``` + +Dynamic routing is accepted only with paired authentication. LibreChat signs +the selected worker into the short-lived Code API JWT as `code_worker_id`. +`X-LibreChat-Code-Worker-ID` remains the transport header, but Code API accepts +it only when it exactly matches that authenticated claim. The resolved worker +is persisted across the queue and programmatic replay boundaries, and Code API +requires both its stored tenant binding and registered worker credential before +creating a lease. + Create a single-use pairing code with the administrator secret: ```bash @@ -41,6 +58,28 @@ curl -fsS https://code.example.com/v1/bridge/pairings \ --data '{"workerId":"my-vm"}' ``` +With dynamic routing enabled, the trusted control plane must bind each pairing +to one tenant and generic principal. Code API treats the principal as lifecycle +and audit metadata; LibreChat remains responsible for resolving user, role, and +group membership before selecting the worker: + +```bash +curl -fsS https://code.example.com/v1/bridge/pairings \ + -H "Authorization: Bearer $CODEAPI_BRIDGE_TOKEN" \ + -H 'Content-Type: application/json' \ + --data '{ + "workerId":"user-vm", + "binding":{ + "tenantId":"tenant-1", + "principal":{"type":"user","id":"user-1"} + } + }' +``` + +Principal types are `deployment`, `tenant`, `user`, `role`, and `group`. +Pairing and registration bodies from the VM cannot replace the server-issued +binding, and credential rotation preserves it. + Redeem the returned code on the VM using [`@librechat/code`](../../packages/code/README.md). The CLI generates its key locally, proves possession on every request, and rotates its short-lived @@ -84,7 +123,14 @@ execution. digest, timestamp, nonce, and credential. - Accepted proof nonces cannot be replayed, credentials rotate before expiry, and an administrator can revoke the active worker identity immediately. -- Code API permits one active assignment per configured worker. +- Assignment leases bind to a stable paired identity rather than an individual + short-lived credential. Rotation preserves that identity; pairing again + replaces it and fences work queued for the previous owner. +- Remote bridge deployments use backend-specific BullMQ queues and serialize + the expected backend on every new job, preventing Lambda or HTTP consumers + from accepting attached-worker executions. +- Code API permits one active assignment per worker. +- Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are rejected. @@ -126,6 +172,6 @@ administrator token immediately. Pairing secures worker transport identity; it cannot attest that a compromised VM truthfully reports or enforces its sandbox capabilities. -The next control-plane layer can add owner-scoped environment records and a -multi-worker directory without changing the execution protocol or moving code -tools into the Agents SDK. +LibreChat's owner-scoped environment registry can issue these principal-bound +pairings without changing the worker execution protocol or moving code tools +into the Agents SDK. diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index fc88c6e3..f4ec7d0b 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -2502,6 +2502,88 @@ test('worker shutdown interrupts reconnect backoff', async () => { await run; }); +test('sandbox completion does not cancel an in-flight credential rotation', async () => { + const key = createBridgeIdentity(); + const identity = { + privateKey: key.privateKey, + credential: 'credential-before-in-flight-rotation', + expiresAt: new Date(Date.now() + 40).toISOString(), + }; + let refreshStarted!: () => void; + const refreshStartedPromise = new Promise((resolve) => { + refreshStarted = resolve; + }); + let refreshCount = 0; + let settleAuthorization = ''; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/credentials/refresh')) { + refreshCount += 1; + if (refreshCount > 1) { + return Response.json({ error: 'stale credential' }, { status: 401 }); + } + refreshStarted(); + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 30); + init?.signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + reject(new DOMException('Aborted', 'AbortError')); + }, + { once: true }, + ); + }); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-in-flight-rotation', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + }); + } + if (url.endsWith('/execute')) { + await refreshStartedPromise; + return Response.json({ session_id: 'run-rotation-race', files: [] }); + } + settleAuthorization = ( + init?.headers as Record + ).Authorization; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl, + credentialRefreshWindowMs: 30, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-rotation-race', + workerId: 'vm-1', + incarnationId, + generation: 5, + leaseToken: 'assignment-rotation-race-lease-token', + expiresAt: new Date(Date.now() + 600_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(refreshCount, 1); + assert.equal(identity.credential, 'credential-after-in-flight-rotation'); + assert.equal( + settleAuthorization, + 'Bridge credential-after-in-flight-rotation', + ); +}); + test('reconnect delay uses bounded exponential jitter', () => { assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 0), 500); assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 1), 1_000); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 7c313fee..ebbd486e 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -387,9 +387,7 @@ export class BridgeWorker { ): Promise { const identity = this.options.identity; if (identity == null) return; - if ( - Date.parse(identity.expiresAt) > validThroughMs - ) { + if (Date.parse(identity.expiresAt) > validThroughMs) { return; } const credential = await this.timedRequest( @@ -432,6 +430,7 @@ export class BridgeWorker { assignment: BridgeAssignment, stopSignal: AbortSignal, serverClockOffsetMs: number, + requestSignal?: AbortSignal, ): Promise { const identity = this.options.identity; if (identity == null) return; @@ -450,7 +449,7 @@ export class BridgeWorker { if (stopSignal.aborted || Date.now() >= assignmentDeadlineMs) return; try { await this.refreshCredential( - stopSignal, + requestSignal, Date.now() + serverClockOffsetMs + refreshWindowMs, ); } catch (error) { @@ -564,6 +563,7 @@ export class BridgeWorker { assignment, credentialController.signal, serverClockOffsetMs, + signal, ).catch((error) => { credentialMaintenanceError = error; executionController.abort(); @@ -652,6 +652,8 @@ export class BridgeWorker { clearTimeout(deadlineTimer); cancellationController.abort(); await cancellationWatcher; + credentialController.abort(); + await credentialMaintenance; try { if (ambiguousSandboxError != null) { throw new BridgeWorkspaceQuarantinedError( @@ -696,8 +698,6 @@ export class BridgeWorker { } finally { heartbeatController.abort(); await heartbeat; - credentialController.abort(); - await credentialMaintenance; signal?.removeEventListener('abort', abortExecution); } } diff --git a/service/src/auth/librechat-jwt.test.ts b/service/src/auth/librechat-jwt.test.ts index 2030b2e7..d20124f5 100644 --- a/service/src/auth/librechat-jwt.test.ts +++ b/service/src/auth/librechat-jwt.test.ts @@ -48,6 +48,7 @@ type JwtClaims = { chc_user_id?: string; auth_context_hash?: string; plan_id?: string; + code_worker_id?: string; }; const originalEnv = new Map(); @@ -75,6 +76,7 @@ function baseClaims(overrides: Partial = {}): JwtClaims { external_user_id: 'chc_123', auth_context_hash: 'hash_123', plan_id: 'prod_plan_123', + code_worker_id: 'code-user_123', ...overrides, }; } @@ -166,6 +168,7 @@ describe('LibreChat JWT auth provider', () => { principalSource: 'openid_reuse', authContextHash: 'hash_123', planId: 'prod_plan_123', + codeWorkerId: 'code-user_123', }); }); diff --git a/service/src/auth/librechat-jwt.ts b/service/src/auth/librechat-jwt.ts index 1e7e8079..e249ce0c 100644 --- a/service/src/auth/librechat-jwt.ts +++ b/service/src/auth/librechat-jwt.ts @@ -39,6 +39,7 @@ interface LibreChatJwtClaims { chc_user_id?: string; // leak-check:allow auth_context_hash?: string; plan_id?: string; + code_worker_id?: string; } interface PublicKeyEntry { @@ -394,6 +395,7 @@ function validateClaims(claims: LibreChatJwtClaims, config: VerificationConfig): const nbf = assertNumericDate(claims.nbf, 'nbf'); const exp = assertNumericDate(claims.exp, 'exp'); const planId = optionalString(claims.plan_id, 'plan_id'); + const codeWorkerId = optionalString(claims.code_worker_id, 'code_worker_id'); const principalSource = assertPrincipalSource(claims.principal_source); const authContextHash = assertString(claims.auth_context_hash, 'auth_context_hash'); @@ -433,6 +435,7 @@ function validateClaims(claims: LibreChatJwtClaims, config: VerificationConfig): principalSource, authContextHash, planId, + codeWorkerId, }; } diff --git a/service/src/auth/principal.ts b/service/src/auth/principal.ts index 94b785f3..615a0ff2 100644 --- a/service/src/auth/principal.ts +++ b/service/src/auth/principal.ts @@ -13,6 +13,7 @@ export type CodeApiPrincipal = { authContextHash?: string; credentialId?: string; planId?: string; + codeWorkerId?: string; }; export function applyPrincipal(req: t.AuthenticatedRequest, principal: CodeApiPrincipal): void { diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index dfba2a8b..08ef1b07 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -13,4 +13,5 @@ export default createBridgeRouter({ authMode: env.BRIDGE_AUTH_MODE, adminToken: env.BRIDGE_TOKEN, configuredWorkerId: env.BRIDGE_WORKER_ID, + allowDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, }); diff --git a/service/src/bridge/pairing.test.ts b/service/src/bridge/pairing.test.ts index 7b5ab124..9f4b7117 100644 --- a/service/src/bridge/pairing.test.ts +++ b/service/src/bridge/pairing.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash } from 'crypto'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; @@ -19,6 +20,94 @@ afterEach(async () => { }); describe('RedisBridgePairingStore', () => { + test('preserves a tenant and generic principal binding across credential rotation', async () => { + const identity = createBridgeIdentity(); + const binding = { + tenantId: 'tenant-1', + principal: { type: 'group' as const, id: 'engineering' }, + }; + const pairing = await pairings.issue('vm-bound', binding); + const issued = await pairings.redeem({ + workerId: 'vm-bound', + code: pairing.code, + publicKey: identity.publicKey, + }); + const requestFor = ( + credential: string, + nonce: string, + ): Parameters[0] => { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/vm-bound/lease', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + return { + ...proof, + workerId: 'vm-bound', + signature: signBridgeRequest(identity.privateKey, proof), + }; + }; + + const originalAuthorization = await pairings.authorize( + requestFor(issued.credential, 'original-bound-worker-proof'), + ); + const rotated = await pairings.rotate('vm-bound'); + + const rotatedAuthorization = await pairings.authorize( + requestFor(rotated.credential, 'bound-worker-proof'), + ); + expect(rotatedAuthorization).toMatchObject({ workerId: 'vm-bound', binding }); + expect(typeof originalAuthorization.identityId).toBe('string'); + expect(rotatedAuthorization.identityId).toBe( + originalAuthorization.identityId, + ); + await expect( + pairings.authorize(requestFor(issued.credential, 'overlap-bound-proof')), + ).resolves.toMatchObject({ + workerId: 'vm-bound', + identityId: originalAuthorization.identityId, + }); + }); + + test('preserves a legacy unmarked identity across its first rotation', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('legacy-vm'); + const issued = await pairings.redeem({ + workerId: 'legacy-vm', + code: pairing.code, + publicKey: identity.publicKey, + }); + const issuedDigest = createHash('sha256') + .update(issued.credential) + .digest('hex'); + const credentialKey = `codeapi:bridge:v1:credential:${issuedDigest}`; + const stored = JSON.parse((await redis.get(credentialKey)) ?? '{}') as { + identityId?: string; + }; + delete stored.identityId; + await redis.set(credentialKey, JSON.stringify(stored), 'EX', 300); + + const rotated = await pairings.rotate('legacy-vm'); + const proof = { + credential: rotated.credential, + method: 'POST', + path: '/v1/bridge/workers/legacy-vm/lease', + timestamp: new Date().toISOString(), + nonce: 'legacy-rotation-proof', + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + const authorization = await pairings.authorize({ + ...proof, + workerId: 'legacy-vm', + signature: signBridgeRequest(identity.privateKey, proof), + }); + + expect(authorization.identityId).toBeUndefined(); + }); + test('redeems a pairing code exactly once for the intended worker identity', async () => { const identity = createBridgeIdentity(); const pairing = await pairings.issue('vm-1'); @@ -40,6 +129,108 @@ describe('RedisBridgePairingStore', () => { ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); }); + test('only the newest pairing code can rebind a worker identity', async () => { + const identity = createBridgeIdentity(); + const older = await pairings.issue('vm-1', { + tenantId: 'tenant-a', + principal: { type: 'user', id: 'user-a' }, + }); + const newer = await pairings.issue('vm-1', { + tenantId: 'tenant-b', + principal: { type: 'user', id: 'user-b' }, + }); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: older.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: newer.code, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('does not let a paused redemption overwrite a newer pairing identity', async () => { + const firstIdentity = createBridgeIdentity(); + const secondIdentity = createBridgeIdentity(); + const firstPairing = await pairings.issue('vm-race', { + tenantId: 'tenant-a', + principal: { type: 'user', id: 'user-a' }, + }); + const originalEval = redis.eval.bind(redis); + let releaseFirst!: () => void; + let firstRedeemed!: () => void; + const firstRedeemedPromise = new Promise((resolve) => { + firstRedeemed = resolve; + }); + const releaseFirstPromise = new Promise((resolve) => { + releaseFirst = resolve; + }); + let paused = false; + redis.eval = (async (script: string, ...args: unknown[]) => { + const result = await (originalEval as (...evalArgs: unknown[]) => Promise)( + script, + ...args, + ); + if (!paused && script.includes('return pairing')) { + paused = true; + firstRedeemed(); + await releaseFirstPromise; + } + return result; + }) as typeof redis.eval; + + try { + const staleRedemption = pairings.redeem({ + workerId: 'vm-race', + code: firstPairing.code, + publicKey: firstIdentity.publicKey, + }); + await firstRedeemedPromise; + const secondPairing = await pairings.issue('vm-race', { + tenantId: 'tenant-b', + principal: { type: 'user', id: 'user-b' }, + }); + const current = await pairings.redeem({ + workerId: 'vm-race', + code: secondPairing.code, + publicKey: secondIdentity.publicKey, + }); + releaseFirst(); + + await expect(staleRedemption).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + const proof = { + credential: current.credential, + method: 'POST', + path: '/v1/bridge/workers/vm-race/lease', + timestamp: new Date().toISOString(), + nonce: 'current-race-proof', + body: JSON.stringify({ protocolVersion: 1, waitMs: 25_000 }), + }; + await expect( + pairings.authorize({ + ...proof, + workerId: 'vm-race', + signature: signBridgeRequest(secondIdentity.privateKey, proof), + }), + ).resolves.toMatchObject({ + binding: { + tenantId: 'tenant-b', + principal: { type: 'user', id: 'user-b' }, + }, + }); + } finally { + redis.eval = originalEval as typeof redis.eval; + releaseFirst(); + } + }); + test('authorizes a credential only with proof from its worker key', async () => { const identity = createBridgeIdentity(); const pairing = await pairings.issue('vm-1'); @@ -314,6 +505,41 @@ describe('RedisBridgePairingStore', () => { ).resolves.toMatchObject({ workerId: 'vm-1' }); }); + test('rejects a stale credential refresh after the worker is paired again', async () => { + const originalIdentity = createBridgeIdentity(); + const originalPairing = await pairings.issue('vm-1'); + const original = await pairings.redeem({ + workerId: 'vm-1', + code: originalPairing.code, + publicKey: originalIdentity.publicKey, + }); + const proof = { + credential: original.credential, + method: 'POST', + path: '/v1/bridge/workers/vm-1/credentials/refresh', + timestamp: new Date().toISOString(), + nonce: 'authorized-before-repairing', + body: JSON.stringify({ protocolVersion: 1 }), + }; + const staleAuthorization = await pairings.authorize({ + ...proof, + workerId: 'vm-1', + signature: signBridgeRequest(originalIdentity.privateKey, proof), + }); + + const replacementIdentity = createBridgeIdentity(); + const replacementPairing = await pairings.issue('vm-1'); + await pairings.redeem({ + workerId: 'vm-1', + code: replacementPairing.code, + publicKey: replacementIdentity.publicKey, + }); + + await expect( + pairings.rotate('vm-1', staleAuthorization.credentialId), + ).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + }); + test('repairing a worker invalidates its previously paired credential', async () => { const firstIdentity = createBridgeIdentity(); const firstPairing = await pairings.issue('vm-1'); diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index c5da8fb4..24deba07 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -19,22 +19,18 @@ local previous = redis.call('GET', KEYS[2]) if not activeDigest or not previous then return 0 end -if activeDigest ~= ARGV[1] and redis.call('GET', KEYS[4]) ~= ARGV[5] then - return 0 +if activeDigest ~= ARGV[1] then + if ARGV[5] == '' or redis.call('GET', KEYS[4]) ~= ARGV[5] then + return 0 + end end redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[4]) -redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) -return 1 -`; -const ISSUE_CREDENTIAL_SCRIPT = ` -local generation = redis.call('GET', KEYS[4]) or '0' -if generation ~= ARGV[5] then - return 0 +if ARGV[5] ~= '' then + redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) +else + redis.call('DEL', KEYS[4]) end -redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[4]) -redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[4]) -redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) return 1 `; const REVOKE_WORKER_SCRIPT = ` @@ -50,18 +46,71 @@ if activeIncarnation then end return 1 `; +const ISSUE_PAIRING_SCRIPT = ` +local previous = redis.call('GET', KEYS[1]) +if previous then + redis.call('DEL', previous) +end +redis.call('DEL', KEYS[3]) +redis.call('SET', KEYS[1], KEYS[2], 'EX', ARGV[2]) +redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[2]) +return 1 +`; +const REDEEM_PAIRING_SCRIPT = ` +local pairing = redis.call('GET', KEYS[1]) +if not pairing then + return nil +end +if redis.call('GET', KEYS[2]) ~= KEYS[1] then + redis.call('DEL', KEYS[1]) + return nil +end +redis.call('DEL', KEYS[1], KEYS[2]) +redis.call('SET', KEYS[3], ARGV[1], 'EX', ARGV[2]) +return pairing +`; +const INSTALL_REDEEMED_CREDENTIAL_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return 0 +end +local generation = redis.call('GET', KEYS[5]) or '0' +if generation ~= ARGV[6] then + return 0 +end +redis.call('SET', KEYS[2], ARGV[3], 'EX', ARGV[4]) +redis.call('SET', KEYS[3], ARGV[2], 'EX', ARGV[4]) +if ARGV[5] ~= '' then + redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) +else + redis.call('DEL', KEYS[4]) +end +redis.call('DEL', KEYS[1]) +return 1 +`; + +export type BridgePrincipalType = 'deployment' | 'tenant' | 'user' | 'role' | 'group'; + +export interface BridgeWorkerBinding { + tenantId: string; + principal: { + type: BridgePrincipalType; + id: string; + }; +} interface StoredPairing { workerId: string; expiresAt: string; generation: number; + binding?: BridgeWorkerBinding; } interface StoredCredential { workerId: string; - identityId: string; + identityId?: string; publicKey: string; expiresAt: string; + binding?: BridgeWorkerBinding; } export interface BridgePairing { @@ -99,10 +148,6 @@ function pairingKey(code: string): string { return `${PREFIX}:pairing:${digest(code)}`; } -function credentialKey(credential: string): string { - return credentialDigestKey(digest(credential)); -} - function credentialDigestKey(credentialDigest: string): string { return `${PREFIX}:credential:${credentialDigest}`; } @@ -119,6 +164,14 @@ function workerPairingGenerationKey(workerId: string): string { return `${PREFIX}:pairing-generation:${workerId}`; } +function workerPairingIndexKey(workerId: string): string { + return `${PREFIX}:pairing-index:${workerId}`; +} + +function workerRedemptionKey(workerId: string): string { + return `${PREFIX}:redemption:${workerId}`; +} + function proofNonceKey(credential: string, nonce: string): string { return `${PREFIX}:proof:${digest(credential)}:${digest(nonce)}`; } @@ -138,7 +191,10 @@ export class RedisBridgePairingStore { private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, ) {} - async issue(workerId: string): Promise { + async issue( + workerId: string, + binding?: BridgeWorkerBinding, + ): Promise { const code = randomBytes(24).toString('base64url'); const expiresAt = new Date( Date.now() + this.pairingTtlSeconds * 1000, @@ -146,12 +202,16 @@ export class RedisBridgePairingStore { const generation = Number( (await this.redis.get(workerPairingGenerationKey(workerId))) ?? '0', ); - const pairing: StoredPairing = { workerId, expiresAt, generation }; - await this.redis.set( - pairingKey(code), + const pairing: StoredPairing = { workerId, expiresAt, generation, binding }; + const codeKey = pairingKey(code); + await this.redis.eval( + ISSUE_PAIRING_SCRIPT, + 3, + workerPairingIndexKey(workerId), + codeKey, + workerRedemptionKey(workerId), JSON.stringify(pairing), - 'EX', - this.pairingTtlSeconds, + String(this.pairingTtlSeconds), ); return { workerId, code, expiresAt }; } @@ -161,8 +221,24 @@ export class RedisBridgePairingStore { code: string; publicKey: string; }): Promise { - const raw = await this.redis.getdel(pairingKey(args.code)); - if (raw == null) { + if (!validEd25519PublicKey(args.publicKey)) { + throw new BridgePairingError( + 'PUBLIC_KEY_INVALID', + 'Worker public key must be an Ed25519 key', + ); + } + const codeKey = pairingKey(args.code); + const redemptionId = randomBytes(18).toString('base64url'); + const raw = await this.redis.eval( + REDEEM_PAIRING_SCRIPT, + 3, + codeKey, + workerPairingIndexKey(args.workerId), + workerRedemptionKey(args.workerId), + redemptionId, + String(this.pairingTtlSeconds), + ); + if (typeof raw !== 'string') { throw new BridgePairingError( 'PAIRING_INVALID', 'Pairing code is invalid or expired', @@ -175,19 +251,14 @@ export class RedisBridgePairingStore { 'Pairing code does not authorize this worker', ); } - if (!validEd25519PublicKey(args.publicKey)) { - throw new BridgePairingError( - 'PUBLIC_KEY_INVALID', - 'Worker public key must be an Ed25519 key', - ); - } - return await this.issueCredential( args.workerId, args.publicKey, undefined, undefined, + pairing.binding, pairing.generation, + redemptionId, ); } @@ -204,8 +275,9 @@ export class RedisBridgePairingStore { workerId: string; credentialId: string; activeCredentialId: string; - identityId: string; + identityId?: string; pairingGeneration: number; + binding?: BridgeWorkerBinding; }> { const proofTime = Date.parse(args.timestamp); if ( @@ -237,7 +309,11 @@ export class RedisBridgePairingStore { const active = activeRaw == null ? undefined : JSON.parse(activeRaw) as StoredCredential; - if (active?.identityId !== stored.identityId) { + if ( + stored.identityId == null || + active?.identityId == null || + stored.identityId !== active.identityId + ) { throw new BridgePairingError( 'CREDENTIAL_INVALID', 'Worker credential is invalid or expired', @@ -273,8 +349,9 @@ export class RedisBridgePairingStore { workerId: stored.workerId, credentialId: credentialDigest, activeCredentialId: activeDigest, - identityId: stored.identityId, + ...(stored.identityId != null ? { identityId: stored.identityId } : {}), pairingGeneration: Number(pairingGeneration ?? '0'), + ...(stored.binding ? { binding: stored.binding } : {}), }; } @@ -297,7 +374,8 @@ export class RedisBridgePairingStore { expectedCredentialId?: string, ): Promise { const identityKey = workerIdentityKey(workerId); - const previousDigest = expectedCredentialId ?? await this.redis.get(identityKey); + const previousDigest = + expectedCredentialId ?? (await this.redis.get(identityKey)); const previousRaw = previousDigest == null ? null @@ -313,7 +391,8 @@ export class RedisBridgePairingStore { workerId, previous.publicKey, previousDigest, - previous.identityId, + previous.identityId ?? null, + previous.binding, ); } @@ -321,15 +400,24 @@ export class RedisBridgePairingStore { workerId: string, publicKey: string, previousDigest?: string, - identityId = randomBytes(18).toString('base64url'), + identityId: string | null | undefined = randomBytes(18).toString('base64url'), + binding?: BridgeWorkerBinding, pairingGeneration?: number, + redemptionId?: string, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); const expiresAt = new Date( Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); - const stored: StoredCredential = { workerId, identityId, publicKey, expiresAt }; + const stableIdentityId = identityId ?? undefined; + const stored: StoredCredential = { + workerId, + ...(stableIdentityId != null ? { identityId: stableIdentityId } : {}), + publicKey, + expiresAt, + binding, + }; if (previousDigest !== undefined) { const rotated = await this.redis.eval( ROTATE_CREDENTIAL_SCRIPT, @@ -342,7 +430,7 @@ export class RedisBridgePairingStore { credentialDigest, JSON.stringify(stored), String(this.credentialTtlSeconds), - identityId, + stableIdentityId ?? '', ); if (rotated !== 1) { throw new BridgePairingError( @@ -352,23 +440,31 @@ export class RedisBridgePairingStore { } return { workerId, credential, expiresAt }; } - const issued = await this.redis.eval( - ISSUE_CREDENTIAL_SCRIPT, - 4, + if (redemptionId == null) { + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing redemption was not fenced', + ); + } + const installed = await this.redis.eval( + INSTALL_REDEEMED_CREDENTIAL_SCRIPT, + 5, + workerRedemptionKey(workerId), credentialDigestKey(credentialDigest), workerIdentityKey(workerId), workerStableIdentityKey(workerId), workerPairingGenerationKey(workerId), - JSON.stringify(stored), + redemptionId, credentialDigest, - identityId, + JSON.stringify(stored), String(this.credentialTtlSeconds), + stableIdentityId ?? '', String(pairingGeneration ?? 0), ); - if (issued !== 1) { + if (installed !== 1) { throw new BridgePairingError( 'PAIRING_INVALID', - 'Pairing code was revoked before redemption completed', + 'Pairing code was superseded before credential installation', ); } return { workerId, credential, expiresAt }; diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index a4563399..2b3baa10 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -25,6 +25,171 @@ afterEach(async () => { }); describe('paired bridge HTTP API', () => { + test('rejects a malformed optional binding for a configured worker', async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings: new RedisBridgePairingStore(redis), + authMode: 'paired', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + + const response = await fetch( + `http://127.0.0.1:${address.port}/v1/bridge/pairings`, + { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + workerId: 'vm-1', + binding: { tenantId: 'tenant-1', principal: { type: 'user' } }, + }), + }, + ); + + expect(response.status).toBe(400); + }); + + test('requires and persists a trusted principal binding for dynamic workers', async () => { + const store = new RedisBridgeStore(redis); + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'paired', + adminToken: 'strong-administrator-bootstrap-token', + allowDynamicWorkers: true, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + const baseUrl = `http://127.0.0.1:${address.port}/v1/bridge`; + const unboundResponse = await fetch(`${baseUrl}/pairings`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ workerId: 'user-vm' }), + }); + expect(unboundResponse.status).toBe(400); + + const binding = { + tenantId: 'tenant-1', + principal: { type: 'user' as const, id: 'user-1' }, + }; + const pairingResponse = await fetch(`${baseUrl}/pairings`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ workerId: 'user-vm', binding }), + }); + const pairing = (await pairingResponse.json()) as { code: string }; + expect(pairingResponse.status).toBe(200); + + const identity = createBridgeIdentity(); + const redemptionResponse = await fetch(`${baseUrl}/pairings/redeem`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'user-vm', + code: pairing.code, + publicKey: identity.publicKey, + }), + }); + const issued = (await redemptionResponse.json()) as { credential: string }; + expect(redemptionResponse.status).toBe(200); + + const path = '/v1/bridge/workers/register'; + const body = JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'user-vm', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + binding: { + tenantId: 'tenant-2', + principal: { type: 'user', id: 'attacker-selected-user' }, + }, + }); + const proof = { + credential: issued.credential, + method: 'POST', + path, + timestamp: new Date().toISOString(), + nonce: 'dynamic-registration-nonce', + body, + }; + const registrationResponse = await fetch( + `http://127.0.0.1:${address.port}${path}`, + { + method: 'POST', + headers: { + Authorization: `Bridge ${issued.credential}`, + 'Content-Type': 'application/json', + 'X-LibreChat-Code-Timestamp': proof.timestamp, + 'X-LibreChat-Code-Nonce': proof.nonce, + 'X-LibreChat-Code-Signature': signBridgeRequest( + identity.privateKey, + proof, + ), + }, + body, + }, + ); + expect(registrationResponse.status).toBe(200); + await expect( + store.dispatch({ + workerId: 'user-vm', + tenantId: binding.tenantId, + requireTenantBinding: true, + body: { language: 'bash' } as never, + headers: {}, + runtimeSessionId: 'stateful-session', + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + + await expect( + store.dispatch({ + workerId: 'user-vm', + tenantId: 'tenant-2', + requireTenantBinding: true, + body: { language: 'bash' } as never, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_UNAUTHORIZED' }); + }); + test('pairs a worker and accepts its proof-of-possession registration', async () => { const app = express(); const store = new RedisBridgeStore(redis); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 8ba474ad..e41336d4 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -3,6 +3,7 @@ import { timingSafeEqual } from 'crypto'; import { Router } from 'express'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; import type { BridgeWorkerRegistration } from '../../../packages/code/src/protocol'; +import type { BridgePrincipalType, BridgeWorkerBinding } from './pairing'; import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; import { @@ -15,6 +16,14 @@ import { BridgeStoreError, RedisBridgeStore } from './store'; const INCARNATION_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; const MAX_LEASE_WAIT_MS = 30_000; +const BRIDGE_BINDING_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const PRINCIPAL_TYPES = new Set([ + 'deployment', + 'tenant', + 'user', + 'role', + 'group', +]); export type BridgeAuthMode = 'static' | 'paired'; @@ -24,6 +33,7 @@ export interface BridgeRouterOptions { authMode: BridgeAuthMode; adminToken: string; configuredWorkerId?: string; + allowDynamicWorkers?: boolean; } function sameToken(left: string, right: string): boolean { @@ -47,6 +57,28 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null; } +function parseBinding(value: unknown): BridgeWorkerBinding | undefined { + if (!isRecord(value) || !isRecord(value.principal)) return undefined; + const { tenantId, principal } = value; + if ( + typeof tenantId !== 'string' || + !BRIDGE_BINDING_ID_PATTERN.test(tenantId) || + typeof principal.type !== 'string' || + !PRINCIPAL_TYPES.has(principal.type as BridgePrincipalType) || + typeof principal.id !== 'string' || + !BRIDGE_BINDING_ID_PATTERN.test(principal.id) + ) { + return undefined; + } + return { + tenantId, + principal: { + type: principal.type as BridgePrincipalType, + id: principal.id, + }, + }; +} + function asyncRoute( handler: (req: Request, res: Response) => Promise, ): RequestHandler { @@ -59,6 +91,8 @@ function sendStoreError(error: BridgeStoreError, res: Response): void { const status = error.code === 'ASSIGNMENT_NOT_FOUND' ? 404 + : error.code === 'WORKER_UNAUTHORIZED' + ? 403 : error.code === 'WORKER_BUSY' ? 503 : 409; @@ -92,9 +126,10 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { const router = Router(); const configuredWorker = (workerId: string): boolean => - options.configuredWorkerId != null && - options.configuredWorkerId !== '' && - workerId === options.configuredWorkerId; + options.allowDynamicWorkers === true || + (options.configuredWorkerId != null && + options.configuredWorkerId !== '' && + workerId === options.configuredWorkerId); const bearerToken = (req: Request): string => req @@ -201,7 +236,20 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { res.status(400).json({ error: 'Invalid bridge worker ID' }); return; } - const pairing = await options.pairings.issue(workerId); + const hasBinding = isRecord(req.body) && + Object.prototype.hasOwnProperty.call(req.body, 'binding'); + const binding = isRecord(req.body) ? parseBinding(req.body.binding) : undefined; + if (hasBinding && binding == null) { + res.status(400).json({ error: 'Invalid bridge worker principal binding' }); + return; + } + if (options.allowDynamicWorkers === true && binding == null) { + res.status(400).json({ + error: 'Dynamic bridge workers require a valid principal binding', + }); + return; + } + const pairing = await options.pairings.issue(workerId, binding); res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...pairing }); })); @@ -283,17 +331,40 @@ router.post( }); return; } + const authorization = options.authMode === 'paired' + ? ( + res.locals.bridgeWorkerAuthorization as { + identityId: string; + pairingGeneration: number; + credentialId: string; + activeCredentialId: string; + binding?: BridgeWorkerBinding; + } + ) + : undefined; + const trustedRegistration: BridgeWorkerRegistration & { + binding?: BridgeWorkerBinding; + credentialId?: string; + identityId?: string; + } = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: registration.workerId, + incarnationId: registration.incarnationId, + capabilities: registration.capabilities, + ...(authorization?.credentialId != null + ? { credentialId: authorization.credentialId } + : {}), + ...(authorization?.identityId != null + ? { identityId: authorization.identityId } + : {}), + ...(authorization?.binding != null + ? { binding: authorization.binding } + : {}), + }; try { await options.store.register( - registration as unknown as BridgeWorkerRegistration, - options.authMode === 'paired' - ? ( - res.locals.bridgeWorkerAuthorization as { - identityId: string; - pairingGeneration: number; - } - ) - : undefined, + trustedRegistration, + authorization, ); } catch (error) { if (error instanceof BridgeStoreError) { @@ -403,6 +474,11 @@ router.post( body.incarnationId, Math.min(requestedWait, MAX_LEASE_WAIT_MS), leaseController.signal, + ( + res.locals.bridgeWorkerAuthorization as + | { identityId: string } + | undefined + )?.identityId, ); if (leaseController.signal.aborted) { if (assignment != null) await options.store.returnLease(assignment); @@ -495,6 +571,11 @@ router.post( req.params.assignmentId, settlement, settlementController.signal, + ( + res.locals.bridgeWorkerAuthorization as + | { identityId: string } + | undefined + )?.identityId, ); if (!settlementController.signal.aborted) { res.json({ diff --git a/service/src/bridge/selection.test.ts b/service/src/bridge/selection.test.ts new file mode 100644 index 00000000..b6309702 --- /dev/null +++ b/service/src/bridge/selection.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from 'bun:test'; + +import { + BridgeWorkerSelectionError, + resolveBridgeWorkerSelection, +} from './selection'; + +describe('bridge worker request selection', () => { + test('uses the configured compatibility worker when no dynamic worker is requested', () => { + expect( + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: 'deployment-worker', + dynamicWorkers: true, + }), + ).toEqual({ workerId: 'deployment-worker', explicit: false }); + }); + + test('selects only the worker authenticated by the LibreChat JWT', () => { + expect( + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: 'deployment-worker', + dynamicWorkers: true, + requestedWorkerId: 'code-user_1', + trustedWorkerId: 'code-user_1', + }), + ).toEqual({ workerId: 'code-user_1', explicit: true }); + + expect( + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: 'deployment-worker', + dynamicWorkers: true, + trustedWorkerId: 'code-user_1', + }), + ).toEqual({ workerId: 'code-user_1', explicit: true }); + }); + + test('rejects a caller-controlled worker header without a matching trusted claim', () => { + expect(() => + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: 'deployment-worker', + dynamicWorkers: true, + requestedWorkerId: 'victim-worker', + }), + ).toThrow('Code bridge worker selection is not authenticated'); + expect(() => + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: 'deployment-worker', + dynamicWorkers: true, + requestedWorkerId: 'victim-worker', + trustedWorkerId: 'caller-worker', + }), + ).toThrow('Code bridge worker selection does not match the authenticated claim'); + }); + + test('rejects dynamic routing on the wrong backend or when it is disabled', () => { + expect(() => + resolveBridgeWorkerSelection({ + backend: 'http', + configuredWorkerId: '', + dynamicWorkers: true, + requestedWorkerId: 'code-user-1', + trustedWorkerId: 'code-user-1', + }), + ).toThrow(BridgeWorkerSelectionError); + expect(() => + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: 'deployment-worker', + dynamicWorkers: false, + requestedWorkerId: 'code-user-1', + trustedWorkerId: 'code-user-1', + }), + ).toThrow('Dynamic code bridge workers are disabled'); + }); + + test('rejects malformed worker IDs before they cross the queue boundary', () => { + expect(() => + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: '', + dynamicWorkers: true, + requestedWorkerId: '../worker', + trustedWorkerId: '../worker', + }), + ).toThrow('Invalid code bridge worker ID'); + expect(() => + resolveBridgeWorkerSelection({ + backend: 'remote-bridge', + configuredWorkerId: '', + dynamicWorkers: true, + requestedWorkerId: 'victim:assignments', + trustedWorkerId: 'victim:assignments', + }), + ).toThrow('Invalid code bridge worker ID'); + }); +}); diff --git a/service/src/bridge/selection.ts b/service/src/bridge/selection.ts new file mode 100644 index 00000000..0959279f --- /dev/null +++ b/service/src/bridge/selection.ts @@ -0,0 +1,72 @@ +export const CODEAPI_BRIDGE_WORKER_HEADER = 'X-LibreChat-Code-Worker-ID'; +export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +export class BridgeWorkerSelectionError extends Error { + constructor( + message: string, + public readonly status: 400 | 403 | 503, + ) { + super(message); + this.name = 'BridgeWorkerSelectionError'; + } +} + +export function resolveBridgeWorkerSelection(args: { + backend: SandboxBackendName; + configuredWorkerId: string; + dynamicWorkers: boolean; + requestedWorkerId?: string; + trustedWorkerId?: string; +}): { workerId: string; explicit: boolean } | undefined { + const requestedWorkerId = args.requestedWorkerId?.trim(); + const trustedWorkerId = args.trustedWorkerId?.trim(); + const hasRequestedWorker = requestedWorkerId != null && requestedWorkerId.length > 0; + const hasTrustedWorker = trustedWorkerId != null && trustedWorkerId.length > 0; + if (hasRequestedWorker || hasTrustedWorker) { + if (args.backend !== 'remote-bridge') { + throw new BridgeWorkerSelectionError( + 'Code bridge worker routing requires the remote-bridge backend', + 400, + ); + } + if (hasRequestedWorker && !hasTrustedWorker) { + throw new BridgeWorkerSelectionError( + 'Code bridge worker selection is not authenticated', + 403, + ); + } + if ( + hasRequestedWorker && + hasTrustedWorker && + requestedWorkerId !== trustedWorkerId + ) { + throw new BridgeWorkerSelectionError( + 'Code bridge worker selection does not match the authenticated claim', + 403, + ); + } + const selectedWorkerId = trustedWorkerId as string; + if (!BRIDGE_WORKER_ID_PATTERN.test(selectedWorkerId)) { + throw new BridgeWorkerSelectionError('Invalid code bridge worker ID', 400); + } + if (!args.dynamicWorkers && selectedWorkerId !== args.configuredWorkerId) { + throw new BridgeWorkerSelectionError('Dynamic code bridge workers are disabled', 403); + } + return { + workerId: selectedWorkerId, + explicit: true, + }; + } + + if (args.backend !== 'remote-bridge') return undefined; + const configuredWorkerId = args.configuredWorkerId.trim(); + if (configuredWorkerId.length === 0) { + throw new BridgeWorkerSelectionError('No code bridge worker was selected', 503); + } + if (!BRIDGE_WORKER_ID_PATTERN.test(configuredWorkerId)) { + throw new BridgeWorkerSelectionError('Invalid configured code bridge worker ID', 503); + } + return { workerId: configuredWorkerId, explicit: false }; +} + +type SandboxBackendName = 'http' | 'lambda-microvm' | 'remote-bridge'; diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts index 24637a20..23b04067 100644 --- a/service/src/bridge/store.test.ts +++ b/service/src/bridge/store.test.ts @@ -23,6 +23,270 @@ afterEach(async () => { }); describe('RedisBridgeStore', () => { + test('rejects a registration whose authenticated identity was replaced', async () => { + await redis.set( + 'codeapi:bridge:v1:identity:fenced-worker', + 'replacement-credential-digest', + ); + + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'fenced-worker', + incarnationId, + credentialId: 'stale-credential-digest', + identityId: 'stale-identity', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }, 'stale-credential-digest'), + ).rejects.toMatchObject({ code: 'WORKER_UNAUTHORIZED' }); + + await expect( + redis.get('codeapi:bridge:v1:worker:fenced-worker'), + ).resolves.toBeNull(); + }); + + test('accepts registration after a same-identity credential rotation', async () => { + await redis.set( + 'codeapi:bridge:v1:identity:rotating-registration-worker', + 'new-active-credential-digest', + ); + await redis.set( + 'codeapi:bridge:v1:stable-identity:rotating-registration-worker', + 'stable-worker-identity', + ); + + await expect( + store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'rotating-registration-worker', + incarnationId, + credentialId: 'old-authenticated-credential-digest', + identityId: 'stable-worker-identity', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }, 'old-authenticated-credential-digest'), + ).resolves.toBeUndefined(); + }); + + test('rejects a dynamic worker lease outside its bound tenant', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'tenant-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + binding: { + tenantId: 'tenant-1', + principal: { type: 'user', id: 'user-1' }, + }, + }); + + await expect( + store.dispatch({ + workerId: 'tenant-worker', + tenantId: 'tenant-2', + requireTenantBinding: true, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_UNAUTHORIZED' }); + }); + + test('does not lease an assignment to a newly rebound worker identity', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'rebound-worker', + incarnationId, + identityId: 'tenant-a-identity', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + binding: { + tenantId: 'tenant-a', + principal: { type: 'user', id: 'user-a' }, + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'rebound-worker', + tenantId: 'tenant-a', + requireTenantBinding: true, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + + await expect( + store.lease( + 'rebound-worker', + incarnationId, + 1_000, + undefined, + 'tenant-b-identity', + ), + ).resolves.toBeUndefined(); + await expect( + store.lease( + 'rebound-worker', + incarnationId, + 1_000, + undefined, + 'tenant-a-identity', + ), + ).resolves.toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + }); + + test('a stale identity poll cannot consume work queued for the replacement identity', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'replacement-worker', + incarnationId, + identityId: 'replacement-identity', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + binding: { + tenantId: 'tenant-a', + principal: { type: 'user', id: 'user-a' }, + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'replacement-worker', + tenantId: 'tenant-a', + requireTenantBinding: true, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + + await expect( + store.lease( + 'replacement-worker', + incarnationId, + 100, + undefined, + 'stale-identity', + ), + ).resolves.toBeUndefined(); + await expect( + store.lease( + 'replacement-worker', + incarnationId, + 1_000, + undefined, + 'replacement-identity', + ), + ).resolves.toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + }); + + test('a stale incarnation poll cannot consume replacement incarnation work', async () => { + const replacementIncarnationId = 'incarnation-00000002'; + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'restarted-worker', + incarnationId: replacementIncarnationId, + identityId: 'stable-restarted-identity', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'restarted-worker', + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + + await expect( + store.lease( + 'restarted-worker', + incarnationId, + 100, + undefined, + 'stable-restarted-identity', + ), + ).resolves.toBeUndefined(); + await expect( + store.lease( + 'restarted-worker', + replacementIncarnationId, + 1_000, + undefined, + 'stable-restarted-identity', + ), + ).resolves.toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + }); + + test('leases queued work after credential refresh preserves the paired identity', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'rotating-worker', + incarnationId, + identityId: 'stable-paired-identity', + credentialId: 'credential-before-refresh', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + binding: { + tenantId: 'tenant-a', + principal: { type: 'user', id: 'user-a' }, + }, + }); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId: 'rotating-worker', + tenantId: 'tenant-a', + requireTenantBinding: true, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + + const assignment = await store.lease( + 'rotating-worker', + incarnationId, + 1_000, + undefined, + 'stable-paired-identity', + ); + + expect(assignment).toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + }); + test('delivers and settles one fenced stateful assignment', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 90eef488..3a43298b 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -9,6 +9,7 @@ import type { } from '../../../packages/code/src/protocol'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import type { BridgeWorkerBinding } from './pairing'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; @@ -28,6 +29,7 @@ export class BridgeStoreError extends Error { constructor( public readonly code: | 'WORKER_OFFLINE' + | 'WORKER_UNAUTHORIZED' | 'WORKER_BUSY' | 'ASSIGNMENT_EXPIRED' | 'ASSIGNMENT_FENCED' @@ -45,12 +47,23 @@ export class BridgeStoreError extends Error { interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; + workerIdentityId?: string; +} + +export interface RegisteredBridgeWorker extends BridgeWorkerRegistration { + binding?: BridgeWorkerBinding; + credentialId?: string; + identityId?: string; } function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } +function workerStableIdentityKey(workerId: string): string { + return `${PREFIX}:stable-identity:${workerId}`; +} + function workerIncarnationKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`; } @@ -221,14 +234,36 @@ export class RedisBridgeStore { } async register( - registration: BridgeWorkerRegistration, - authorization?: { identityId: string; pairingGeneration: number }, + registration: RegisteredBridgeWorker, + authorization?: string | { + identityId?: string; + pairingGeneration?: number; + activeCredentialId?: string; + }, ): Promise { + const authorizationObject = + typeof authorization === 'object' ? authorization : undefined; + const expectedActiveCredentialId = + typeof authorization === 'string' + ? authorization + : authorizationObject?.activeCredentialId; const script = [ 'if ARGV[5] ~= "" then', ' local pairingGeneration = redis.call(\'GET\', KEYS[7]) or "0"', - ' if pairingGeneration ~= ARGV[5] then return -4 end', - ' if redis.call(\'GET\', KEYS[8]) ~= ARGV[6] then return -4 end', + ' if pairingGeneration ~= ARGV[5] then return -5 end', + ' if ARGV[6] ~= "" then', + ' if redis.call(\'GET\', KEYS[8]) ~= ARGV[6] then return -5 end', + ' elseif ARGV[7] ~= "" and redis.call(\'GET\', KEYS[9]) ~= ARGV[7] then return -5', + ' end', + 'end', + 'if ARGV[8] ~= "" then', + ' local stableIdentity = redis.call(\'GET\', KEYS[8])', + ' if stableIdentity and stableIdentity ~= ARGV[8] then return -4 end', + ' if not stableIdentity then', + ' if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[9]) ~= ARGV[7] then return -4 end', + ' redis.call(\'SET\', KEYS[8], ARGV[8], "EX", ARGV[3])', + ' end', + 'elseif ARGV[7] ~= "" and redis.call(\'GET\', KEYS[9]) ~= ARGV[7] then return -4', 'end', 'if redis.call(\'EXISTS\', KEYS[3]) == 1 then return -2 end', 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return -1 end', @@ -251,7 +286,7 @@ export class RedisBridgeStore { await boundedCommand( this.redis.eval( script, - 8, + 9, workerKey(registration.workerId), incarnationFenceKey(registration.workerId, registration.incarnationId), quarantineKey(registration.workerId, registration.incarnationId), @@ -260,12 +295,17 @@ export class RedisBridgeStore { lockIncarnationKey(registration.workerId), `${PREFIX}:pairing-generation:${registration.workerId}`, `${PREFIX}:stable-identity:${registration.workerId}`, + `${PREFIX}:identity:${registration.workerId}`, registration.incarnationId, JSON.stringify(registration), String(this.workerTtlSeconds), `${PREFIX}:worker:${encodeURIComponent(registration.workerId)}:incarnation:`, - authorization == null ? '' : String(authorization.pairingGeneration), - authorization?.identityId ?? '', + authorizationObject?.pairingGeneration == null + ? '' + : String(authorizationObject.pairingGeneration), + authorizationObject?.identityId ?? '', + expectedActiveCredentialId ?? '', + registration.identityId ?? '', ), this.redisCommandTimeoutMs, 'Bridge worker registration', @@ -290,6 +330,12 @@ export class RedisBridgeStore { ); } if (result === -4) { + throw new BridgeStoreError( + 'WORKER_UNAUTHORIZED', + 'Bridge worker authorization was revoked before registration completed', + ); + } + if (result === -5) { throw new BridgeStoreError( 'WORKER_FENCED', 'Bridge worker authorization was revoked before registration completed', @@ -299,6 +345,8 @@ export class RedisBridgeStore { async dispatch(args: { workerId: string; + tenantId?: string; + requireTenantBinding?: boolean; body: t.PayloadBody; headers: Record; runtimeSessionId?: string; @@ -320,6 +368,18 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} is offline`, ); } + if ( + (args.requireTenantBinding === true && registration.binding == null) || + (registration.binding != null && + (args.tenantId == null || + args.tenantId.length === 0 || + registration.binding.tenantId !== args.tenantId)) + ) { + throw new BridgeStoreError( + 'WORKER_UNAUTHORIZED', + `Bridge worker ${args.workerId} is not authorized for this tenant`, + ); + } if ( args.runtimeSessionId !== undefined && registration.capabilities.statefulWorkspace !== true @@ -384,6 +444,9 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), + ...(registration.identityId != null + ? { workerIdentityId: registration.identityId } + : {}), expiresAt: new Date(args.deadlineAtMs).toISOString(), runtimeSessionId: args.runtimeSessionId, request: { @@ -477,6 +540,7 @@ export class RedisBridgeStore { incarnationId: string, waitMs: number, signal?: AbortSignal, + identityId?: string, ): Promise { const deadline = Date.now() + waitMs; let firstPoll = true; @@ -488,7 +552,7 @@ export class RedisBridgeStore { let assignmentId: string | null; try { assignmentId = await this.leaseCommand( - this.claimOrPopLease(workerId, incarnationId), + this.claimOrPopLease(workerId, incarnationId, identityId), signal, 'Bridge lease claim', ); @@ -536,6 +600,14 @@ export class RedisBridgeStore { 'Bridge worker incarnation was replaced', ); } + if (assignment.workerIdentityId !== identityId) { + await this.leaseCommand( + this.discardLeaseClaim(workerId, incarnationId, assignmentId), + signal, + 'Bridge unauthorized lease discard', + ); + continue; + } if (Date.parse(assignment.expiresAt) <= Date.now()) { const acknowledged = (await this.leaseCommand( @@ -561,7 +633,11 @@ export class RedisBridgeStore { await this.returnLease(assignment); return undefined; } - const { leaseTokenHash: _leaseTokenHash, ...wireAssignment } = assignment; + const { + leaseTokenHash: _leaseTokenHash, + workerIdentityId: _workerIdentityId, + ...wireAssignment + } = assignment; return { ...wireAssignment, remainingMs: Math.max( @@ -643,9 +719,15 @@ export class RedisBridgeStore { private async claimOrPopLease( workerId: string, incarnationId: string, + identityId?: string, ): Promise { const result = await this.redis.eval( [ + "if ARGV[1] ~= '' then", + " if redis.call('GET', KEYS[3]) ~= ARGV[1] then return nil end", + "elseif redis.call('EXISTS', KEYS[3]) == 1 then", + ' return nil', + 'end', "local claimed = redis.call('GET', KEYS[2])", 'if claimed then return claimed end', "local ttl = redis.call('TTL', KEYS[1])", @@ -654,9 +736,11 @@ export class RedisBridgeStore { "redis.call('SET', KEYS[2], assignment, 'EX', math.max(1, ttl))", 'return assignment', ].join('\n'), - 2, + 3, queueKey(workerId, incarnationId), leaseClaimKey(workerId, incarnationId), + workerStableIdentityKey(workerId), + identityId ?? '', ); return result == null ? null : String(result); } @@ -760,6 +844,7 @@ export class RedisBridgeStore { assignmentId: string, settlement: CodeBridgeSettlement, signal?: AbortSignal, + identityId?: string, ): Promise { const serializedSettlement = JSON.stringify(settlement); const existingSettlement = await this.leaseCommand( @@ -800,7 +885,8 @@ export class RedisBridgeStore { settlement.incarnationId !== assignment.incarnationId || registration?.incarnationId !== settlement.incarnationId || settlement.generation !== assignment.generation || - tokenHash(settlement.leaseToken) !== assignment.leaseTokenHash + tokenHash(settlement.leaseToken) !== assignment.leaseTokenHash || + assignment.workerIdentityId !== identityId ) { throw new BridgeStoreError( 'ASSIGNMENT_FENCED', @@ -829,6 +915,11 @@ export class RedisBridgeStore { workspaceQuarantineKey(workerId, assignment.runtimeSessionId), ); } + const hasWorkspace = assignment.runtimeSessionId !== undefined; + settlementKeys.push( + `${PREFIX}:stable-identity:${workerId}`, + workerIncarnationKey(workerId), + ); const script = [ 'local existing = redis.call(\'GET\', KEYS[2])', 'if existing then', @@ -836,11 +927,17 @@ export class RedisBridgeStore { ' return -1', 'end', 'if redis.call(\'EXISTS\', KEYS[1]) == 0 then return 0 end', - 'if #KEYS == 6 and redis.call(\'GET\', KEYS[6]) ~= ARGV[3] then return -2 end', + 'if ARGV[6] == "1" and redis.call(\'GET\', KEYS[6]) ~= ARGV[3] then return -2 end', 'if ARGV[4] ~= "rejected" and redis.call(\'EXISTS\', KEYS[5]) == 0 then return -3 end', + 'local stableIdentityKey = KEYS[#KEYS - 1]', + 'if ARGV[5] ~= "" then', + ' if redis.call(\'GET\', stableIdentityKey) ~= ARGV[5] then return -4 end', + 'elseif redis.call(\'EXISTS\', stableIdentityKey) == 1 then return -4', + 'end', + 'if redis.call(\'GET\', KEYS[#KEYS]) ~= ARGV[7] then return -4 end', 'redis.call(\'SET\', KEYS[2], ARGV[1], \"EX\", ARGV[2])', 'if redis.call(\'GET\', KEYS[3]) == ARGV[3] then redis.call(\'DEL\', KEYS[3], KEYS[4]) end', - 'if #KEYS == 6 and ARGV[4] == \"rejected\" then redis.call(\'DEL\', KEYS[6]) end', + 'if ARGV[6] == "1" and ARGV[4] == "rejected" then redis.call(\'DEL\', KEYS[6]) end', 'return 1', ].join('\n'); const accepted = Number( @@ -853,6 +950,9 @@ export class RedisBridgeStore { String(ttlSeconds), assignmentId, settlement.status, + identityId ?? '', + hasWorkspace ? '1' : '0', + settlement.incarnationId, ), signal, 'Bridge settlement commit', @@ -876,6 +976,12 @@ export class RedisBridgeStore { 'Bridge assignment expired before settlement was committed', ); } + if (accepted === -4) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Bridge assignment owner changed before settlement was committed', + ); + } if (accepted !== 1 && accepted !== 2) { throw new BridgeStoreError( 'ASSIGNMENT_EXPIRED', @@ -992,9 +1098,9 @@ export class RedisBridgeStore { private async registration( workerId: string, - ): Promise { + ): Promise { const raw = await this.redis.get(workerKey(workerId)); - return raw == null ? undefined : (JSON.parse(raw) as BridgeWorkerRegistration); + return raw == null ? undefined : (JSON.parse(raw) as RegisteredBridgeWorker); } private assertDispatchActive( diff --git a/service/src/config.ts b/service/src/config.ts index 605c701b..95df4eba 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -365,6 +365,8 @@ export const env = { * - `remote-bridge`: dispatch to an outbound-connected @librechat/code worker. */ SANDBOX_BACKEND: sandboxBackend, + /** Permit trusted callers to route each execution to a paired worker ID. */ + BRIDGE_DYNAMIC_WORKERS: process.env.CODEAPI_BRIDGE_DYNAMIC_WORKERS === 'true', /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ diff --git a/service/src/execution-profile.test.ts b/service/src/execution-profile.test.ts index da20bce2..7ba4640f 100644 --- a/service/src/execution-profile.test.ts +++ b/service/src/execution-profile.test.ts @@ -2,8 +2,11 @@ import { describe, expect, test } from 'bun:test'; import { checkExecutionProfileExpectation, queueNamesForExecutionProfile, + queueNameForExecution, resolveExecutionProfile, resolveExecutionProfileSource, + resolveQueuedSandboxBackend, + validateQueuedSandboxBackend, validateQueuedExecutionProfile, } from './execution-profile'; @@ -55,6 +58,42 @@ describe('execution profile queue isolation', () => { other: 'stateful-other-queue', }); }); + + test('routes a persisted remote bridge replay to the bridge queue on a lambda API', () => { + expect( + queueNameForExecution( + 'python', + 'stateful', + 'explicit', + 'remote-bridge', + ), + ).toBe('remote-bridge-python-queue'); + }); + + test('isolates outbound bridge jobs from Lambda consumers', () => { + expect( + queueNamesForExecutionProfile('stateful', 'explicit', 'remote-bridge'), + ).toEqual({ + python: 'remote-bridge-python-queue', + other: 'remote-bridge-other-queue', + }); + expect( + queueNamesForExecutionProfile('stateful', 'explicit', 'lambda-microvm'), + ).toEqual({ + python: 'stateful-python-queue', + other: 'stateful-other-queue', + }); + }); + + test('labels API-only stateful jobs with their Lambda worker backend', () => { + expect(resolveQueuedSandboxBackend('stateful', 'http')).toBe('lambda-microvm'); + expect(resolveQueuedSandboxBackend('default', 'http', 'explicit')).toBe('http'); + expect(resolveQueuedSandboxBackend('stateful', 'remote-bridge')).toBe('remote-bridge'); + }); + + test('leaves the backend unfenced for inferred stateless legacy queues', () => { + expect(resolveQueuedSandboxBackend('default', 'http', 'inferred')).toBeUndefined(); + }); }); describe('execution profile request assertion', () => { @@ -107,3 +146,29 @@ describe('queued execution profile validation', () => { ); }); }); + +describe('queued sandbox backend validation', () => { + test('accepts matching and legacy jobs', () => { + expect(() => + validateQueuedSandboxBackend('remote-bridge', 'remote-bridge'), + ).not.toThrow(); + expect(() => validateQueuedSandboxBackend(undefined, 'http')).not.toThrow(); + expect(() => + validateQueuedSandboxBackend(undefined, 'remote-bridge', 'legacy-bridge-worker'), + ).not.toThrow(); + }); + + test('rejects invalid and cross-backend jobs', () => { + expect(() => validateQueuedSandboxBackend('invalid', 'http')).toThrow( + 'Queued job has invalid sandbox backend', + ); + expect(() => + validateQueuedSandboxBackend('remote-bridge', 'lambda-microvm'), + ).toThrow( + 'Queued job targets the remote-bridge sandbox backend, but worker serves lambda-microvm', + ); + expect(() => + validateQueuedSandboxBackend(undefined, 'lambda-microvm', 'legacy-bridge-worker'), + ).toThrow('Legacy queued bridge job cannot run on the lambda-microvm sandbox backend'); + }); +}); diff --git a/service/src/execution-profile.ts b/service/src/execution-profile.ts index c4951903..38e9bc8c 100644 --- a/service/src/execution-profile.ts +++ b/service/src/execution-profile.ts @@ -1,4 +1,9 @@ export const EXECUTION_PROFILES = ['default', 'stateful'] as const; +export const SANDBOX_BACKENDS = [ + 'http', + 'lambda-microvm', + 'remote-bridge', +] as const; export type ExecutionProfile = typeof EXECUTION_PROFILES[number]; export type ExecutionProfileSource = 'explicit' | 'inferred'; @@ -11,6 +16,30 @@ export interface ExecutionProfileQueueNames { other: string; } +export type SandboxBackendName = typeof SANDBOX_BACKENDS[number]; + +/** Resolve the backend owned by the queue consumer rather than the API pod. + * Stateful API-only pods intentionally retain the HTTP local default while + * dispatching to Lambda workers. */ +export function resolveQueuedSandboxBackend( + profile: ExecutionProfile, + apiBackend: SandboxBackendName, + source: ExecutionProfileSource = 'explicit', +): SandboxBackendName | undefined { + if (profile === 'stateful' && apiBackend === 'http') { + return 'lambda-microvm'; + } + /* An inferred default profile still uses the pre-fencing legacy queues. + * Its API-only process cannot distinguish the supported HTTP and Lambda + * consumers because Lambda-only configuration belongs to the worker pod. + * Preserve that rollout topology by leaving the backend absent, exactly as + * pre-fencing producers did; explicit profiles regain strict fencing. */ + if (profile === 'default' && source === 'inferred' && apiBackend === 'http') { + return undefined; + } + return apiBackend; +} + export function resolveExecutionProfile( raw: string | undefined, runtimeSessionMode: 'stateless' | 'affinity' | 'strict', @@ -54,10 +83,17 @@ const EXPLICIT_PROFILE_QUEUE_NAMES: Record { validateApiHardenedConfig(); validateApiBridgePolicy(); validateExecutionProfilePolicy({ requireBackendMatch: false }); - /* No validateSandboxBackendPolicy() here: an API-only pod authenticates and + validateApiSandboxBackendPolicy(); + /* No full validateSandboxBackendPolicy() here: an API-only pod authenticates and * enqueues jobs, it never constructs the Lambda backend or checkpoint store. * Bridge credentials are validated separately above because this process * exposes the public registration, lease, and settlement routes. @@ -254,12 +261,7 @@ export async function gracefulShutdown(): Promise { } // Close queue connections (both API and Worker need this) - await Promise.all([ - pyQueue.close(), - otherQueue.close(), - pyQueueEvents.close(), - otherQueueEvents.close() - ]); + await closeQueueConnections(); logger.info('Queue connections closed'); // Only disconnect Redis if explicitly requested diff --git a/service/src/local-api.ts b/service/src/local-api.ts index 66896b63..df35cb5e 100644 --- a/service/src/local-api.ts +++ b/service/src/local-api.ts @@ -14,7 +14,7 @@ import bridgeRouter from './bridge'; import { requestErrorLogger, requestNotFoundLogger } from './middleware/request-error-logger'; import { executionProfileMiddleware } from './middleware/execution-profile'; import { localAuth } from './auth/local'; -import { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, connection } from './queue'; +import { pyQueue, otherQueue, connection, closeQueueConnections } from './queue'; import { setStartupComplete } from './lifecycle'; // Workers are imported to ensure they're started with the process import './workers'; @@ -98,12 +98,7 @@ async function localShutdown(): Promise { localShuttingDown = true; logger.info('Shutting down local server...'); try { - await Promise.all([ - pyQueue.close(), - otherQueue.close(), - pyQueueEvents.close(), - otherQueueEvents.close() - ]); + await closeQueueConnections(); try { await shutdownTelemetry(); } catch (telemetryError) { diff --git a/service/src/queue.ts b/service/src/queue.ts index 91f97c36..54fea308 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -7,7 +7,15 @@ import type * as tls from 'tls'; import type * as t from './types'; import { Jobs } from './enum'; import { env } from './config'; -import { queueNamesForExecutionProfile } from './execution-profile'; +import { + queueNameForExecution, + queueNamesForExecutionProfile, +} from './execution-profile'; +import type { + ExecutionProfile, + ExecutionProfileSource, + SandboxBackendName, +} from './execution-profile'; import logger from './logger'; import { redisKeepAliveOptions } from './redis-options'; import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; @@ -60,18 +68,52 @@ const connection = new IORedis({ const queueNames = queueNamesForExecutionProfile( env.EXECUTION_PROFILE, env.EXECUTION_PROFILE_SOURCE, + env.SANDBOX_BACKEND, ); -const pyQueue = new Queue(queueNames.python, { connection }); -const otherQueue = new Queue(queueNames.other, { connection }); +export interface QueueBinding { + queue: Queue; + events: QueueEvents; + language: 'python' | 'bash'; +} + +const queueResources = new Map< + string, + { queue: Queue; events: QueueEvents } +>(); -const pyQueueEvents = new QueueEvents(queueNames.python, { connection }); -const otherQueueEvents = new QueueEvents(queueNames.other, { connection }); +function getQueueResources( + name: string, +): { queue: Queue; events: QueueEvents } { + const existing = queueResources.get(name); + if (existing != null) return existing; + + const queue = new Queue(name, { connection }); + const events = new QueueEvents(name, { connection }); + setMaxListeners(0, queue, events); + const resources = { queue, events }; + queueResources.set(name, resources); + return resources; +} + +export function getExecutionQueueBinding( + language: 'python' | 'bash', + backend: SandboxBackendName | undefined = env.SANDBOX_BACKEND, + profile: ExecutionProfile = env.EXECUTION_PROFILE, + source: ExecutionProfileSource = env.EXECUTION_PROFILE_SOURCE, +): QueueBinding { + const name = queueNameForExecution( + language, + profile, + source, + backend, + ); + return { ...getQueueResources(name), language }; +} + +const { queue: pyQueue, events: pyQueueEvents } = getQueueResources(queueNames.python); +const { queue: otherQueue, events: otherQueueEvents } = getQueueResources(queueNames.other); const queueMetricStates = ['waiting', 'active', 'delayed'] as const; -const queueMetricSources = [ - { name: queueNames.python, queue: pyQueue }, - { name: queueNames.other, queue: otherQueue }, -] as const; const QUEUE_METRICS_TIMEOUT_MS = 1000; async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { @@ -90,7 +132,7 @@ async function withTimeout(promise: Promise, timeoutMs: number, message: s } registerBullmqQueueMetricsCollector(async () => { - await Promise.all(queueMetricSources.map(async ({ name, queue }) => { + await Promise.all([...queueResources.entries()].map(async ([name, { queue }]) => { try { const counts = await withTimeout( queue.getJobCounts(...queueMetricStates), @@ -116,4 +158,13 @@ registerBullmqQueueMetricsCollector(async () => { * BullMQ coordination objects. */ setMaxListeners(0, pyQueue, otherQueue, pyQueueEvents, otherQueueEvents); +export async function closeQueueConnections(): Promise { + await Promise.all( + [...queueResources.values()].flatMap(({ queue, events }) => [ + queue.close(), + events.close(), + ]), + ); +} + export { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection }; diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts new file mode 100644 index 00000000..c1b9e2c9 --- /dev/null +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, test } from 'bun:test'; + +import type { SandboxExecuteContext, SandboxTransportRequest } from './types'; +import type { RedisBridgeStore } from '../bridge/store'; + +import { BridgeStoreError } from '../bridge/store'; +import { RemoteBridgeSandboxBackend } from './remote-bridge'; + +function request(): SandboxTransportRequest { + return { + body: { language: 'bash' } as never, + headers: {}, + }; +} + +function context(): SandboxExecuteContext { + return { + executionId: 'execution-1', + language: 'bash', + isSynthetic: false, + signal: new AbortController().signal, + tenantId: 'tenant-1', + bridgeWorkerId: 'user-vm', + runtimeSessionMode: 'strict', + }; +} + +describe('RemoteBridgeSandboxBackend', () => { + test('dispatches a dynamically selected worker with a required tenant binding', async () => { + let dispatched: Parameters[0] | undefined; + const store = { + dispatch: async ( + args: Parameters[0], + ): ReturnType => { + dispatched = args; + return { + protocolVersion: 1 as const, + generation: 1, + leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', + status: 'fulfilled' as const, + result: { + session_id: 'session-1', + language: 'bash', + version: '5.2.0', + files: [], + }, + }; + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + + await expect(backend.execute(request(), context())).resolves.toMatchObject({ + session_id: 'session-1', + }); + expect(dispatched).toMatchObject({ + workerId: 'user-vm', + tenantId: 'tenant-1', + requireTenantBinding: true, + }); + }); + + test('maps tenant authorization rejection to a bridge backend error', async () => { + const store = { + dispatch: async (): ReturnType => { + throw new BridgeStoreError('WORKER_UNAUTHORIZED', 'private tenant detail'); + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + + await expect(backend.execute(request(), context())).rejects.toMatchObject({ + code: 'BRIDGE_WORKER_UNAUTHORIZED', + }); + }); + + test('keeps an explicitly selected singleton on its unbound compatibility route', async () => { + let dispatched: Parameters[0] | undefined; + const store = { + dispatch: async ( + args: Parameters[0], + ): ReturnType => { + dispatched = args; + return { + protocolVersion: 1 as const, + generation: 1, + leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', + status: 'fulfilled' as const, + result: { + session_id: 'session-1', + language: 'bash', + version: '5.2.0', + files: [], + }, + }; + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend( + store, + 'deployment-worker', + false, + ); + + await backend.execute(request(), { + ...context(), + bridgeWorkerId: 'deployment-worker', + }); + + expect(dispatched).toMatchObject({ + workerId: 'deployment-worker', + requireTenantBinding: false, + }); + }); + + test('requires a binding for the selected default worker in dynamic mode', async () => { + let dispatched: Parameters[0] | undefined; + const store = { + dispatch: async ( + args: Parameters[0], + ): ReturnType => { + dispatched = args; + return { + protocolVersion: 1 as const, + generation: 1, + leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', + status: 'fulfilled' as const, + result: { + session_id: 'session-1', + language: 'bash', + version: '5.2.0', + files: [], + }, + }; + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend( + store, + 'deployment-worker', + true, + ); + + await backend.execute(request(), { + ...context(), + bridgeWorkerId: 'deployment-worker', + }); + + expect(dispatched).toMatchObject({ + workerId: 'deployment-worker', + requireTenantBinding: true, + }); + }); +}); diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index a30ab748..b1a94ae0 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -15,15 +15,17 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { readonly name = 'remote-bridge' as const; constructor( - private readonly store: RedisBridgeStore = bridgeStore, + private readonly store: Pick = bridgeStore, private readonly workerId: string = env.BRIDGE_WORKER_ID, + private readonly dynamicWorkers: boolean = env.BRIDGE_DYNAMIC_WORKERS, ) {} async execute( req: SandboxTransportRequest, ctx: SandboxExecuteContext, ): Promise { - if (!this.workerId) { + const workerId = ctx.bridgeWorkerId ?? this.workerId; + if (workerId.length === 0) { throw new SandboxBackendError( 'BRIDGE_WORKER_OFFLINE', 'No bridge worker is configured', @@ -32,7 +34,11 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { const sessionResultFinalizer = ctx.sessionResultFinalizer; try { const settlement = await this.store.dispatch({ - workerId: this.workerId, + workerId, + tenantId: ctx.tenantId, + requireTenantBinding: + ctx.bridgeWorkerId != null && + (this.dynamicWorkers || ctx.bridgeWorkerId !== this.workerId), body: req.body, headers: req.headers, runtimeSessionId: ctx.runtimeSessionId, @@ -57,6 +63,13 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { return settlement.result as SandboxRawResponse; } catch (error) { if (!(error instanceof BridgeStoreError)) throw error; + if (error.code === 'WORKER_UNAUTHORIZED') { + throw new SandboxBackendError( + 'BRIDGE_WORKER_UNAUTHORIZED', + error.message, + error, + ); + } if (error.code === 'WORKER_BUSY') { throw new SandboxBackendError( 'BRIDGE_WORKER_BUSY', diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index 15bb6942..64e5b6e0 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -37,6 +37,8 @@ export interface SandboxExecuteContext { deadlineAtMs?: number; tenantId?: string; canonicalUserId?: string; + /** Trusted API-selected outbound worker. Presence requires a tenant-bound credential. */ + bridgeWorkerId?: string; /** Absent ⇒ stateless execution (no runtime session affinity). */ runtimeSessionId?: string; runtimeSessionMode: t.RuntimeSessionMode; @@ -64,6 +66,7 @@ export interface SandboxBackend { export type SandboxBackendErrorCode = | 'RUNTIME_SESSION_BUSY' | 'BRIDGE_WORKER_OFFLINE' + | 'BRIDGE_WORKER_UNAUTHORIZED' | 'BRIDGE_WORKER_BUSY' | 'BRIDGE_EXECUTION_FAILED' | 'BRIDGE_DEADLINE_EXCEEDED' diff --git a/service/src/secure-startup.test.ts b/service/src/secure-startup.test.ts index 85f0e854..78a49497 100644 --- a/service/src/secure-startup.test.ts +++ b/service/src/secure-startup.test.ts @@ -3,6 +3,7 @@ import { env } from './config'; import { validateApiBridgePolicy, validateApiHardenedConfig, + validateApiSandboxBackendPolicy, validateEgressGatewayHardenedConfig, validateExecutionProfilePolicy, validateSandboxBackendPolicy, @@ -15,6 +16,7 @@ const saved = { executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: env.SANDBOX_BACKEND, + bridgeDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, bridgeWorkerId: env.BRIDGE_WORKER_ID, bridgeAuthMode: env.BRIDGE_AUTH_MODE, bridgeToken: env.BRIDGE_TOKEN, @@ -56,6 +58,7 @@ function restore(): void { env.EXECUTION_PROFILE = saved.executionProfile; env.EXECUTION_PROFILE_SOURCE = saved.executionProfileSource; env.SANDBOX_BACKEND = saved.sandboxBackend; + env.BRIDGE_DYNAMIC_WORKERS = saved.bridgeDynamicWorkers; env.BRIDGE_WORKER_ID = saved.bridgeWorkerId; env.BRIDGE_AUTH_MODE = saved.bridgeAuthMode; env.BRIDGE_TOKEN = saved.bridgeToken; @@ -313,6 +316,36 @@ describe('sandbox backend policy', () => { expect(() => validateSandboxBackendPolicy()).not.toThrow(); }); + test('allows dynamic-only paired workers without a configured default', () => { + env.SANDBOX_BACKEND = 'remote-bridge'; + env.RUNTIME_SESSION_MODE = 'strict'; + env.PTC_MODE = 'replay'; + env.BRIDGE_DYNAMIC_WORKERS = true; + env.BRIDGE_WORKER_ID = ''; + env.BRIDGE_TOKEN = 'development-bridge-token'; + env.BRIDGE_AUTH_MODE = 'static'; + + expect(() => validateSandboxBackendPolicy()).toThrow( + 'CODEAPI_BRIDGE_AUTH_MODE=paired', + ); + + env.BRIDGE_AUTH_MODE = 'paired'; + expect(() => validateSandboxBackendPolicy()).not.toThrow(); + }); + + test('requires paired dynamic worker auth in an API-only process', () => { + env.SANDBOX_BACKEND = 'http'; + env.BRIDGE_DYNAMIC_WORKERS = true; + env.BRIDGE_AUTH_MODE = 'static'; + + expect(() => validateApiSandboxBackendPolicy()).toThrow( + 'CODEAPI_BRIDGE_AUTH_MODE=paired', + ); + + env.BRIDGE_AUTH_MODE = 'paired'; + expect(() => validateApiSandboxBackendPolicy()).not.toThrow(); + }); + test('hardened remote bridge requires replay PTC, paired auth, and a strong administrator token', () => { env.SANDBOX_BACKEND = 'remote-bridge'; env.RUNTIME_SESSION_MODE = 'affinity'; diff --git a/service/src/secure-startup.ts b/service/src/secure-startup.ts index f6f13390..c1740790 100644 --- a/service/src/secure-startup.ts +++ b/service/src/secure-startup.ts @@ -65,11 +65,13 @@ export function validateApiBridgePolicy(): void { env.SANDBOX_BACKEND === 'remote-bridge' || env.BRIDGE_AUTH_MODE === 'paired'; if (bridgeEnabled) { - requireValue('CODEAPI_BRIDGE_WORKER_ID', env.BRIDGE_WORKER_ID); - if (!isValidBridgeWorkerId(env.BRIDGE_WORKER_ID ?? '')) { - throw new SecureStartupConfigError( - 'CODEAPI_BRIDGE_WORKER_ID must match the bridge worker ID format', - ); + if (!env.BRIDGE_DYNAMIC_WORKERS) { + requireValue('CODEAPI_BRIDGE_WORKER_ID', env.BRIDGE_WORKER_ID); + if (!isValidBridgeWorkerId(env.BRIDGE_WORKER_ID ?? '')) { + throw new SecureStartupConfigError( + 'CODEAPI_BRIDGE_WORKER_ID must match the bridge worker ID format', + ); + } } requireValue('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); } @@ -144,11 +146,20 @@ export function validateExecutionProfilePolicy(options: { } } +export function validateApiSandboxBackendPolicy(): void { + if (env.BRIDGE_DYNAMIC_WORKERS && env.BRIDGE_AUTH_MODE !== 'paired') { + throw new SecureStartupConfigError( + 'Dynamic remote bridge workers require CODEAPI_BRIDGE_AUTH_MODE=paired', + ); + } +} + /** * Backend-selection policy. Unlike the hardened-mode validators, this runs * unconditionally: a misconfigured backend must never half-start. */ export function validateSandboxBackendPolicy(): void { + validateApiSandboxBackendPolicy(); if (env.RUNTIME_SESSION_MODE !== 'stateless' && env.SANDBOX_BACKEND === 'http') { throw new SecureStartupConfigError( `CODEAPI_RUNTIME_SESSION_MODE=${env.RUNTIME_SESSION_MODE} requires ` diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index eade27fb..baa350e6 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -2,11 +2,15 @@ import axios from 'axios'; import { nanoid } from 'nanoid'; import { Router } from 'express'; import type { Response } from 'express'; -import type { Queue, QueueEvents } from 'bullmq'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { executionLimiter } from '../middleware/limits'; -import { pyQueue, pyQueueEvents, otherQueue, otherQueueEvents, connection } from '../queue'; +import { + pyQueue, + pyQueueEvents, + connection, + getExecutionQueueBinding, +} from '../queue'; import { createProgrammaticPayload, extractPendingFromStdout } from '../preamble'; import { findBashToolNameCollision } from '../preamble-bash'; import type { LCTool } from '../preamble'; @@ -25,6 +29,8 @@ import { } from '../metrics'; import { Jobs } from '../enum'; import { env, jobCompletionWaitTimeoutMs } from '../config'; +import { resolveQueuedSandboxBackend } from '../execution-profile'; +import { publicExecutionFailure } from '../utils'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, @@ -35,7 +41,15 @@ import { import { findUnregisteredToolCall } from '../tool-scope'; import { summarizeRequestedFiles } from '../execution-log'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; -import { buildReplayExecutionState } from './programmatic-state'; +import { + buildReplayExecutionState, + resolveReplayStateSandboxBackend, +} from './programmatic-state'; +import { + BridgeWorkerSelectionError, + CODEAPI_BRIDGE_WORKER_HEADER, + resolveBridgeWorkerSelection, +} from '../bridge/selection'; import logger from '../logger'; import { type ExecutionState, @@ -328,19 +342,6 @@ async function waitForExecutionState( // Replay mode helpers // --------------------------------------------------------------------------- -interface QueueBinding { - queue: Queue; - events: QueueEvents; - language: 'python' | 'bash'; -} - -function pickQueue(language: 'python' | 'bash'): QueueBinding { - if (language === 'bash') { - return { queue: otherQueue, events: otherQueueEvents, language: 'bash' }; - } - return { queue: pyQueue, events: pyQueueEvents, language: 'python' }; -} - function buildReplayPayload( req: t.AuthenticatedRequest, state: ExecutionState, @@ -396,7 +397,21 @@ async function runReplayIteration( }); } - const { queue, events, language } = pickQueue(state.language ?? 'python'); + const replayBackend = + state.sandboxBackend ?? + (state.bridgeWorkerId != null + ? 'remote-bridge' + : resolveQueuedSandboxBackend( + env.EXECUTION_PROFILE, + env.SANDBOX_BACKEND, + env.EXECUTION_PROFILE_SOURCE, + )); + const { queue, events, language } = getExecutionQueueBinding( + state.language ?? 'python', + replayBackend, + state.executionProfile ?? env.EXECUTION_PROFILE, + state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, + ); const job = await queue.add(Jobs.execute, { code: state.userCode ?? '', userId, @@ -407,7 +422,9 @@ async function runReplayIteration( executionId: state.execution_id, tenantId: state.tenantId, canonicalUserId: state.canonicalUserId, - executionProfile: env.EXECUTION_PROFILE, + executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, + sandboxBackend: replayBackend, + ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, executionManifestClaims: sandboxSecurity.executionManifestClaims, @@ -439,9 +456,10 @@ async function handleReplayInitial( params: { apiKeyId: string; userId: string; + bridgeWorkerId?: string; }, ): Promise { - const { apiKeyId, userId } = params; + const { apiKeyId, userId, bridgeWorkerId } = params; const { code, tools, @@ -560,6 +578,15 @@ async function handleReplayInitial( isPyPlot, timeout, language, + bridgeWorkerId, + executionProfile: env.EXECUTION_PROFILE, + executionProfileSource: env.EXECUTION_PROFILE_SOURCE, + sandboxBackend: resolveReplayStateSandboxBackend({ + executionProfile: env.EXECUTION_PROFILE, + executionProfileSource: env.EXECUTION_PROFILE_SOURCE, + apiSandboxBackend: env.SANDBOX_BACKEND, + bridgeWorkerId, + }), }); /** Replay mode persists the full request (`userCode` + `tools` + `files`) * inside `ExecutionState` so continuations can re-enqueue without the @@ -832,7 +859,8 @@ async function runAndRespond( logger.error('Replay iteration failed', { execution_id: state.execution_id, err }); await cleanupExecution(state.execution_id, 'replay'); if (!isDisconnected()) { - const message = (err as Error).message; + const publicFailure = publicExecutionFailure(err); + const message = publicFailure?.body.message ?? (err as Error).message; res.status(200).json({ status: 'error', error: message !== '' ? message : 'Sandbox execution failed', @@ -1023,6 +1051,26 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } = req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; + let bridgeWorkerId: string | undefined; + if (continuation_token == null || continuation_token === '') { + try { + const bridgeSelection = resolveBridgeWorkerSelection({ + backend: env.SANDBOX_BACKEND, + configuredWorkerId: env.BRIDGE_WORKER_ID, + dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), + trustedWorkerId: principal.codeWorkerId, + }); + bridgeWorkerId = bridgeSelection?.explicit === true + ? bridgeSelection.workerId + : undefined; + } catch (error) { + if (error instanceof BridgeWorkerSelectionError) { + return res.status(error.status).json({ error: error.message }); + } + throw error; + } + } if ( requestedLanguage !== undefined && @@ -1079,9 +1127,9 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR }); } if (env.PTC_MODE === 'replay') { - return await handleReplayInitial(req, res, { apiKeyId, userId }); + return await handleReplayInitial(req, res, { apiKeyId, userId, bridgeWorkerId }); } - return await handleBlocking(req, res, { apiKeyId, userId }); + return await handleBlocking(req, res, { apiKeyId, userId, bridgeWorkerId }); } catch (err) { logger.error(`[${INSTANCE_ID}] Programmatic routing error:`, err); if (!res.headersSent) { @@ -1099,9 +1147,9 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR async function handleBlocking( req: t.AuthenticatedRequest, res: Response, - params: { apiKeyId: string; userId: string }, + params: { apiKeyId: string; userId: string; bridgeWorkerId?: string }, ): Promise> { - const { apiKeyId, userId } = params; + const { apiKeyId, userId, bridgeWorkerId } = params; const { code, tools, @@ -1282,6 +1330,7 @@ async function handleBlocking( principalSource: identity.principalSource, authContextHash: identity.authContextHash, apiKeyId, + bridgeWorkerId, startTime: Date.now(), lastActivity: Date.now(), mode: 'blocking', @@ -1378,6 +1427,12 @@ async function handleBlocking( tenantId: identity.storageNamespace, canonicalUserId: identity.canonicalUserId, executionProfile: env.EXECUTION_PROFILE, + sandboxBackend: resolveQueuedSandboxBackend( + env.EXECUTION_PROFILE, + env.SANDBOX_BACKEND, + env.EXECUTION_PROFILE_SOURCE, + ), + ...(bridgeWorkerId != null ? { bridgeWorkerId } : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, executionManifestClaims: sandboxSecurity.executionManifestClaims, diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index f710668a..fc84d8f8 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -1,7 +1,10 @@ import { describe, expect, test } from 'bun:test'; import type { CodeApiAuthContext, RequestFile } from '../types'; import type { LCTool } from '../preamble'; -import { buildReplayExecutionState } from './programmatic-state'; +import { + buildReplayExecutionState, + resolveReplayStateSandboxBackend, +} from './programmatic-state'; const TOOLS = [ { @@ -22,7 +25,9 @@ const FILES = [ }, ] as RequestFile[]; -function build(overrides: Partial[0]> = {}) { +function build( + overrides: Partial[0]> = {}, +): ReturnType { return buildReplayExecutionState({ executionId: 'exec_123', sessionId: 'session_123', @@ -35,12 +40,35 @@ function build(overrides: Partial[0 isPyPlot: false, timeout: 300000, language: 'python', + executionProfile: 'default', + executionProfileSource: 'inferred', now: 1778250000000, ...overrides, }); } describe('buildReplayExecutionState', () => { + test('persists the resolved queue consumer backend for split stateful deployments', () => { + expect( + resolveReplayStateSandboxBackend({ + executionProfile: 'stateful', + executionProfileSource: 'explicit', + apiSandboxBackend: 'http', + }), + ).toBe('lambda-microvm'); + }); + + test('pins bridge replay state to the remote bridge backend', () => { + expect( + resolveReplayStateSandboxBackend({ + executionProfile: 'stateful', + executionProfileSource: 'explicit', + apiSandboxBackend: 'http', + bridgeWorkerId: 'worker-1', + }), + ).toBe('remote-bridge'); + }); + test('persists canonical LibreChat auth context for replay continuations', () => { const authContext: CodeApiAuthContext = { userId: 'user_canonical', @@ -52,7 +80,13 @@ describe('buildReplayExecutionState', () => { authContextHash: 'hash_123', }; - const state = build({ authContext }); + const state = build({ + authContext, + bridgeWorkerId: 'code-user_123', + sandboxBackend: 'remote-bridge', + executionProfile: 'stateful', + executionProfileSource: 'explicit', + }); expect(state).toMatchObject({ execution_id: 'exec_123', @@ -67,6 +101,10 @@ describe('buildReplayExecutionState', () => { principalSource: 'openid_reuse', authContextHash: 'hash_123', apiKeyId: 'key_legacy', + bridgeWorkerId: 'code-user_123', + sandboxBackend: 'remote-bridge', + executionProfile: 'stateful', + executionProfileSource: 'explicit', mode: 'replay', userCode: 'print("hello")', tools: TOOLS, diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index f469606f..25571fed 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -2,6 +2,26 @@ import type * as t from '../types'; import type { LCTool } from '../preamble'; import type { ExecutionState } from './replay-state'; import { buildExecutionIdentity, type ExecutionIdentity } from '../execution-identity'; +import { resolveQueuedSandboxBackend } from '../execution-profile'; +import type { + ExecutionProfile, + ExecutionProfileSource, + SandboxBackendName, +} from '../execution-profile'; + +export function resolveReplayStateSandboxBackend(params: { + executionProfile: ExecutionProfile; + executionProfileSource: ExecutionProfileSource; + apiSandboxBackend: SandboxBackendName; + bridgeWorkerId?: string; +}): SandboxBackendName | undefined { + if (params.bridgeWorkerId != null) return 'remote-bridge'; + return resolveQueuedSandboxBackend( + params.executionProfile, + params.apiSandboxBackend, + params.executionProfileSource, + ); +} export interface BuildReplayExecutionStateParams { executionId: string; @@ -17,6 +37,10 @@ export interface BuildReplayExecutionStateParams { isPyPlot: boolean; timeout: number; language: 'python' | 'bash'; + bridgeWorkerId?: string; + sandboxBackend?: SandboxBackendName; + executionProfile: ExecutionProfile; + executionProfileSource: ExecutionProfileSource; now?: number; } @@ -41,6 +65,10 @@ export function buildReplayExecutionState( principalSource: identity.principalSource, authContextHash: identity.authContextHash, apiKeyId: params.apiKeyId, + bridgeWorkerId: params.bridgeWorkerId, + sandboxBackend: params.sandboxBackend, + executionProfile: params.executionProfile, + executionProfileSource: params.executionProfileSource, startTime: now, lastActivity: now, mode: 'replay', diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 562e06dd..4d2a0501 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -24,6 +24,11 @@ import { nanoid } from 'nanoid'; import type { Redis } from 'ioredis'; import type * as t from '../types'; import type { LCTool } from '../preamble'; +import type { + ExecutionProfile, + ExecutionProfileSource, + SandboxBackendName, +} from '../execution-profile'; import { connection } from '../queue'; import { env } from '../config'; import { internalServiceHeaders } from '../internal-service-auth'; @@ -109,6 +114,14 @@ export interface ExecutionState { * after one `EXECUTION_STATE_TTL` window post a trusted-source * apiKeyId invariant. */ apiKeyId?: string; + /** Authenticated worker selection retained across every replay iteration. */ + bridgeWorkerId?: string; + /** Original queue/backend target retained across replay continuations. */ + sandboxBackend?: SandboxBackendName; + /** Original producer profile retained so continuations use the same queue. */ + executionProfile?: ExecutionProfile; + /** Original profile source retained because inferred profiles use legacy queues. */ + executionProfileSource?: ExecutionProfileSource; startTime: number; /** * Wall-clock ms of the last interaction that advanced this execution (initial diff --git a/service/src/service/router.ts b/service/src/service/router.ts index f355c2dd..542fe4c0 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -25,7 +25,13 @@ import { Jobs, Languages } from '../enum'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; import { createUploadSessionRegistrar } from './upload-session'; import { prepareSandboxJobSecurity } from '../sandbox-egress'; +import { + BridgeWorkerSelectionError, + CODEAPI_BRIDGE_WORKER_HEADER, + resolveBridgeWorkerSelection, +} from '../bridge/selection'; import logger from '../logger'; +import { resolveQueuedSandboxBackend } from '../execution-profile'; const { INSTANCE_ID } = env; const JOB_COMPLETION_WAIT_TIMEOUT_MS = jobCompletionWaitTimeoutMs( @@ -140,6 +146,25 @@ router.post('/exec', executionLimiter, async (req: t.AuthenticatedRequest, res) return res.status(400).json({ error: `Unsupported language: ${rawLang}` }); } + let bridgeWorkerId: string | undefined; + try { + const bridgeSelection = resolveBridgeWorkerSelection({ + backend: env.SANDBOX_BACKEND, + configuredWorkerId: env.BRIDGE_WORKER_ID, + dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), + trustedWorkerId: principal.codeWorkerId, + }); + bridgeWorkerId = bridgeSelection?.explicit === true + ? bridgeSelection.workerId + : undefined; + } catch (error) { + if (error instanceof BridgeWorkerSelectionError) { + return res.status(error.status).json({ error: error.message }); + } + throw error; + } + let runtimeSessionId: string | undefined; try { runtimeSessionId = resolveRuntimeSessionIdForExecRequest({ @@ -247,6 +272,12 @@ router.post('/exec', executionLimiter, async (req: t.AuthenticatedRequest, res) tenantId: identity.storageNamespace, canonicalUserId: identity.canonicalUserId, executionProfile: env.EXECUTION_PROFILE, + sandboxBackend: resolveQueuedSandboxBackend( + env.EXECUTION_PROFILE, + env.SANDBOX_BACKEND, + env.EXECUTION_PROFILE_SOURCE, + ), + ...(bridgeWorkerId != null ? { bridgeWorkerId } : {}), ...(runtimeSessionId != null ? { runtimeSessionId } : {}), runtimeSessionMode, executionManifestClaims: sandboxSecurity.executionManifestClaims, diff --git a/service/src/types/service.ts b/service/src/types/service.ts index a642dda3..555056d5 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -3,7 +3,7 @@ import type { Request } from 'express'; import type { ExecutionManifestClaims } from '../execution-manifest'; import type { ExecutionIdentity } from '../execution-identity'; import type { CodeApiPrincipal } from '../auth/principal'; -import type { ExecutionProfile } from '../execution-profile'; +import type { ExecutionProfile, SandboxBackendName } from '../execution-profile'; import { Jobs } from '@/enum/service'; /** @@ -251,8 +251,12 @@ export type JobData = { executionId?: string; tenantId?: string; canonicalUserId?: string; + /** Trusted dynamic outbound worker selection. */ + bridgeWorkerId?: string; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; + /** Required sandbox transport. Optional only for jobs queued before fencing. */ + sandboxBackend?: SandboxBackendName; /** * Server-derived runtime session identity. Absence is stateless unless * strict mode requires it; explicit exemptions document intentional gaps. diff --git a/service/src/utils.test.ts b/service/src/utils.test.ts index 879736c5..86e67a9c 100644 --- a/service/src/utils.test.ts +++ b/service/src/utils.test.ts @@ -48,7 +48,7 @@ describe('isValidResourceId (heterogeneous resource identifiers)', () => { expect(isValidResourceId('682f49b90f07376815c38ef2')).toBe(true); }); - test("accepts 17-char `agent_` slug", () => { + test('accepts 17-char `agent_` slug', () => { expect(isValidResourceId('agent_abc12345678')).toBe(true); }); @@ -145,15 +145,39 @@ describe('sandbox error formatting', () => { }); }); - test('maps remote bridge failures without exposing worker details', () => { + test('maps bridge authorization and availability failures without leaking worker details', () => { + const unauthorized = publicExecutionFailure( + new Error('BRIDGE_WORKER_UNAUTHORIZED: Worker private-vm belongs to tenant-secret'), + ); + expect(unauthorized).toEqual({ + status: 403, + body: { + error: 'bridge_worker_unauthorized', + message: 'Code environment is not authorized for this tenant', + }, + }); + expect(JSON.stringify(unauthorized)).not.toContain('private-vm'); + expect(JSON.stringify(unauthorized)).not.toContain('tenant-secret'); + + expect( + publicExecutionFailure( + new Error('BRIDGE_WORKER_OFFLINE: Worker private-vm has not checked in'), + ), + ).toEqual({ + status: 503, + body: { + error: 'bridge_worker_offline', + message: 'Code environment is offline', + }, + }); + const cases = [ - ['BRIDGE_WORKER_OFFLINE', 503, 'Remote code worker is unavailable'], - ['BRIDGE_WORKER_BUSY', 409, 'Remote code worker is busy'], - ['BRIDGE_EXECUTION_FAILED', 502, 'Remote code execution failed'], + ['BRIDGE_WORKER_BUSY', 409, 'Code environment is busy'], + ['BRIDGE_EXECUTION_FAILED', 502, 'Code environment execution failed'], [ 'BRIDGE_DEADLINE_EXCEEDED', 504, - 'Remote code execution deadline exceeded', + 'Code environment execution timed out', ], ] as const; for (const [code, status, message] of cases) { @@ -177,7 +201,7 @@ describe('sandbox error formatting', () => { status: 502, body: { error: 'bridge_execution_failed', - message: 'Remote code execution failed', + message: 'Code environment execution failed', }, }); expect(JSON.stringify(failure)).not.toContain('private second line'); diff --git a/service/src/utils.ts b/service/src/utils.ts index e47a7d4c..aae2d05e 100644 --- a/service/src/utils.ts +++ b/service/src/utils.ts @@ -137,6 +137,7 @@ export function publicExecutionFailure(error: unknown): { status: number; body: const code = backendMatch[1]; const statuses: Record = { RUNTIME_SESSION_BUSY: 409, + BRIDGE_WORKER_UNAUTHORIZED: 403, BRIDGE_WORKER_OFFLINE: 503, BRIDGE_WORKER_BUSY: 409, BRIDGE_EXECUTION_FAILED: 502, @@ -151,10 +152,11 @@ export function publicExecutionFailure(error: unknown): { status: number; body: const status = statuses[code] ?? (sessionInputFailure ? 500 : 503); const publicMessages: Record = { RUNTIME_SESSION_BUSY: 'Runtime session is busy', - BRIDGE_WORKER_OFFLINE: 'Remote code worker is unavailable', - BRIDGE_WORKER_BUSY: 'Remote code worker is busy', - BRIDGE_EXECUTION_FAILED: 'Remote code execution failed', - BRIDGE_DEADLINE_EXCEEDED: 'Remote code execution deadline exceeded', + BRIDGE_WORKER_UNAUTHORIZED: 'Code environment is not authorized for this tenant', + BRIDGE_WORKER_OFFLINE: 'Code environment is offline', + BRIDGE_WORKER_BUSY: 'Code environment is busy', + BRIDGE_EXECUTION_FAILED: 'Code environment execution failed', + BRIDGE_DEADLINE_EXCEEDED: 'Code environment execution timed out', MICROVM_LAUNCH_FAILED: 'Sandbox launch failed', MICROVM_LAUNCH_THROTTLED: 'Sandbox capacity is temporarily unavailable', MICROVM_UNHEALTHY: 'Sandbox runtime is unavailable', diff --git a/service/src/workers.ts b/service/src/workers.ts index ed2a46dd..d2048dc8 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -17,7 +17,10 @@ import { isSyntheticPrincipalSource } from './auth/synthetic'; import { withSpan, withTraceContext } from './telemetry'; import { workerDeadlineFailure } from './worker-error'; import logger from './logger'; -import { validateQueuedExecutionProfile } from './execution-profile'; +import { + validateQueuedExecutionProfile, + validateQueuedSandboxBackend, +} from './execution-profile'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; @@ -38,7 +41,7 @@ async function processJob(job: t.ExecuteJob): Promise { } async function processJobInner(job: t.ExecuteJob): Promise { - const { code, payload, isPyPlot } = job.data; + const { payload, isPyPlot } = job.data; const isSyntheticJob = job.data.isSynthetic === true || isSyntheticPrincipalSource(job.data.principalSource); const language = payload?.language ?? 'unknown'; const endTimer = jobProcessingDuration.startTimer({ language }); @@ -60,6 +63,11 @@ async function processJobInner(job: t.ExecuteJob): Promise { throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); } validateQueuedExecutionProfile(job.data.executionProfile, env.EXECUTION_PROFILE); + validateQueuedSandboxBackend( + job.data.sandboxBackend, + env.SANDBOX_BACKEND, + job.data.bridgeWorkerId, + ); let sandboxPayload = payload; let executionManifestClaims = job.data.executionManifestClaims; let egressGrantToken = job.data.egressGrantToken; @@ -139,6 +147,7 @@ async function processJobInner(job: t.ExecuteJob): Promise { deadlineAtMs, tenantId: job.data.tenantId, canonicalUserId: job.data.canonicalUserId, + bridgeWorkerId: job.data.bridgeWorkerId, runtimeSessionId: runtimeSession.runtimeSessionId, runtimeSessionMode: runtimeSession.runtimeSessionMode, /* Stateful backends run this as a commit barrier after user code but From ea872fc153d7f03247204d4946abb62e1707faa7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 1 Sep 2026 13:34:40 -0400 Subject: [PATCH 006/116] =?UTF-8?q?=F0=9F=A7=AF=20fix:=20Fence=20Worker=20?= =?UTF-8?q?Pairing=20Revocation=20(#70)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add outbound stateful code bridge * test: cover remote bridge startup policy * fix: harden remote bridge lifecycle fencing * feat: add secure code worker pairing * fix: harden paired worker lifecycle * fix: require paired auth on hardened APIs * fix: harden bridge pairing startup policy * fix: preserve bridge fencing through pairing * fix: distinguish assignment settlement conflicts * feat: add principal-bound bridge workers * fix: authenticate principal worker routing * fix: fence bridge identity and backend routing * fix: fence bridge redemption and queue routing * fix: preserve legacy routing and assignment auth * fix: preserve long-lived bridge assignments * fix: persist replay queue backend * fix: fence bridge replay and credential rotation * fix: address principal worker review findings * fix: reconcile principal workers with bridge fencing * fix: fence principal worker lifecycle transitions * fix: fence bridge settlement ownership * fix: fence bridge leases to active principals * fix: invalidate pending worker pairings on revoke * fix: package bridge protocol in API image * fix: fence mixed-version pairing revocation * fix: redeem valid legacy pairing codes * fix: harden pairing rollout compatibility * fix: make pairing revocation atomic * fix: preserve pairing identity across rollouts * fix: reopen pairing cleanup after rollbacks * fix: bound legacy pairing migration scans * fix: retry interrupted pairing migrations * fix: harden pairing migration cleanup * fix: make pairing cleanup recoverable * fix: preserve pairing recovery across lifecycle rollout * fix: drain API pods before pairing rollback * fix: fence rollback reentry and pairing epochs * fix: close rollback verification gaps * fix: keep rollback drain on one cluster * fix: bound rollback recovery triggers --- .github/workflows/ci.yml | 3 + helm/codeapi/README.md | 26 + helm/codeapi/scripts/safe-pairing-rollback.sh | 190 ++++++ helm/codeapi/templates/api-deployment.yaml | 10 + helm/codeapi/values.yaml | 12 +- service/Dockerfile.api | 2 + service/src/bridge/pairing.test.ts | 585 +++++++++++++++++- service/src/bridge/pairing.ts | 470 ++++++++++---- tests/bridge_pairing_rollout.sh | 66 ++ 9 files changed, 1251 insertions(+), 113 deletions(-) create mode 100755 helm/codeapi/scripts/safe-pairing-rollback.sh create mode 100755 tests/bridge_pairing_rollout.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e22864..7ba2ef1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh + - name: Bridge pairing rollout safety + run: tests/bridge_pairing_rollout.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index 9b3efbb0..fcdef01d 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -53,6 +53,32 @@ platform rather than templated here: external ingress/service mesh, KEDA-style queue-depth autoscaling, and cloud-IAM secret delivery (the env hooks below cover all of them). +**Pairing-fence rollbacks.** Do not use a direct `helm rollback` from a chart +revision containing the bridge pairing fence to an older revision. Helm runs +rollback hooks from the target revision, so a pre-fence target cannot stop its +own old and new API replicas from overlapping. Use the chart's fail-closed +helper instead: + +```bash +helm/codeapi/scripts/safe-pairing-rollback.sh RELEASE REVISION NAMESPACE +``` + +The helper records an out-of-band rollback epoch, deletes the API HPA, scales +the live fenced API deployment to zero, verifies that the Deployment and every +matching pod have converged to zero, and only then invokes `helm rollback`. +When a fenced revision is deployed again, the epoch forces one fresh cleanup of +legacy pairing codes even if the original migration window has expired. This +causes an API outage by design. If rollback fails, the helper repeats the drain +after re-discovering every API Deployment and explicitly deletes any remaining +API pods, so a partially applied rollback cannot leave a mixed-version API +running. The operator running it needs permission to read/scale Deployments, +delete HPAs and pods, and create or update the rollback ConfigMap. +Pass the intended cluster context to both `kubectl` and `helm` before invoking +the helper; it rejects forwarded kubeconfig, context, identity, API-server, and +namespace flags and Helm-specific target environment overrides so the drain and +rollback cannot target different clusters. Termination signals during Helm +also trigger a final recovery drain before the helper exits. + **Execution profile.** By default this chart leaves `CODEAPI_EXECUTION_PROFILE` unset. Its bundled HTTP/stateless configuration is inferred as the AWS-free `default` profile and retains the existing diff --git a/helm/codeapi/scripts/safe-pairing-rollback.sh b/helm/codeapi/scripts/safe-pairing-rollback.sh new file mode 100755 index 00000000..c70bddc7 --- /dev/null +++ b/helm/codeapi/scripts/safe-pairing-rollback.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 RELEASE REVISION [NAMESPACE] [helm rollback flags...]" >&2 + exit 64 +} + +release=${1:-} +revision=${2:-} +namespace=${3:-default} +if [[ -z "$release" || ! "$revision" =~ ^[1-9][0-9]*$ ]]; then + usage +fi +shift $(( $# >= 3 ? 3 : $# )) + +if [[ ! "$release" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "invalid Helm release name: $release" >&2 + exit 64 +fi +if [[ ! "$namespace" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "invalid Kubernetes namespace: $namespace" >&2 + exit 64 +fi +for flag in "$@"; do + case "$flag" in + -n|-n?*|--namespace|--namespace=*|--kube-context|--kube-context=*|\ + --kubeconfig|--kubeconfig=*|--kube-apiserver|--kube-apiserver=*|\ + --kube-ca-file|--kube-ca-file=*|--kube-token|--kube-token=*|\ + --kube-tls-server-name|--kube-tls-server-name=*|\ + --kube-as-user|--kube-as-user=*|--kube-as-group|--kube-as-group=*|\ + --kube-insecure-skip-tls-verify|--kube-insecure-skip-tls-verify=*) + echo "refusing target-changing Helm rollback flag: $flag" >&2 + exit 64 + ;; + esac +done +for variable in \ + HELM_KUBEAPISERVER \ + HELM_KUBEASGROUPS \ + HELM_KUBEASUSER \ + HELM_KUBECAFILE \ + HELM_KUBECONTEXT \ + HELM_KUBEINSECURE_SKIP_TLS_VERIFY \ + HELM_KUBETLS_SERVER_NAME \ + HELM_KUBETOKEN \ + HELM_NAMESPACE; do + if [[ -n ${!variable:-} ]]; then + echo "refusing Helm target override from environment: $variable" >&2 + exit 64 + fi +done + +timeout=${CODEAPI_ROLLBACK_TIMEOUT:-10m} +selector="app.kubernetes.io/instance=${release},app.kubernetes.io/component=api" + +discover_api_deployments() { + local output + output=$(kubectl --namespace "$namespace" get deployment \ + --selector "$selector" --output name) || return + deployments=() + if [[ -n "$output" ]]; then + mapfile -t deployments <<< "$output" + fi +} + +list_api_pods() { + local output + output=$(kubectl --namespace "$namespace" get pod \ + --selector "$selector" --output name) || return + pods=() + if [[ -n "$output" ]]; then + mapfile -t pods <<< "$output" + fi +} + +discover_api_deployments +if (( ${#deployments[@]} != 1 )); then + echo "expected exactly one Code API deployment for $selector" >&2 + exit 1 +fi +deployment=${deployments[0]} + +fence=$(kubectl --namespace "$namespace" get "$deployment" \ + --output 'jsonpath={.spec.template.metadata.annotations.codeapi\.librechat\.ai/pairing-fence-version}') +if [[ -z "$fence" ]]; then + echo "refusing rollback: the live API deployment has no pairing fence" >&2 + exit 1 +fi + +deployment_name=${deployment#*/} +rollback_config_map=${deployment_name%-api}-pairing-rollback +rollback_epoch="$(date +%s)-${RANDOM}-${RANDOM}" + +echo "Recording pairing rollback epoch $rollback_epoch..." >&2 +kubectl --namespace "$namespace" create configmap "$rollback_config_map" \ + --from-literal="epoch=$rollback_epoch" --dry-run=client --output yaml | \ + kubectl --namespace "$namespace" apply --filename - + +drain_api() { + local pod_action=${1:-wait} + local replica_state desired current ready available updated + + # Helm may have partially installed a target with a different fullname. + # Resolve every matching API Deployment on each drain attempt. + discover_api_deployments + if (( ${#deployments[@]} == 0 )) && [[ "$pod_action" != delete ]]; then + echo "refusing rollback: no API deployment matched $selector" >&2 + return 1 + fi + + echo "Deleting API autoscalers before the rollback fence is lowered..." >&2 + kubectl --namespace "$namespace" delete horizontalpodautoscaler \ + --selector "$selector" --ignore-not-found --wait=true + + echo "Scaling the fenced API deployment to zero..." >&2 + for deployment in "${deployments[@]}"; do + kubectl --namespace "$namespace" scale "$deployment" --replicas=0 + kubectl --namespace "$namespace" rollout status "$deployment" \ + --timeout "$timeout" + done + + list_api_pods + if (( ${#pods[@]} > 0 )); then + if [[ "$pod_action" == delete ]]; then + kubectl --namespace "$namespace" delete pod \ + --selector "$selector" --wait=true --timeout "$timeout" + else + kubectl --namespace "$namespace" wait "${pods[@]}" \ + --for=delete --timeout "$timeout" + fi + fi + + # Relist immediately before Helm can lower the fence. This catches a new + # matching pod that appeared after the first snapshot. + discover_api_deployments + for deployment in "${deployments[@]}"; do + replica_state=$(kubectl --namespace "$namespace" get "$deployment" \ + --output 'jsonpath={.spec.replicas},{.status.replicas},{.status.readyReplicas},{.status.availableReplicas},{.status.updatedReplicas}') || return + IFS=, read -r desired current ready available updated <<< "$replica_state" + if [[ ${desired:-0} != 0 || ${current:-0} != 0 || ${ready:-0} != 0 || + ${available:-0} != 0 || ${updated:-0} != 0 ]]; then + echo "refusing rollback: API deployment did not converge to zero replicas" >&2 + return 1 + fi + done + list_api_pods + if (( ${#pods[@]} > 0 )); then + echo "refusing rollback: API pods appeared after the drain" >&2 + return 1 + fi +} + +drain_api wait + +echo "All fenced API pods are gone; starting Helm rollback..." >&2 +rollback_pid= +recover_interrupted_rollback() { + local exit_status=$1 + trap - HUP INT TERM + if [[ -n "$rollback_pid" ]]; then + kill -TERM "$rollback_pid" 2>/dev/null || true + wait "$rollback_pid" 2>/dev/null || true + fi + echo "Helm rollback interrupted; restoring the fail-closed API drain..." >&2 + set -e + drain_api delete + exit "$exit_status" +} +trap 'recover_interrupted_rollback 129' HUP +trap 'recover_interrupted_rollback 130' INT +trap 'recover_interrupted_rollback 143' TERM + +helm rollback "$release" "$revision" \ + --namespace "$namespace" --wait --wait-for-jobs --timeout "$timeout" "$@" & +rollback_pid=$! +set +e +wait "$rollback_pid" +rollback_status=$? +set -e +rollback_pid= +trap - HUP INT TERM + +if (( rollback_status == 0 )); then + exit 0 +else + echo "Helm rollback failed; restoring the fail-closed API drain..." >&2 + drain_api delete + exit "$rollback_status" +fi diff --git a/helm/codeapi/templates/api-deployment.yaml b/helm/codeapi/templates/api-deployment.yaml index bf5f91e0..69985db4 100644 --- a/helm/codeapi/templates/api-deployment.yaml +++ b/helm/codeapi/templates/api-deployment.yaml @@ -17,11 +17,15 @@ spec: {{- if not .Values.api.autoscaling.enabled }} replicas: {{ .Values.api.replicaCount }} {{- end }} + strategy: + {{- toYaml .Values.api.strategy | nindent 4 }} selector: matchLabels: {{- include "codeapi.api.selectorLabels" . | nindent 6 }} template: metadata: + annotations: + codeapi.librechat.ai/pairing-fence-version: "1" labels: {{- include "codeapi.api.selectorLabels" . | nindent 8 }} spec: @@ -55,6 +59,12 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH + valueFrom: + configMapKeyRef: + name: {{ include "codeapi.fullname" . }}-pairing-rollback + key: epoch + optional: true # Service URLs - name: FILE_SERVER_URL value: "http://{{ include "codeapi.fullname" . }}-file-server:{{ .Values.fileServer.service.port }}" diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 75abaaf7..ee997ce8 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -68,10 +68,20 @@ api: enabled: true replicaCount: 2 # Start with 2 API pods + # Pairing revocation relies on every serving replica honoring the Redis + # generation fence. Recreate prevents a pre-fence binary from redeeming an + # already-revoked code during the first rollout of paired bridge workers. + # Roll back to pre-fence revisions only with scripts/safe-pairing-rollback.sh. + strategy: + type: Recreate + rollingUpdate: null + image: repository: codeapi-api tag: latest - pullPolicy: IfNotPresent # Use Always in production + # Recreate is a security fence only if replacement pods cannot reuse a + # cached pre-fence image behind the mutable default tag. + pullPolicy: Always # Resource limits resources: diff --git a/service/Dockerfile.api b/service/Dockerfile.api index f1fdf9c8..2921401e 100644 --- a/service/Dockerfile.api +++ b/service/Dockerfile.api @@ -21,6 +21,7 @@ COPY service/src ./src COPY packages/code/src /packages/code/src COPY service/scripts ./scripts COPY shared /shared +COPY packages/code /packages/code COPY service/tsconfig.json ./ RUN bun build ./src/api-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' RUN bun build ./scripts/rehydrate-session-cache.ts --minify --outdir .build-migrations --target bun --external '@opentelemetry/*' @@ -49,6 +50,7 @@ COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY packages/code/src /packages/code/src COPY shared /shared +COPY packages/code /packages/code COPY service/tsconfig.json ./ EXPOSE 3112 9230 CMD ["bun", "run", "--watch", "src/api-server.ts"] diff --git a/service/src/bridge/pairing.test.ts b/service/src/bridge/pairing.test.ts index 9f4b7117..36b2ea6d 100644 --- a/service/src/bridge/pairing.test.ts +++ b/service/src/bridge/pairing.test.ts @@ -65,7 +65,7 @@ describe('RedisBridgePairingStore', () => { originalAuthorization.identityId, ); await expect( - pairings.authorize(requestFor(issued.credential, 'overlap-bound-proof')), + pairings.authorize(requestFor(issued.credential, 'superseded-bound-proof')), ).resolves.toMatchObject({ workerId: 'vm-bound', identityId: originalAuthorization.identityId, @@ -129,6 +129,27 @@ describe('RedisBridgePairingStore', () => { ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); }); + test('preserves a pairing code after public-key validation fails', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-public-key-retry'); + + await expect( + pairings.redeem({ + workerId: 'vm-public-key-retry', + code: pairing.code, + publicKey: 'not-a-public-key', + }), + ).rejects.toMatchObject({ code: 'PUBLIC_KEY_INVALID' }); + + await expect( + pairings.redeem({ + workerId: 'vm-public-key-retry', + code: pairing.code, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-public-key-retry' }); + }); + test('only the newest pairing code can rebind a worker identity', async () => { const identity = createBridgeIdentity(); const older = await pairings.issue('vm-1', { @@ -174,16 +195,15 @@ describe('RedisBridgePairingStore', () => { }); let paused = false; redis.eval = (async (script: string, ...args: unknown[]) => { - const result = await (originalEval as (...evalArgs: unknown[]) => Promise)( - script, - ...args, - ); - if (!paused && script.includes('return pairing')) { + if (!paused && script.includes('if pairing ~= ARGV[1]')) { paused = true; firstRedeemed(); await releaseFirstPromise; } - return result; + return await (originalEval as (...evalArgs: unknown[]) => Promise)( + script, + ...args, + ); }) as typeof redis.eval; try { @@ -426,7 +446,556 @@ describe('RedisBridgePairingStore', () => { ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); }); - test('rotation keeps the prior same-identity credential usable for recovery', async () => { + test('revocation invalidates an unredeemed pairing code', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + + await pairings.revoke('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + }); + + test('revocation removes pairing codes issued by a pre-fence replica', async () => { + const legacyCode = 'legacy-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ workerId: 'vm-1', expiresAt: new Date(Date.now() + 60_000).toISOString() }), + 'EX', + 60, + ); + + await pairings.revoke('vm-1'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('retries legacy cleanup after a transient scan failure', async () => { + const legacyCode = 'retryable-legacy-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-scan-retry', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const scan = redis.scan.bind(redis); + let failScan = true; + redis.scan = (async (...args: Parameters) => { + if (failScan) { + failScan = false; + throw new Error('transient scan failure'); + } + return scan(...args); + }) as Redis['scan']; + + await expect(pairings.revoke('vm-scan-retry')).rejects.toThrow( + 'transient scan failure', + ); + redis.scan = scan; + await pairings.revoke('vm-scan-retry'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('retries a claimed cleanup after the migration deadline', async () => { + const legacyCode = 'post-deadline-retry-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-post-deadline-retry', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:migration:legacy-pairing-scan-until', + String(Date.now() + 15), + ); + const scan = redis.scan.bind(redis); + let failScan = true; + redis.scan = (async (...args: Parameters) => { + if (failScan) { + failScan = false; + await new Promise((resolve) => setTimeout(resolve, 30)); + throw new Error('scan failed after deadline'); + } + return scan(...args); + }) as Redis['scan']; + + await expect(pairings.revoke('vm-post-deadline-retry')).rejects.toThrow( + 'scan failed after deadline', + ); + await pairings.revoke('vm-post-deadline-retry'); + + redis.scan = scan; + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('renews the cleanup claim while a shared-keyspace scan is in flight', async () => { + const store = new RedisBridgePairingStore(redis, 600, 300, 30); + const legacyCode = 'renewed-claim-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-renewed-claim', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const scan = redis.scan.bind(redis); + let scanCalls = 0; + let renewCalls = 0; + let markScanStarted = () => {}; + const scanStarted = new Promise((resolve) => { + markScanStarted = resolve; + }); + redis.scan = (async (...args: Parameters) => { + scanCalls += 1; + if (scanCalls === 1) { + markScanStarted(); + await new Promise((resolve) => setTimeout(resolve, 80)); + } + return scan(...args); + }) as Redis['scan']; + const originalEval = redis.eval.bind(redis); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes("redis.call('PEXPIRE'")) renewCalls += 1; + return await (originalEval as (...evalArgs: unknown[]) => Promise)( + script, + ...args, + ); + }) as typeof redis.eval; + + try { + const first = store.revoke('vm-renewed-claim'); + await scanStarted; + await new Promise((resolve) => setTimeout(resolve, 45)); + expect(renewCalls).toBeGreaterThan(0); + await first; + + expect(scanCalls).toBe(1); + await expect(redis.get(legacyKey)).resolves.toBeNull(); + } finally { + redis.scan = scan; + redis.eval = originalEval as typeof redis.eval; + } + }); + + test('rescans an ambiguous marker written by the preceding build', async () => { + const legacyCode = 'ambiguous-predecessor-marker-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const workerId = 'vm-ambiguous-predecessor-marker'; + await redis.set( + legacyKey, + JSON.stringify({ + workerId, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + `codeapi:bridge:v1:migration:legacy-pairing-scanned:${workerId}`, + 'predecessor-random-token', + 'PX', + 60_000, + ); + + await pairings.revoke(workerId); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('starts the advertised pairing lifetime after legacy cleanup', async () => { + const store = new RedisBridgePairingStore(redis, 60); + const scan = redis.scan.bind(redis); + const originalNow = Date.now; + let now = originalNow(); + Date.now = () => now; + redis.scan = (async (...args: Parameters) => { + const result = await scan(...args); + now += 10; + return result; + }) as Redis['scan']; + + try { + const pairing = await store.issue('vm-post-cleanup-expiry'); + expect(Date.parse(pairing.expiresAt) - Date.now()).toBe(60_000); + } finally { + Date.now = originalNow; + redis.scan = scan; + } + }); + + test('waits for a failed in-progress cleanup and confirms legacy removal itself', async () => { + const legacyCode = 'overlapping-legacy-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-overlapping-cleanup', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const scan = redis.scan.bind(redis); + let releaseFirstScan = () => {}; + const firstScanGate = new Promise((resolve) => { + releaseFirstScan = resolve; + }); + let markFirstScanStarted = () => {}; + const firstScanStarted = new Promise((resolve) => { + markFirstScanStarted = resolve; + }); + let scanCalls = 0; + redis.scan = (async (...args: Parameters) => { + scanCalls += 1; + if (scanCalls === 1) { + markFirstScanStarted(); + await firstScanGate; + throw new Error('interrupted claimed scan'); + } + return scan(...args); + }) as Redis['scan']; + + const interrupted = pairings.revoke('vm-overlapping-cleanup'); + await firstScanStarted; + const overlapping = pairings.revoke('vm-overlapping-cleanup'); + let overlappingSettled = false; + void overlapping.then( + () => { + overlappingSettled = true; + }, + () => { + overlappingSettled = true; + }, + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(overlappingSettled).toBe(false); + releaseFirstScan(); + + await expect(interrupted).rejects.toThrow('interrupted claimed scan'); + await expect(overlapping).resolves.toBeUndefined(); + redis.scan = scan; + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('legacy cleanup does not delete generation-fenced pairings', async () => { + const fencedCode = 'concurrent-generation-pairing'; + const fencedKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(fencedCode) + .digest('hex')}`; + await redis.set( + fencedKey, + JSON.stringify({ + workerId: 'vm-generation-fenced', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + generation: 'new-generation', + }), + 'EX', + 60, + ); + + await pairings.revoke('vm-generation-fenced'); + + await expect(redis.get(fencedKey)).resolves.not.toBeNull(); + }); + + test('reopens legacy cleanup after a rollback outlives the prior scan window', async () => { + const legacyCode = 'later-rollback-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const deadlineKey = 'codeapi:bridge:v1:migration:legacy-pairing-scan-until'; + await redis.set(deadlineKey, '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-later-rollback', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:pairing-index:vm-later-rollback', + legacyKey, + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:migration:legacy-pairing-scanned:vm-later-rollback', + 'done', + ); + + await pairings.revoke('vm-later-rollback'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + await expect(redis.get(deadlineKey)).resolves.toBe('0'); + }); + + test('does not restart an expired migration window without rollback evidence', async () => { + const legacyCode = 'unindexed-post-migration-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const deadlineKey = 'codeapi:bridge:v1:migration:legacy-pairing-scan-until'; + await redis.set(deadlineKey, '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-no-rollback-signal', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + + await pairings.revoke('vm-no-rollback-signal'); + + await expect(redis.get(legacyKey)).resolves.not.toBeNull(); + await expect(redis.get(deadlineKey)).resolves.toBe('0'); + }); + + test('restarts legacy cleanup once for an explicit rollback epoch', async () => { + const legacyCode = 'unindexed-rollback-epoch-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const deadlineKey = 'codeapi:bridge:v1:migration:legacy-pairing-scan-until'; + await redis.set(deadlineKey, '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-rollback-epoch', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:migration:legacy-pairing-scanned:vm-rollback-epoch', + 'done', + ); + const rollbackAwarePairings = new RedisBridgePairingStore( + redis, + 600, + 300, + 5_000, + 'rollback-epoch-1', + ); + + await rollbackAwarePairings.revoke('vm-rollback-epoch'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + await expect(redis.get(deadlineKey)).resolves.toBe('0'); + const epochHash = createHash('sha256') + .update('rollback-epoch-1') + .digest('hex'); + const epochStateKey = + `codeapi:bridge:v1:migration:legacy-pairing-scanned:` + + `vm-rollback-epoch:${epochHash}`; + await expect(redis.get(epochStateKey)).resolves.toBe('done'); + await expect(redis.pttl(epochStateKey)).resolves.toBe(-1); + }); + + test('cleans a revoked legacy code before rollback-epoch redemption', async () => { + const identity = createBridgeIdentity(); + const workerId = 'vm-rollback-redeem'; + const legacyCode = 'revoked-rollback-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set('codeapi:bridge:v1:migration:legacy-pairing-scan-until', '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const rollbackAwarePairings = new RedisBridgePairingStore( + redis, + 600, + 300, + 5_000, + 'rollback-epoch-redeem', + ); + + await expect( + rollbackAwarePairings.redeem({ + workerId, + code: legacyCode, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('does not scan for a nonexistent rollback-epoch pairing code', async () => { + const identity = createBridgeIdentity(); + const scan = redis.scan.bind(redis); + let scanCalls = 0; + redis.scan = (async (...args: Parameters) => { + scanCalls += 1; + return scan(...args); + }) as Redis['scan']; + const rollbackAwarePairings = new RedisBridgePairingStore( + redis, + 600, + 300, + 5_000, + 'rollback-epoch-missing-code', + ); + + try { + await expect( + rollbackAwarePairings.redeem({ + workerId: 'attacker-chosen-worker', + code: 'nonexistent-code', + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + expect(scanCalls).toBe(0); + await expect( + redis.keys( + 'codeapi:bridge:v1:migration:legacy-pairing-scanned:attacker-chosen-worker*', + ), + ).resolves.toEqual([]); + } finally { + redis.scan = scan; + } + }); + + test('redeems an unrevoked pairing code issued by a pre-fence replica', async () => { + const identity = createBridgeIdentity(); + const legacyCode = 'unrevoked-legacy-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ workerId: 'vm-1', expiresAt: new Date(Date.now() + 60_000).toISOString() }), + 'EX', + 60, + ); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: legacyCode, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('redeems an indexed legacy code issued after rollback', async () => { + const identity = createBridgeIdentity(); + await pairings.issue('vm-rollback'); + const legacyCode = 'rollback-issued-legacy-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-rollback', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:pairing-index:vm-rollback', + legacyKey, + 'EX', + 60, + ); + + await expect( + pairings.redeem({ + workerId: 'vm-rollback', + code: legacyCode, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-rollback' }); + }); + + test('replacement invalidates a pairing code issued by a pre-fence replica', async () => { + const identity = createBridgeIdentity(); + const legacyCode = 'replaced-legacy-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ workerId: 'vm-1', expiresAt: new Date(Date.now() + 60_000).toISOString() }), + 'EX', + 60, + ); + + await pairings.issue('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: legacyCode, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + }); + + test('issuing a replacement invalidates the prior unredeemed pairing code', async () => { + const identity = createBridgeIdentity(); + const first = await pairings.issue('vm-1'); + const replacement = await pairings.issue('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: first.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: replacement.code, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('rotation retains the prior same-identity credential for recovery', async () => { const identity = createBridgeIdentity(); const pairing = await pairings.issue('vm-1'); const original = await pairings.redeem({ diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 24deba07..74c75b7c 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -13,6 +13,43 @@ const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; const PROOF_NONCE_TTL_SECONDS = 2 * 60; const PROOF_CLOCK_SKEW_MS = 60_000; +const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; +const LEGACY_SCAN_POLL_INTERVAL_MS = 25; +const LEGACY_SCAN_PENDING = 'pending'; +const LEGACY_SCAN_COMPLETE = 'done'; +const ISSUE_PAIRING_SCRIPT = ` +local previous = redis.call('GET', KEYS[1]) +if previous then + redis.call('DEL', previous) +end +redis.call('SET', KEYS[1], KEYS[2], 'EX', ARGV[2]) +redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[2]) +return 1 +`; +const REDEEM_PAIRING_SCRIPT = ` +local pairing = redis.call('GET', KEYS[1]) +if pairing ~= ARGV[1] then + return 0 +end +local generation = redis.call('GET', KEYS[2]) +if ARGV[2] == '' then + if generation and redis.call('GET', KEYS[5]) ~= KEYS[1] then + redis.call('DEL', KEYS[1]) + return 0 + end +elseif (generation or '0') ~= ARGV[2] then + redis.call('DEL', KEYS[1]) + return 0 +end +redis.call('DEL', KEYS[1]) +if redis.call('GET', KEYS[5]) == KEYS[1] then + redis.call('DEL', KEYS[5]) +end +redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) +redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) +redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[4]) +return 1 +`; const ROTATE_CREDENTIAL_SCRIPT = ` local activeDigest = redis.call('GET', KEYS[1]) local previous = redis.call('GET', KEYS[2]) @@ -33,59 +70,57 @@ else end return 1 `; -const REVOKE_WORKER_SCRIPT = ` -local activeDigest = redis.call('GET', KEYS[1]) +const REVOKE_PAIRING_SCRIPT = ` +local indexed = redis.call('GET', KEYS[1]) +local credential = redis.call('GET', KEYS[3]) local activeIncarnation = redis.call('GET', KEYS[6]) -redis.call('INCR', KEYS[4]) -redis.call('DEL', KEYS[1], KEYS[2], KEYS[5], KEYS[6]) -if activeDigest then - redis.call('DEL', KEYS[3] .. activeDigest) +redis.call('INCR', KEYS[2]) +if indexed then + redis.call('DEL', indexed) end +if credential then + redis.call('DEL', KEYS[3]) + redis.call('DEL', ARGV[1] .. credential) +end +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6]) if activeIncarnation then - redis.call('SET', ARGV[1] .. activeIncarnation .. ':fenced', '1') + redis.call('SET', ARGV[2] .. activeIncarnation .. ':fenced', '1') end return 1 `; -const ISSUE_PAIRING_SCRIPT = ` -local previous = redis.call('GET', KEYS[1]) -if previous then - redis.call('DEL', previous) +const RELEASE_LEGACY_SCAN_CLAIM_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) end -redis.call('DEL', KEYS[3]) -redis.call('SET', KEYS[1], KEYS[2], 'EX', ARGV[2]) -redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[2]) -return 1 +return 0 `; -const REDEEM_PAIRING_SCRIPT = ` -local pairing = redis.call('GET', KEYS[1]) -if not pairing then - return nil +const NORMALIZE_LEGACY_SCAN_STATE_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('SET', KEYS[1], ARGV[2]) + return 1 end -if redis.call('GET', KEYS[2]) ~= KEYS[1] then - redis.call('DEL', KEYS[1]) - return nil -end -redis.call('DEL', KEYS[1], KEYS[2]) -redis.call('SET', KEYS[3], ARGV[1], 'EX', ARGV[2]) -return pairing +return 0 `; -const INSTALL_REDEEMED_CREDENTIAL_SCRIPT = ` -if redis.call('GET', KEYS[1]) ~= ARGV[1] then - return 0 +const RENEW_LEGACY_SCAN_CLAIM_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) end -local generation = redis.call('GET', KEYS[5]) or '0' -if generation ~= ARGV[6] then - return 0 -end -redis.call('SET', KEYS[2], ARGV[3], 'EX', ARGV[4]) -redis.call('SET', KEYS[3], ARGV[2], 'EX', ARGV[4]) -if ARGV[5] ~= '' then - redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) -else - redis.call('DEL', KEYS[4]) +return 0 +`; +const COMPLETE_LEGACY_SCAN_CLAIM_SCRIPT = ` +if redis.call('GET', KEYS[2]) == ARGV[1] then + local remaining = tonumber(ARGV[3]) + if ARGV[4] == '1' then + redis.call('SET', KEYS[1], ARGV[2]) + elseif remaining > 0 then + redis.call('SET', KEYS[1], ARGV[2], 'PX', remaining) + else + redis.call('DEL', KEYS[1]) + end + redis.call('DEL', KEYS[2]) + return 1 end -redis.call('DEL', KEYS[1]) -return 1 +return 0 `; export type BridgePrincipalType = 'deployment' | 'tenant' | 'user' | 'role' | 'group'; @@ -101,12 +136,13 @@ export interface BridgeWorkerBinding { interface StoredPairing { workerId: string; expiresAt: string; - generation: number; + generation?: number; binding?: BridgeWorkerBinding; } interface StoredCredential { workerId: string; + /** Stable across refreshes; replaced only when the worker is paired again. */ identityId?: string; publicKey: string; expiresAt: string; @@ -168,8 +204,17 @@ function workerPairingIndexKey(workerId: string): string { return `${PREFIX}:pairing-index:${workerId}`; } -function workerRedemptionKey(workerId: string): string { - return `${PREFIX}:redemption:${workerId}`; +function legacyPairingScanDeadlineKey(): string { + return `${PREFIX}:migration:legacy-pairing-scan-until`; +} + +function legacyPairingWorkerScanKey( + workerId: string, + rollbackEpoch?: string, +): string { + const epoch = rollbackEpoch?.trim(); + const epochSuffix = epoch ? `:${digest(epoch)}` : ''; + return `${PREFIX}:migration:legacy-pairing-scanned:${workerId}${epochSuffix}`; } function proofNonceKey(credential: string, nonce: string): string { @@ -189,12 +234,19 @@ export class RedisBridgePairingStore { private readonly redis: Redis, private readonly pairingTtlSeconds = DEFAULT_PAIRING_TTL_SECONDS, private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, + private readonly legacyScanClaimTtlMs = LEGACY_SCAN_CLAIM_TTL_MS, + private readonly rollbackEpoch = + process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', ) {} async issue( workerId: string, binding?: BridgeWorkerBinding, ): Promise { + // Pre-index binaries cannot remove a superseded code themselves. During + // the one pairing-TTL migration window, find and delete those records so + // rolling back cannot make a replaced code valid again. + await this.removeLegacyPairings(workerId); const code = randomBytes(24).toString('base64url'); const expiresAt = new Date( Date.now() + this.pairingTtlSeconds * 1000, @@ -206,10 +258,9 @@ export class RedisBridgePairingStore { const codeKey = pairingKey(code); await this.redis.eval( ISSUE_PAIRING_SCRIPT, - 3, + 2, workerPairingIndexKey(workerId), codeKey, - workerRedemptionKey(workerId), JSON.stringify(pairing), String(this.pairingTtlSeconds), ); @@ -221,45 +272,78 @@ export class RedisBridgePairingStore { code: string; publicKey: string; }): Promise { + const codeKey = pairingKey(args.code); + const raw = await this.redis.get(codeKey); + if (raw == null) { + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code is invalid or expired', + ); + } + const pairing = JSON.parse(raw) as StoredPairing; + if (pairing.workerId !== args.workerId) { + await this.redis.del(codeKey); + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code does not authorize this worker', + ); + } if (!validEd25519PublicKey(args.publicKey)) { throw new BridgePairingError( 'PUBLIC_KEY_INVALID', 'Worker public key must be an Ed25519 key', ); } - const codeKey = pairingKey(args.code); - const redemptionId = randomBytes(18).toString('base64url'); - const raw = await this.redis.eval( - REDEEM_PAIRING_SCRIPT, - 3, - codeKey, - workerPairingIndexKey(args.workerId), - workerRedemptionKey(args.workerId), - redemptionId, - String(this.pairingTtlSeconds), - ); - if (typeof raw !== 'string') { + // Validate the supplied code before it can trigger a shared-keyspace scan. + // A rollback epoch means any generation-less code may have survived a + // legacy revoke, so clean the authenticated worker and reject that code. + if ( + pairing.generation == null && + this.rollbackEpoch.trim().length > 0 + ) { + await this.removeLegacyPairings(args.workerId); throw new BridgePairingError( 'PAIRING_INVALID', 'Pairing code is invalid or expired', ); } - const pairing = JSON.parse(raw) as StoredPairing; - if (pairing.workerId !== args.workerId) { + + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date( + Date.now() + this.credentialTtlSeconds * 1000, + ).toISOString(); + const identityId = randomBytes(18).toString('base64url'); + const stored: StoredCredential = { + workerId: args.workerId, + identityId, + publicKey: args.publicKey, + expiresAt, + binding: pairing.binding, + }; + const accepted = await this.redis.eval( + REDEEM_PAIRING_SCRIPT, + 6, + codeKey, + workerPairingGenerationKey(pairing.workerId), + credentialDigestKey(credentialDigest), + workerIdentityKey(args.workerId), + workerPairingIndexKey(args.workerId), + workerStableIdentityKey(args.workerId), + raw, + pairing.generation == null ? '' : String(pairing.generation), + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + identityId, + ); + if (accepted !== 1) { throw new BridgePairingError( 'PAIRING_INVALID', - 'Pairing code does not authorize this worker', + 'Pairing code is invalid or expired', ); } - return await this.issueCredential( - args.workerId, - args.publicKey, - undefined, - undefined, - pairing.binding, - pairing.generation, - redemptionId, - ); + return { workerId: args.workerId, credential, expiresAt }; } async authorize(args: { @@ -356,19 +440,199 @@ export class RedisBridgePairingStore { } async revoke(workerId: string): Promise { + await this.removeLegacyPairings(workerId); + // Fence redemption and consume the currently indexed code atomically. An + // issue that linearized before this script is always removed; an issue + // that linearizes afterward installs a distinct generation and code. await this.redis.eval( - REVOKE_WORKER_SCRIPT, + REVOKE_PAIRING_SCRIPT, 6, + workerPairingIndexKey(workerId), + workerPairingGenerationKey(workerId), workerIdentityKey(workerId), workerStableIdentityKey(workerId), - `${PREFIX}:credential:`, - workerPairingGenerationKey(workerId), `${PREFIX}:worker:${encodeURIComponent(workerId)}`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, + `${PREFIX}:credential:`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, ); } + private async removeLegacyPairings(workerId: string): Promise { + const deadlineKey = legacyPairingScanDeadlineKey(); + const now = Date.now(); + const migrationWindowMs = this.pairingTtlSeconds * 1000; + const proposedDeadline = now + migrationWindowMs; + let rawDeadline = await this.redis.get(deadlineKey); + if (rawDeadline == null) { + const initialized = await this.redis.set( + deadlineKey, + String(proposedDeadline), + 'NX', + ); + rawDeadline = initialized === 'OK' + ? String(proposedDeadline) + : await this.redis.get(deadlineKey); + } else if ((await this.redis.pttl(deadlineKey)) > 0) { + // Markers from the preceding build expired and reopened forever. Keep + // their original deadline, but make it durable so normal idle periods + // cannot start another migration window. + await this.redis.persist(deadlineKey); + } + + const indexedKey = await this.redis.get(workerPairingIndexKey(workerId)); + const indexedRaw = indexedKey == null ? null : await this.redis.get(indexedKey); + let rollbackDetected = false; + if (indexedRaw != null) { + try { + rollbackDetected = (JSON.parse(indexedRaw) as StoredPairing).generation == null; + } catch { + rollbackDetected = false; + } + } + + const deadline = Number(rawDeadline); + const stateKey = legacyPairingWorkerScanKey(workerId, this.rollbackEpoch); + const rollbackEpochDetected = this.rollbackEpoch.trim().length > 0; + while (true) { + const state = await this.redis.get(stateKey); + if (state === LEGACY_SCAN_COMPLETE && !rollbackDetected) return; + if (state === LEGACY_SCAN_PENDING) break; + if (state == null) { + if ( + !rollbackDetected && + !rollbackEpochDetected && + (!Number.isFinite(deadline) || Date.now() > deadline) + ) { + return; + } + const initialized = await this.redis.set( + stateKey, + LEGACY_SCAN_PENDING, + 'NX', + ); + if (initialized === 'OK') break; + continue; + } + // Predecessor builds stored an unqualified random token before scanning. + // It cannot prove whether that scan completed, so normalize it to a + // durable retry requirement instead of treating it as success. + const normalized = await this.redis.eval( + NORMALIZE_LEGACY_SCAN_STATE_SCRIPT, + 1, + stateKey, + state, + LEGACY_SCAN_PENDING, + ); + if (normalized === 1) break; + } + + const claimKey = `${stateKey}:claim`; + let scanClaim: { key: string; token: string } | undefined; + while (scanClaim == null) { + const state = await this.redis.get(stateKey); + if (state === LEGACY_SCAN_COMPLETE || state == null) return; + const token = `claim:${randomBytes(24).toString('base64url')}`; + const claimed = await this.redis.set( + claimKey, + token, + 'PX', + Math.max(1, this.legacyScanClaimTtlMs), + 'NX', + ); + if (claimed === 'OK') { + scanClaim = { key: claimKey, token }; + break; + } + await new Promise((resolve) => + setTimeout(resolve, LEGACY_SCAN_POLL_INTERVAL_MS), + ); + } + + let renewalError: unknown; + let renewal = Promise.resolve(); + let renewalInFlight = false; + const renewClaim = async (): Promise => { + const renewed = await this.redis.eval( + RENEW_LEGACY_SCAN_CLAIM_SCRIPT, + 1, + scanClaim.key, + scanClaim.token, + String(Math.max(1, this.legacyScanClaimTtlMs)), + ); + if (renewed !== 1) { + throw new Error('Legacy pairing cleanup claim was lost'); + } + }; + const renewalTimer = setInterval(() => { + if (renewalInFlight || renewalError != null) return; + renewalInFlight = true; + renewal = renewClaim() + .catch((error: unknown) => { + renewalError = error; + }) + .finally(() => { + renewalInFlight = false; + }); + }, Math.max(1, Math.floor(this.legacyScanClaimTtlMs / 3))); + renewalTimer.unref?.(); + + try { + let cursor = '0'; + do { + const [nextCursor, keys] = await this.redis.scan( + cursor, + 'MATCH', + `${PREFIX}:pairing:*`, + 'COUNT', + 100, + ); + if (renewalError != null) throw renewalError; + cursor = nextCursor; + if (keys.length === 0) continue; + const values = await this.redis.mget(...keys); + const matching = keys.filter((_key, index) => { + const raw = values[index]; + if (raw == null) return false; + try { + const pairing = JSON.parse(raw) as Partial; + return pairing.workerId === workerId && pairing.generation == null; + } catch { + return false; + } + }); + if (matching.length > 0) await this.redis.del(...matching); + } while (cursor !== '0'); + clearInterval(renewalTimer); + await renewal; + if (renewalError != null) throw renewalError; + await renewClaim(); + const completed = await this.redis.eval( + COMPLETE_LEGACY_SCAN_CLAIM_SCRIPT, + 2, + stateKey, + scanClaim.key, + scanClaim.token, + LEGACY_SCAN_COMPLETE, + String(deadline - Date.now()), + rollbackEpochDetected ? '1' : '0', + ); + if (completed !== 1) { + await this.removeLegacyPairings(workerId); + } + } catch (error) { + clearInterval(renewalTimer); + await renewal; + await this.redis.eval( + RELEASE_LEGACY_SCAN_CLAIM_SCRIPT, + 1, + scanClaim.key, + scanClaim.token, + ); + throw error; + } + } + async rotate( workerId: string, expectedCredentialId?: string, @@ -391,8 +655,8 @@ export class RedisBridgePairingStore { workerId, previous.publicKey, previousDigest, - previous.identityId ?? null, previous.binding, + previous.identityId ?? null, ); } @@ -400,17 +664,18 @@ export class RedisBridgePairingStore { workerId: string, publicKey: string, previousDigest?: string, - identityId: string | null | undefined = randomBytes(18).toString('base64url'), binding?: BridgeWorkerBinding, - pairingGeneration?: number, - redemptionId?: string, + identityId?: string | null, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); const expiresAt = new Date( Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); - const stableIdentityId = identityId ?? undefined; + const stableIdentityId = + identityId === undefined + ? randomBytes(18).toString('base64url') + : identityId ?? undefined; const stored: StoredCredential = { workerId, ...(stableIdentityId != null ? { identityId: stableIdentityId } : {}), @@ -438,34 +703,31 @@ export class RedisBridgePairingStore { 'Worker credential is invalid or expired', ); } - return { workerId, credential, expiresAt }; - } - if (redemptionId == null) { - throw new BridgePairingError( - 'PAIRING_INVALID', - 'Pairing redemption was not fenced', + } else { + const transaction = this.redis.multi(); + transaction.set( + credentialDigestKey(credentialDigest), + JSON.stringify(stored), + 'EX', + this.credentialTtlSeconds, ); - } - const installed = await this.redis.eval( - INSTALL_REDEEMED_CREDENTIAL_SCRIPT, - 5, - workerRedemptionKey(workerId), - credentialDigestKey(credentialDigest), - workerIdentityKey(workerId), - workerStableIdentityKey(workerId), - workerPairingGenerationKey(workerId), - redemptionId, - credentialDigest, - JSON.stringify(stored), - String(this.credentialTtlSeconds), - stableIdentityId ?? '', - String(pairingGeneration ?? 0), - ); - if (installed !== 1) { - throw new BridgePairingError( - 'PAIRING_INVALID', - 'Pairing code was superseded before credential installation', + transaction.set( + workerIdentityKey(workerId), + credentialDigest, + 'EX', + this.credentialTtlSeconds, ); + if (stableIdentityId != null) { + transaction.set( + workerStableIdentityKey(workerId), + stableIdentityId, + 'EX', + this.credentialTtlSeconds, + ); + } else { + transaction.del(workerStableIdentityKey(workerId)); + } + await transaction.exec(); } return { workerId, credential, expiresAt }; } diff --git a/tests/bridge_pairing_rollout.sh b/tests/bridge_pairing_rollout.sh new file mode 100755 index 00000000..c4246db6 --- /dev/null +++ b/tests/bridge_pairing_rollout.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +values=helm/codeapi/values.yaml +deployment=helm/codeapi/templates/api-deployment.yaml +rollback=helm/codeapi/scripts/safe-pairing-rollback.sh + +if ! grep -A 12 '^api:$' "$values" | grep -q '^ strategy:$'; then + echo 'api.strategy must be configured for pairing-safe rollouts' >&2 + exit 1 +fi +if ! grep -A 2 '^ strategy:$' "$values" | grep -q '^ type: Recreate$'; then + echo 'api.strategy.type must default to Recreate while pre-fence replicas may exist' >&2 + exit 1 +fi +if ! grep -A 3 '^ strategy:$' "$values" | grep -q '^ rollingUpdate: null$'; then + echo 'api.strategy must clear rollingUpdate when switching existing deployments to Recreate' >&2 + exit 1 +fi +if ! grep -q 'toYaml .Values.api.strategy' "$deployment"; then + echo 'the API Deployment must render api.strategy' >&2 + exit 1 +fi +if ! grep -q 'codeapi.librechat.ai/pairing-fence-version: "1"' "$deployment"; then + echo 'the first pairing-fence chart upgrade must revise the API pod template' >&2 + exit 1 +fi +if ! grep -A 8 '^ image:$' "$values" | grep -q '^ pullPolicy: Always$'; then + echo 'the fenced API rollout must pull the current image even when the default tag is mutable' >&2 + exit 1 +fi +if [[ ! -x "$rollback" ]]; then + echo 'the pairing-safe rollback helper must be executable' >&2 + exit 1 +fi +bash -n "$rollback" +if "$rollback" codeapi 1 default --kube-context other >/dev/null 2>&1; then + echo 'rollback must reject a Helm context that differs from the kubectl drain' >&2 + exit 1 +fi +if "$rollback" codeapi 1 default --kubeconfig=/tmp/other >/dev/null 2>&1; then + echo 'rollback must reject a Helm kubeconfig that differs from the kubectl drain' >&2 + exit 1 +fi +if HELM_KUBECONTEXT=other "$rollback" codeapi 1 default >/dev/null 2>&1; then + echo 'rollback must reject a Helm context inherited from the environment' >&2 + exit 1 +fi +if ! grep -q 'delete horizontalpodautoscaler' "$rollback" || + ! grep -q 'scale "$deployment" --replicas=0' "$rollback" || + ! grep -q -- '--for=delete' "$rollback" || + ! grep -q 'create configmap "$rollback_config_map"' "$rollback" || + ! grep -q 'replica_state=' "$rollback" || + ! grep -q 'discover_api_deployments' "$rollback" || + ! grep -q 'list_api_pods' "$rollback" || + ! grep -q '^ drain_api delete$' "$rollback" || + ! grep -q 'recover_interrupted_rollback' "$rollback" || + ! grep -q 'helm rollback' "$rollback"; then + echo 'rollback must record an epoch, remove autoscaling, verify the drain, and fail closed' >&2 + exit 1 +fi +if ! grep -q 'CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH' "$deployment" || + ! grep -q 'optional: true' "$deployment"; then + echo 'the API Deployment must consume the optional rollback epoch' >&2 + exit 1 +fi From 45449eebbada686bec97826b1496c9b53ae12ec1 Mon Sep 17 00:00:00 2001 From: Chotto Magic <29515187+chottokun@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:20:43 +0900 Subject: [PATCH 007/116] fix(codeapi): add UTF-8 charset defaults to busboy in upload routes (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /upload and /upload/batch endpoints in router.ts construct busboy without specifying defCharset or defParamCharset. Busboy defaults to Latin-1 (ISO-8859-1) for multipart parameter decoding, which causes non-ASCII filenames (e.g. Japanese characters) to be garbled. file-server.ts already sets defCharset: 'utf8' and defParamCharset: 'utf8' (lines 336-337), but router.ts was missing the same options — an asymmetry that surfaces as mojibake when uploading files with non-ASCII names through the API gateway. Add both options to the two busboy() calls in router.ts for consistency with file-server.ts. --- service/src/service/router.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 542fe4c0..971bb7fd 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -396,6 +396,8 @@ router.post('/upload', uploadLimiter, async (req: t.AuthenticatedRequest, res: R const bb = busboy({ headers: req.headers, limits: { fileSize: planFileSize }, + defCharset: 'utf8', + defParamCharset: 'utf8', preservePath: true, }); @@ -606,6 +608,8 @@ router.post('/upload/batch', uploadLimiter, async (req: t.AuthenticatedRequest, const bb = busboy({ headers: req.headers, limits: { fileSize: planFileSize, files: MAX_BATCH_FILES }, + defCharset: 'utf8', + defParamCharset: 'utf8', preservePath: true, }); From a7afac59d3d94cf4e40065d9eabff8eebe4a10da Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 1 Sep 2026 22:21:07 -0400 Subject: [PATCH 008/116] =?UTF-8?q?=F0=9F=A7=B6=20fix:=20Large=20Tool=20In?= =?UTF-8?q?puts=20Break=20Bash=20PTC=20Replay=20and=20Pending-Call=20Seria?= =?UTF-8?q?lization=20(#75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: large tool inputs break bash PTC replay and pending-call serialization * fix: reject multi-document tool input and invalid PTC byte-cap env values * fix: validate floored PTC byte caps to avoid a zero-cap collapse --------- Co-authored-by: kenzaelk98 <254484110+kenzaelk98@users.noreply.github.com> --- service/scripts/test-ptc-replay-bash-smoke.ts | 41 +++++++++++++++++++ service/src/config.test.ts | 28 +++++++++++++ service/src/config.ts | 14 +++++++ service/src/preamble-bash.ts | 39 ++++++++++++------ service/src/service/replay-state.ts | 4 +- 5 files changed, 111 insertions(+), 15 deletions(-) diff --git a/service/scripts/test-ptc-replay-bash-smoke.ts b/service/scripts/test-ptc-replay-bash-smoke.ts index e01331b1..05937d32 100644 --- a/service/scripts/test-ptc-replay-bash-smoke.ts +++ b/service/scripts/test-ptc-replay-bash-smoke.ts @@ -519,5 +519,46 @@ get_weather '{"city":"Rome"}' ); } +{ + // Regression: input past ARG_MAX (~128KB) must still replay from history, + // not get silently re-issued as a new pending call each run. + const bigExpr = '1+1;#' + 'x'.repeat(300_000); + const user = ` +result=$(calculate '{"expression":"${bigExpr}"}') +echo "Result: $result" +`; + const r1 = runBash(assemble(user), {}); + const p1 = extractPending(r1.stdout); + assert(r1.exitCode === 0, 'large_input: first run exit 0'); + assert(p1.pending?.[0]?.call_id === 'call_001', 'large_input: first pending is call_001'); + assert( + typeof p1.pending?.[0]?.input_hash === 'string' && p1.pending[0].input_hash.length === 64, + 'large_input: pending carries a hash for the oversized input', + ); + + const history = { + call_001: { + result: 'ok-large', + tool_name: 'calculate', + input_hash: hashToolInput({ expression: bigExpr }), + received_at: 1, + }, + }; + const r2 = runBash(assemble(user), history); + const p2 = extractPending(r2.stdout); + assert(r2.exitCode === 0, 'large_input_replay: exit 0'); + assert(p2.pending === null, 'large_input_replay: no pending re-emitted for the same oversized call'); + assert(p2.stdout.includes('Result: "ok-large"'), 'large_input_replay: cached result used instead of re-invoking'); +} + +{ + // Regression: a stray second JSON document must be rejected, not silently + // truncated to the first one. + const user = `calculate '{"expression":"1+1"} {"expression":"2+2"}'`; + const r = runBash(assemble(user), {}); + assert(r.exitCode === 1, 'multi_doc_input: exit 1'); + assert(r.stderr.includes('must be a single JSON object'), 'multi_doc_input: stderr explains the rejection'); +} + console.log(`\n${passed} passed, ${failed} failed`); if (failed > 0) process.exit(1); diff --git a/service/src/config.test.ts b/service/src/config.test.ts index 87658f98..8fe24399 100644 --- a/service/src/config.test.ts +++ b/service/src/config.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { parsePlanLimits, resolveBridgeAuthMode, + resolvePositiveIntEnv, resolveRuntimeSessionMode, resolveSandboxBackend, } from './config'; @@ -41,6 +42,33 @@ describe('sandbox execution configuration', () => { }); }); +describe('resolvePositiveIntEnv', () => { + test('falls back to the default when unset or blank', () => { + expect(resolvePositiveIntEnv(undefined, 42)).toBe(42); + expect(resolvePositiveIntEnv('', 42)).toBe(42); + expect(resolvePositiveIntEnv(' ', 42)).toBe(42); + }); + + test('accepts positive finite values and floors them', () => { + expect(resolvePositiveIntEnv('100', 42)).toBe(100); + expect(resolvePositiveIntEnv('100.9', 42)).toBe(100); + }); + + test('falls back to the default for zero, negative, non-finite, or non-numeric values', () => { + expect(resolvePositiveIntEnv('0', 42)).toBe(42); + expect(resolvePositiveIntEnv('-5', 42)).toBe(42); + expect(resolvePositiveIntEnv('Infinity', 42)).toBe(42); + expect(resolvePositiveIntEnv('-Infinity', 42)).toBe(42); + expect(resolvePositiveIntEnv('NaN', 42)).toBe(42); + expect(resolvePositiveIntEnv('not-a-number', 42)).toBe(42); + }); + + test('falls back to the default when flooring would collapse a fraction to zero', () => { + expect(resolvePositiveIntEnv('0.5', 42)).toBe(42); + expect(resolvePositiveIntEnv('0.9', 42)).toBe(42); + }); +}); + describe('parsePlanLimits', () => { test('returns an empty catalog when unset or blank', () => { expect(parsePlanLimits(undefined)).toEqual({}); diff --git a/service/src/config.ts b/service/src/config.ts index 95df4eba..f0be8ba4 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -210,6 +210,17 @@ export function lambdaMicrovmNumericConfigError( return undefined; } +export function resolvePositiveIntEnv(raw: string | undefined, defaultValue: number): number { + if (raw == null || raw.trim() === '') { + return defaultValue; + } + const parsed = Math.floor(Number(raw)); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + return defaultValue; + } + return parsed; +} + export function resolveEgressGrantTtlSeconds(rawTtlSeconds: string | undefined, jobTimeoutMs: number): number { const defaultTtlSeconds = Math.max(1, Math.ceil((jobTimeoutMs + EGRESS_GRANT_GRACE_MS) / 1000)); if (rawTtlSeconds == null || rawTtlSeconds.trim() === '') { @@ -292,6 +303,9 @@ export const env = { EGRESS_GATEWAY_FILE_SERVER_URL: process.env.EGRESS_GATEWAY_FILE_SERVER_URL ?? process.env.FILE_SERVER_URL ?? 'http://localhost:3000', EGRESS_GATEWAY_TOOL_CALL_SERVER_URL: process.env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL ?? process.env.TOOL_CALL_SERVER_URL ?? 'http://localhost:3033', EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES: Number(process.env.EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES) || 1024 * 1024, + // Per-entry / aggregate caps for PTC tool results persisted in `tool_history:` (see replay-state.ts). + PTC_MAX_TOOL_RESULT_BYTES: resolvePositiveIntEnv(process.env.PTC_MAX_TOOL_RESULT_BYTES, 5_000_000), + PTC_MAX_TOOL_HISTORY_TOTAL_BYTES: resolvePositiveIntEnv(process.env.PTC_MAX_TOOL_HISTORY_TOTAL_BYTES, 40_000_000), EGRESS_GATEWAY_MAX_FILE_BYTES: Number(process.env.EGRESS_GATEWAY_MAX_FILE_BYTES ?? process.env.SANDBOX_MAX_FILE_SIZE) || 10_000_000, EGRESS_GATEWAY_MAX_PATH_LENGTH: Number(process.env.EGRESS_GATEWAY_MAX_PATH_LENGTH ?? process.env.SANDBOX_MAX_PATH_LENGTH) || 256, EGRESS_GATEWAY_MAX_NESTING_DEPTH: Number(process.env.EGRESS_GATEWAY_MAX_NESTING_DEPTH ?? process.env.SANDBOX_MAX_NESTING_DEPTH) || 10, diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index 2a27c996..4ec3fc05 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -417,18 +417,20 @@ _ptc_next_call_id() { _ptc_history_matches_by_signature() { local _ptc_name="$1" - local _ptc_input="$2" + local _ptc_input_file="$2" local _ptc_input_hash="$3" local _ptc_call_site="$4" if [ ! -r "$_PTC_HISTORY_PATH" ]; then return 0 fi + # Path, not inline: large input can exceed ARG_MAX via --argjson. jq -c \\ --arg nm "$_ptc_name" \\ --arg site "$_ptc_call_site" \\ --arg hash "$_ptc_input_hash" \\ - --argjson inp "$_ptc_input" \\ - 'to_entries + --slurpfile inp_arr "$_ptc_input_file" \\ + '($inp_arr[0]) as $inp + | to_entries | map(select((.value | type) == "object" and .value.tool_name == $nm and ((.value.input_hash == $hash) or (.value.input == $inp)))) @@ -474,13 +476,15 @@ _ptc_print_history_entry() { _ptc_history_entry_matches_current_call() { local _ptc_entry="$1" local _ptc_name="$2" - local _ptc_input="$3" + local _ptc_input_file="$3" local _ptc_input_hash="$4" + # Path, same ARG_MAX reason as above. printf '%s' "$_ptc_entry" | jq -e \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ - --argjson inp "$_ptc_input" \\ - 'if type != "object" then true + --slurpfile inp_arr "$_ptc_input_file" \\ + '($inp_arr[0]) as $inp + | if type != "object" then true elif (has("tool_name") and .tool_name != $nm) then false elif (has("input_hash") or has("input")) then ((.input_hash == $hash) or (.input == $inp)) @@ -494,8 +498,9 @@ _ptc_call_tool() { local _ptc_input="\${2:-\$_ptc_default_input}" local _ptc_call_site="\${BASH_LINENO[1]:-\${BASH_LINENO[0]:-0}}" - if ! printf '%s' "$_ptc_input" | jq -e 'type == "object"' >/dev/null 2>&1; then - _ptc_write_error "tool input for $_ptc_name must be a JSON object, got: $_ptc_input" + # Reject extra trailing JSON values instead of silently dropping them. + if ! printf '%s' "$_ptc_input" | jq -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then + _ptc_write_error "tool input for $_ptc_name must be a single JSON object, got: $_ptc_input" exit 1 fi @@ -505,8 +510,13 @@ _ptc_call_tool() { exit 1 fi + # Large input can exceed ARG_MAX via --argjson; write once, reuse path below. + local _ptc_input_tmp + _ptc_input_tmp="$(mktemp -t _ptc_input.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_input.XXXXXX)" + printf '%s' "$_ptc_input" > "$_ptc_input_tmp" + local _ptc_matches - _ptc_matches=$(_ptc_history_matches_by_signature "$_ptc_name" "$_ptc_input" "$_ptc_input_hash" "$_ptc_call_site") + _ptc_matches=$(_ptc_history_matches_by_signature "$_ptc_name" "$_ptc_input_tmp" "$_ptc_input_hash" "$_ptc_call_site") if ! _ptc_acquire_lock; then exit 1 @@ -522,6 +532,7 @@ _ptc_call_tool() { printf '%s\\n' "$_ptc_matched_call_id" >> "$_PTC_CONSUMED_FILE" _ptc_mark_counter_at_least "$_ptc_matched_call_id" _ptc_release_lock + rm -f "$_ptc_input_tmp" _ptc_print_history_entry "$_ptc_matched_entry" return $? fi @@ -538,26 +549,28 @@ _ptc_call_tool() { if [ -z "$_ptc_entry" ] || [ "$_ptc_entry" = "null" ]; then break fi - if _ptc_history_entry_matches_current_call "$_ptc_entry" "$_ptc_name" "$_ptc_input" "$_ptc_input_hash"; then + if _ptc_history_entry_matches_current_call "$_ptc_entry" "$_ptc_name" "$_ptc_input_tmp" "$_ptc_input_hash"; then printf '%s\\n' "$_ptc_call_id" >> "$_PTC_CONSUMED_FILE" _ptc_release_lock + rm -f "$_ptc_input_tmp" _ptc_print_history_entry "$_ptc_entry" return $? fi done - if ! jq -c -n \\ + if ! printf '%s' "$_ptc_input" | jq -c -n \\ --arg cid "$_ptc_call_id" \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ --arg site "$_ptc_call_site" \\ - --argjson inp "$_ptc_input" \\ - '{call_id:$cid,tool_name:$nm,input:$inp,input_hash:$hash,call_site:$site}' >> "$_PTC_PENDING_FILE"; then + '{call_id:$cid,tool_name:$nm,input:input,input_hash:$hash,call_site:$site}' >> "$_PTC_PENDING_FILE"; then _ptc_write_error "failed to serialize pending tool call for $_ptc_name" _ptc_release_lock + rm -f "$_ptc_input_tmp" exit 1 fi _ptc_release_lock + rm -f "$_ptc_input_tmp" exit 0 } diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 4d2a0501..2254ee21 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -65,13 +65,13 @@ export const REPLAY_LOCK_TTL_MS = Math.max(10 * 60 * 1000, env.JOB_TIMEOUT * 2 + * Redis hash and the `_ptc_history.json` injected into the sandbox bounded * regardless of how pathological a single tool result is. Scaled * proportionally with `MAX_EXECUTION_STATE_BYTES` (ratio 1:2 vs exec_state). */ -export const MAX_TOOL_RESULT_BYTES = 5_000_000; +export const MAX_TOOL_RESULT_BYTES = env.PTC_MAX_TOOL_RESULT_BYTES; /** Aggregate cap across ALL results persisted for a single execution. Scaled * proportionally with `MAX_EXECUTION_STATE_BYTES` (ratio 4:1 vs exec_state) * so a long replay flow can accumulate ~8 saturating tool results before * being asked to break work into a fresh execution. */ -export const MAX_TOOL_HISTORY_TOTAL_BYTES = 40_000_000; +export const MAX_TOOL_HISTORY_TOTAL_BYTES = env.PTC_MAX_TOOL_HISTORY_TOTAL_BYTES; /** Maximum number of keys `scanKeys` will return in a single call. The janitor * runs every `STALE_CLEANUP_INTERVAL` and processes whatever this yields; if From b43d1271bb756f0fb1a2670bb570c7cb529a5322 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 2 Sep 2026 04:26:13 +0200 Subject: [PATCH 009/116] fix(runner): reject NUL bytes in file paths (#46) --- api/src/validation.test.ts | 8 ++++++++ api/src/validation.ts | 3 +++ 2 files changed, 11 insertions(+) diff --git a/api/src/validation.test.ts b/api/src/validation.test.ts index 0596c2f5..a7566859 100644 --- a/api/src/validation.test.ts +++ b/api/src/validation.test.ts @@ -126,6 +126,10 @@ describe('validateFilePath', () => { expect(() => validateFilePath('.', submissionDir)).toThrow(ValidationError); }); + it('rejects NUL bytes in file names', () => { + expect(() => validateFilePath('file\0.txt', submissionDir)).toThrow(/NUL/); + }); + it('rejects path traversal with .. segments', () => { expect(() => validateFilePath('../etc/passwd', submissionDir)).toThrow(ValidationError); expect(() => validateFilePath('a/../../escape', submissionDir)).toThrow(ValidationError); @@ -188,4 +192,8 @@ describe('isValidFilePath', () => { expect(isValidFilePath('', submissionDir)).toBe(false); expect(isValidFilePath('a'.repeat(MAX_LEN + 10), submissionDir)).toBe(false); }); + + it('returns false for paths containing NUL bytes', () => { + expect(isValidFilePath('file\0.txt', submissionDir)).toBe(false); + }); }); diff --git a/api/src/validation.ts b/api/src/validation.ts index a29f3998..8ab04719 100644 --- a/api/src/validation.ts +++ b/api/src/validation.ts @@ -72,6 +72,9 @@ export function validateFilePath(name: string, submissionDir: string): void { if (!name || name === '.') { throw new ValidationError('File path must not be empty'); } + if (name.includes('\0')) { + throw new ValidationError('File path must not contain NUL bytes'); + } /* Reject absolute paths up front. `path.resolve(submissionDir, name)` * ignores `submissionDir` when `name` is absolute, so an absolute path * that happens to point inside `submissionDir` (e.g. the exact on-disk From d25bdf140621e7b22ba4154cafcc84257c943e86 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 1 Sep 2026 22:35:44 -0400 Subject: [PATCH 010/116] =?UTF-8?q?=F0=9F=A7=B0=20feat:=20Add=20Stateful?= =?UTF-8?q?=20Runtime=20Supervisor=20(#72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add Stateful Runtime Supervisor 🧰 * fix: harden runtime supervisor lifecycle --- packages/code/README.md | 19 +++-- packages/code/package.json | 4 + packages/code/src/cli.ts | 6 +- packages/code/src/index.ts | 1 + packages/code/src/runtime.test.ts | 75 ++++++++++++++++++ packages/code/src/runtime.ts | 75 ++++++++++++++++++ packages/code/src/worker.test.ts | 111 +++++++++++++++++++++++++++ packages/code/src/worker.ts | 121 +++++++++++++++++++----------- 8 files changed, 358 insertions(+), 54 deletions(-) create mode 100644 packages/code/src/runtime.test.ts create mode 100644 packages/code/src/runtime.ts diff --git a/packages/code/README.md b/packages/code/README.md index 883b2695..82d4fa22 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,11 +3,11 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. -The CLI is a transport bridge, not a sandbox. Run it beside a Code Interpreter -sandbox (NsJail for trusted local development, or the hardened microVM stack for -untrusted internet traffic). It connects outbound to Code API, long-polls for -assignments, forwards them to the local sandbox, and returns fenced results. -The VM does not need an inbound public port. +The CLI owns the runtime-supervisor seam. The bundled endpoint adapter connects +to an already-running loopback Code Interpreter sandbox; future adapters create +and isolate the runtime themselves. It connects outbound to Code API, +long-polls for assignments, sends them to the local runtime, and returns fenced +results. The VM does not need an inbound public port. ## Pair @@ -58,8 +58,9 @@ Optional environment variables: - `LIBRECHAT_CODE_POLICY`: local policy description hashed into the worker's registration; defaults to `default-deny`. - `LIBRECHAT_CODE_STATEFUL_WORKSPACE`: defaults to `false`. Set it to `true` - only when the local sandbox supervisor provides a distinct persistent runner - for every runtime session. In that mode the endpoint must contain a + only when the local runtime supervisor provides a distinct persistent runner + for every runtime session. The bundled endpoint adapter requires the endpoint + to contain a `{runtimeSessionId}` placeholder, for example `http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2`. The worker URL- encodes and substitutes the assigned session ID before execution. Hintless @@ -68,7 +69,9 @@ Optional environment variables: A single built-in sandbox runner binds itself to one runtime session and must not be advertised as stateful. Use the default stateless capability until a -session-routing supervisor is configured. +session-routing supervisor is configured. The endpoint adapter is a +compatibility adapter: it validates and routes a session but cannot create, +discard, or attest the underlying sandbox on its own. Static worker authentication is rejected when Code API hardened mode is enabled. Expose only the sandbox loopback endpoint to the CLI, and enforce diff --git a/packages/code/package.json b/packages/code/package.json index f195881e..0a4257d4 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -18,6 +18,10 @@ "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" + }, + "./runtime": { + "types": "./dist/runtime.d.ts", + "import": "./dist/runtime.js" } }, "bin": { diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 7edd8a26..b93374fc 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -8,6 +8,7 @@ import { saveBridgeIdentity, } from './storage.js'; import { BridgeWorker } from './worker.js'; +import { EndpointRuntimeSupervisor } from './runtime.js'; import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -119,7 +120,10 @@ async function run(runtimeSessionId?: string): Promise { token: configuredToken, identity: workerIdentity, workerId, - sandboxEndpoint, + runtimeSupervisor: new EndpointRuntimeSupervisor({ + endpoint: sandboxEndpoint, + statefulWorkspace, + }), capabilities, onIdentityChange: pairedIdentity && identityPath diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index c65b9f15..0f142908 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -2,4 +2,5 @@ export * from './protocol.js'; export * from './identity.js'; export * from './pairing.js'; export * from './storage.js'; +export * from './runtime.js'; export * from './worker.js'; diff --git a/packages/code/src/runtime.test.ts b/packages/code/src/runtime.test.ts new file mode 100644 index 00000000..d7bde841 --- /dev/null +++ b/packages/code/src/runtime.test.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { EndpointRuntimeSupervisor } from './runtime.js'; + +test('endpoint runtime supervisor resolves an isolated endpoint for stateful work', async () => { + const supervisor = new EndpointRuntimeSupervisor({ + endpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2/', + statefulWorkspace: true, + }); + + const lease = await supervisor.acquire({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + generation: 1, + leaseToken: 'lease-token', + expiresAt: new Date().toISOString(), + runtimeSessionId: 'rt/user 1', + request: { body: {}, headers: {} }, + }); + + assert.equal(lease.sessionId, 'rt/user 1'); + assert.equal( + lease.endpoint, + 'http://127.0.0.1:2000/sessions/rt%2Fuser%201/api/v2', + ); +}); + +test('endpoint runtime supervisor refuses stateful work without an isolated route', async () => { + const supervisor = new EndpointRuntimeSupervisor({ + endpoint: 'http://127.0.0.1:2000/api/v2', + statefulWorkspace: true, + }); + + await assert.rejects( + supervisor.acquire({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + generation: 1, + leaseToken: 'lease-token', + expiresAt: new Date().toISOString(), + runtimeSessionId: 'rt-1', + request: { body: {}, headers: {} }, + }), + /runtime supervisor endpoint containing/, + ); +}); + +test('endpoint runtime supervisor gives stateless work an ephemeral session route', async () => { + const supervisor = new EndpointRuntimeSupervisor({ + endpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + statefulWorkspace: false, + }); + + const lease = await supervisor.acquire({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + generation: 1, + leaseToken: 'lease-token', + expiresAt: new Date().toISOString(), + request: { body: {}, headers: {} }, + }); + + assert.equal(lease.sessionId, 'assignment-assignment-1'); + assert.equal( + lease.endpoint, + 'http://127.0.0.1:2000/sessions/assignment-assignment-1/api/v2', + ); +}); diff --git a/packages/code/src/runtime.ts b/packages/code/src/runtime.ts new file mode 100644 index 00000000..d65febca --- /dev/null +++ b/packages/code/src/runtime.ts @@ -0,0 +1,75 @@ +import type { BridgeAssignment } from './protocol.js'; + +export const RUNTIME_SESSION_PLACEHOLDER = '{runtimeSessionId}'; + +export interface RuntimeLease { + endpoint: string; + sessionId?: string; + release?(): Promise; +} + +export interface RuntimeSupervisor { + acquire(assignment: BridgeAssignment, signal?: AbortSignal): Promise; + reset(runtimeSessionId: string, signal?: AbortSignal): Promise; + quarantine(runtimeSessionId: string, reason: string, cause?: unknown): Promise; +} + +export interface EndpointRuntimeSupervisorOptions { + endpoint: string; + statefulWorkspace: boolean; +} + +function normalizedEndpoint(value: string): string { + return value.replace(/\/+$/, ''); +} + +function assignmentSessionId(assignment: BridgeAssignment): string | undefined { + if (assignment.runtimeSessionId != null) return assignment.runtimeSessionId; + if (assignment.assignmentId.length === 0) return undefined; + return `assignment-${assignment.assignmentId}`; +} + +/** + * Compatibility adapter for an already-running loopback sandbox supervisor. + * New runtime adapters own provisioning and return the same lease shape. + */ +export class EndpointRuntimeSupervisor implements RuntimeSupervisor { + private readonly endpoint: string; + + constructor(private readonly options: EndpointRuntimeSupervisorOptions) { + this.endpoint = normalizedEndpoint(options.endpoint); + if (this.endpoint.length === 0) { + throw new Error('Runtime supervisor endpoint is required'); + } + } + + async acquire(assignment: BridgeAssignment): Promise { + const sessionId = assignmentSessionId(assignment); + if (assignment.runtimeSessionId != null) { + if (!this.options.statefulWorkspace) { + throw new Error('Stateful assignments require a stateful runtime supervisor'); + } + if (!this.endpoint.includes(RUNTIME_SESSION_PLACEHOLDER)) { + throw new Error( + 'Stateful assignments require a runtime supervisor endpoint containing {runtimeSessionId}', + ); + } + } + if (sessionId == null || !this.endpoint.includes(RUNTIME_SESSION_PLACEHOLDER)) { + return { endpoint: this.endpoint }; + } + return { + endpoint: this.endpoint.replace( + RUNTIME_SESSION_PLACEHOLDER, + encodeURIComponent(sessionId), + ), + sessionId, + }; + } + + async reset(_runtimeSessionId: string): Promise {} + + async quarantine(runtimeSessionId: string, _reason: string, _cause?: unknown): Promise { + await this.reset(runtimeSessionId); + } +} diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index f4ec7d0b..f66074a3 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -8,6 +8,7 @@ import { } from './worker.js'; import type { BridgeAssignment } from './protocol.js'; +import type { RuntimeSupervisor } from './runtime.js'; const incarnationId = 'incarnation-00000001'; @@ -83,6 +84,116 @@ test('worker forwards a fenced assignment to the sandbox and settles the result' }); }); +test('worker delegates runtime acquisition, release, and reset to its supervisor', async () => { + const calls: string[] = []; + const supervisor: RuntimeSupervisor = { + async acquire(assignment) { + calls.push(`acquire:${assignment.runtimeSessionId}`); + return { + endpoint: 'http://127.0.0.1:3000/runtime', + sessionId: assignment.runtimeSessionId, + async release() { + calls.push(`release:${assignment.runtimeSessionId}`); + }, + }; + }, + async reset(runtimeSessionId) { + calls.push(`reset:${runtimeSessionId}`); + }, + async quarantine() {}, + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + runtimeSupervisor: supervisor, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'oci', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/execute')) { + assert.equal(url, 'http://127.0.0.1:3000/runtime/execute'); + return Response.json({ session_id: 'run-1', files: [] }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + const assignment: BridgeAssignment = { + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }; + + await worker.executeAndSettle(assignment); + await worker.resetWorkspace('rt-user-1'); + + assert.deepEqual(calls, [ + 'acquire:rt-user-1', + 'release:rt-user-1', + 'reset:rt-user-1', + ]); +}); + +test('worker asks its supervisor to quarantine an ambiguous stateful runtime', async () => { + const quarantined: Array<{ sessionId: string; reason: string }> = []; + const supervisor: RuntimeSupervisor = { + async acquire() { + return { endpoint: 'http://127.0.0.1:3000/runtime', sessionId: 'rt-user-1' }; + }, + async reset() {}, + async quarantine(sessionId, reason) { + quarantined.push({ sessionId, reason }); + }, + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + runtimeSupervisor: supervisor, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'oci', + runtimes: ['bash'], + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/execute')) { + throw new TypeError('connection reset'); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + runtimeSessionId: 'rt-user-1', + request: { body: { language: 'bash' }, headers: {} }, + }), + BridgeWorkspaceQuarantinedError, + ); + + assert.equal(quarantined.length, 1); + assert.equal(quarantined[0]?.sessionId, 'rt-user-1'); + assert.match(quarantined[0]?.reason ?? '', /ambiguous sandbox execution/); +}); + test('worker acknowledges a discarded workspace through the reset endpoint', async () => { let requestBody: Record | undefined; const worker = new BridgeWorker({ diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index ebbd486e..b4afe1b6 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -5,6 +5,7 @@ import { BridgeProtocolError, bridgeWorkerPath, } from './protocol.js'; +import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; import type { @@ -16,13 +17,16 @@ import type { BridgeWorkerCredentialResponse, BridgeWorkerRegistrationResponse, } from './protocol.js'; +import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; export interface BridgeWorkerOptions { codeApiUrl: string; token?: string; identity?: BridgeWorkerIdentity; workerId: string; - sandboxEndpoint: string; + /** @deprecated Use runtimeSupervisor for new runtime adapters. */ + sandboxEndpoint?: string; + runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; leaseWaitMs?: number; leaseTransportGraceMs?: number; @@ -67,7 +71,6 @@ const CREDENTIAL_REFRESH_RETRY_DELAY_MS = 100; const SETTLEMENT_RETRY_DELAY_MS = 100; const REJECTION_ACK_GRACE_MS = 30_000; const MAX_SETTLEMENT_ERROR_LENGTH = 4_096; -const RUNTIME_SESSION_PLACEHOLDER = '{runtimeSessionId}'; export function reconnectDelayMs( attempt: number, @@ -121,7 +124,7 @@ export class BridgeWorkspaceQuarantinedError extends Error { export class BridgeWorker { private readonly fetchImpl: typeof fetch; private readonly codeApiUrl: string; - private readonly sandboxEndpoint: string; + private readonly runtimeSupervisor: RuntimeSupervisor; private readonly incarnationId: string; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; @@ -133,9 +136,22 @@ export class BridgeWorker { 'Bridge worker requires a static token or paired identity', ); } + if (options.runtimeSupervisor != null && options.sandboxEndpoint != null) { + throw new BridgeProtocolError( + 'Bridge worker accepts either a runtime supervisor or sandbox endpoint, not both', + ); + } + if (options.runtimeSupervisor == null && !options.sandboxEndpoint?.trim()) { + throw new BridgeProtocolError('Bridge worker requires a runtime supervisor'); + } this.fetchImpl = options.fetchImpl ?? fetch; this.codeApiUrl = normalizedBaseUrl(options.codeApiUrl); - this.sandboxEndpoint = normalizedBaseUrl(options.sandboxEndpoint); + this.runtimeSupervisor = + options.runtimeSupervisor ?? + new EndpointRuntimeSupervisor({ + endpoint: options.sandboxEndpoint ?? '', + statefulWorkspace: options.capabilities.statefulWorkspace, + }); this.incarnationId = options.incarnationId ?? randomBytes(18).toString('base64url'); } @@ -197,6 +213,7 @@ export class BridgeWorker { if (runtimeSessionId.trim().length === 0) { throw new BridgeProtocolError('Runtime session ID is required'); } + await this.runtimeSupervisor.reset(runtimeSessionId, signal); await this.timedRequest( `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, { @@ -558,6 +575,7 @@ export class BridgeWorker { let ambiguousSandboxError: unknown; let sandboxRejectedExecution = false; let sandboxStarted = false; + let runtimeLease: RuntimeLease | undefined; try { credentialMaintenance = this.maintainCredential( assignment, @@ -568,13 +586,18 @@ export class BridgeWorker { credentialMaintenanceError = error; executionController.abort(); }); - const sandboxExecuteUrl = - `${this.sandboxEndpointFor(assignment)}/execute`; - const sandboxSessionId = this.sandboxSessionIdFor(assignment); + runtimeLease = await this.runtimeSupervisor.acquire( + assignment, + executionController.signal, + ); + if (executionController.signal.aborted) { + throw executionController.signal.reason ?? new DOMException('aborted', 'AbortError'); + } + const sandboxExecuteUrl = `${runtimeLease.endpoint.replace(/\/+$/, '')}/execute`; const headers = { ...assignment.request.headers, - ...(sandboxSessionId - ? { 'X-Runtime-Session-Id': sandboxSessionId } + ...(runtimeLease.sessionId + ? { 'X-Runtime-Session-Id': runtimeLease.sessionId } : {}), }; const sandboxRequestBody = JSON.stringify(assignment.request.body); @@ -656,7 +679,8 @@ export class BridgeWorker { await credentialMaintenance; try { if (ambiguousSandboxError != null) { - throw new BridgeWorkspaceQuarantinedError( + throw await this.quarantineWorkspace( + assignment.runtimeSessionId, `Stateful workspace ${assignment.runtimeSessionId} was quarantined after an ambiguous sandbox execution`, ambiguousSandboxError, ); @@ -697,21 +721,13 @@ export class BridgeWorker { } } finally { heartbeatController.abort(); - await heartbeat; - signal?.removeEventListener('abort', abortExecution); - } - } - - private sandboxSessionIdFor( - assignment: BridgeAssignment, - ): string | undefined { - if (assignment.runtimeSessionId != null) { - return assignment.runtimeSessionId; - } - if (this.sandboxEndpoint.includes(RUNTIME_SESSION_PLACEHOLDER)) { - return `assignment-${assignment.assignmentId}`; + try { + await this.releaseRuntimeLease(runtimeLease, assignment); + } finally { + await heartbeat; + signal?.removeEventListener('abort', abortExecution); + } } - return undefined; } private assignmentRemainingMs(assignment: BridgeAssignment): number { @@ -724,28 +740,40 @@ export class BridgeWorker { return Math.max(0, Date.parse(assignment.expiresAt) - Date.now()); } - private sandboxEndpointFor(assignment: BridgeAssignment): string { - if (assignment.runtimeSessionId == null) { - if (!this.sandboxEndpoint.includes(RUNTIME_SESSION_PLACEHOLDER)) { - return this.sandboxEndpoint; - } - return this.sandboxEndpoint.replace( - RUNTIME_SESSION_PLACEHOLDER, - encodeURIComponent(`assignment-${assignment.assignmentId}`), + private async releaseRuntimeLease( + lease: RuntimeLease | undefined, + assignment: BridgeAssignment, + ): Promise { + if (lease?.release == null) return; + try { + await lease.release(); + } catch (error) { + if (assignment.runtimeSessionId == null) throw error; + throw await this.quarantineWorkspace( + assignment.runtimeSessionId, + `Stateful workspace ${assignment.runtimeSessionId} could not release its runtime lease`, + error, ); } - if ( - this.options.capabilities.statefulWorkspace !== true || - !this.sandboxEndpoint.includes(RUNTIME_SESSION_PLACEHOLDER) - ) { - throw new BridgeProtocolError( - 'Stateful assignments require a sandbox endpoint template containing {runtimeSessionId}', + } + + private async quarantineWorkspace( + runtimeSessionId: string | undefined, + message: string, + cause?: unknown, + ): Promise { + if (runtimeSessionId == null) { + return new BridgeWorkspaceQuarantinedError(message, cause); + } + try { + await this.runtimeSupervisor.quarantine(runtimeSessionId, message, cause); + return new BridgeWorkspaceQuarantinedError(message, cause); + } catch (error) { + return new BridgeWorkspaceQuarantinedError( + `${message}; local runtime quarantine could not be confirmed`, + error, ); } - return this.sandboxEndpoint.replace( - RUNTIME_SESSION_PLACEHOLDER, - encodeURIComponent(assignment.runtimeSessionId), - ); } private async delay(ms: number, signal: AbortSignal): Promise { @@ -831,7 +859,8 @@ export class BridgeWorker { ): Promise { if (signal?.aborted === true) { if (assignment.runtimeSessionId != null) { - throw new BridgeWorkspaceQuarantinedError( + throw await this.quarantineWorkspace( + assignment.runtimeSessionId, `Stateful workspace ${assignment.runtimeSessionId} was quarantined before settlement during shutdown`, signal.reason, ); @@ -871,7 +900,8 @@ export class BridgeWorker { assignment.runtimeSessionId != null && settlement.status === 'fulfilled' ) { - throw new BridgeWorkspaceQuarantinedError( + throw await this.quarantineWorkspace( + assignment.runtimeSessionId, `Stateful workspace ${assignment.runtimeSessionId} was quarantined after Code API rejected its fulfilled settlement`, error, ); @@ -894,7 +924,8 @@ export class BridgeWorker { assignment.runtimeSessionId != null && settlement.status === 'fulfilled' ) { - throw new BridgeWorkspaceQuarantinedError( + throw await this.quarantineWorkspace( + assignment.runtimeSessionId, `Stateful workspace ${assignment.runtimeSessionId} was quarantined after ambiguous settlement delivery`, lastError, ); From 733c832680b0e5c8a76c0f24b27440b890fef7b3 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Wed, 2 Sep 2026 04:36:35 +0200 Subject: [PATCH 011/116] fix(runner): fail safe on invalid output size config (#47) --- api/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/config.ts b/api/src/config.ts index 7724ff0c..443745f9 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -53,7 +53,7 @@ export const config = { disable_networking: (process.env.SANDBOX_DISABLE_NETWORKING ?? 'true') === 'true', use_cgroupv2: (process.env.SANDBOX_USE_CGROUPV2 ?? 'true') === 'true', allowed_local_network_port: Number(process.env.SANDBOX_ALLOWED_LOCAL_NETWORK_PORT ?? 0), - output_max_size: Number(process.env.SANDBOX_OUTPUT_MAX_SIZE ?? 1024), + output_max_size: safeInt(process.env.SANDBOX_OUTPUT_MAX_SIZE, 1024), max_process_count: Number(process.env.SANDBOX_MAX_PROCESS_COUNT ?? 64), max_open_files: Number(process.env.SANDBOX_MAX_OPEN_FILES ?? 2048), max_file_size: Number(process.env.SANDBOX_MAX_FILE_SIZE ?? 10000000), From 0e2aec1b0c71d710027c93783cba388f8d0c04c1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 02:26:11 -0400 Subject: [PATCH 012/116] =?UTF-8?q?=F0=9F=A7=B0=20feat:=20Add=20Docker=20R?= =?UTF-8?q?untime=20Supervisor=20(#73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Docker runtime supervisor * fix: preserve Docker runtime lifecycle safety * fix: close interrupted Docker lifecycle paths --- packages/code/README.md | 18 ++ packages/code/src/runtime.test.ts | 209 +++++++++++++++++---- packages/code/src/runtime.ts | 289 +++++++++++++++++++++++++++++- packages/code/src/worker.ts | 44 +++-- 4 files changed, 516 insertions(+), 44 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 82d4fa22..a4f78283 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -37,6 +37,24 @@ Use `--identity ` while pairing and `LIBRECHAT_CODE_IDENTITY_FILE=` while running to override the identity file location. +## Docker runtime supervisor (programmatic adapter) + +`DockerRuntimeSupervisor` is the first self-contained local OCI adapter. It +owns one named container per runtime session, does not publish the runner port, +starts the container with `--network none`, drops every Linux capability, and +sets `no-new-privileges`. The trusted worker invokes the runner only through +`docker exec` to `127.0.0.1` inside that container. The sandbox therefore has +neither an inbound host port nor network egress. + +It is exported for use by a deployment-specific worker launcher. It requires a +runtime image that provides the Code Interpreter `/api/v2/health` and +`/api/v2/execute` endpoints and supports +`SANDBOX_SESSION_WORKSPACE_ENABLED=true`. A dedicated LibreChat runtime image +and CLI selector are the next layer; this adapter intentionally does not turn +an arbitrary image into a supported security boundary. Image-specific Linux +capabilities must be explicitly configured by the trusted launcher; the +default grants none. + ## Static compatibility mode Non-hardened development deployments may still run with a static token: diff --git a/packages/code/src/runtime.test.ts b/packages/code/src/runtime.test.ts index d7bde841..b5ba6075 100644 --- a/packages/code/src/runtime.test.ts +++ b/packages/code/src/runtime.test.ts @@ -1,26 +1,32 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { EndpointRuntimeSupervisor } from './runtime.js'; +import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; -test('endpoint runtime supervisor resolves an isolated endpoint for stateful work', async () => { - const supervisor = new EndpointRuntimeSupervisor({ - endpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2/', - statefulWorkspace: true, - }); +import type { ContainerRuntimeClient } from './runtime.js'; - const lease = await supervisor.acquire({ - protocolVersion: 1, +function assignment(runtimeSessionId?: string) { + return { + protocolVersion: 1 as const, assignmentId: 'assignment-1', workerId: 'worker-1', incarnationId: 'incarnation-1', generation: 1, leaseToken: 'lease-token', expiresAt: new Date().toISOString(), - runtimeSessionId: 'rt/user 1', + ...(runtimeSessionId ? { runtimeSessionId } : {}), request: { body: {}, headers: {} }, + }; +} + +test('endpoint runtime supervisor resolves an isolated endpoint for stateful work', async () => { + const supervisor = new EndpointRuntimeSupervisor({ + endpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2/', + statefulWorkspace: true, }); + const lease = await supervisor.acquire(assignment('rt/user 1')); + assert.equal(lease.sessionId, 'rt/user 1'); assert.equal( lease.endpoint, @@ -35,17 +41,7 @@ test('endpoint runtime supervisor refuses stateful work without an isolated rout }); await assert.rejects( - supervisor.acquire({ - protocolVersion: 1, - assignmentId: 'assignment-1', - workerId: 'worker-1', - incarnationId: 'incarnation-1', - generation: 1, - leaseToken: 'lease-token', - expiresAt: new Date().toISOString(), - runtimeSessionId: 'rt-1', - request: { body: {}, headers: {} }, - }), + supervisor.acquire(assignment('rt-1')), /runtime supervisor endpoint containing/, ); }); @@ -56,16 +52,7 @@ test('endpoint runtime supervisor gives stateless work an ephemeral session rout statefulWorkspace: false, }); - const lease = await supervisor.acquire({ - protocolVersion: 1, - assignmentId: 'assignment-1', - workerId: 'worker-1', - incarnationId: 'incarnation-1', - generation: 1, - leaseToken: 'lease-token', - expiresAt: new Date().toISOString(), - request: { body: {}, headers: {} }, - }); + const lease = await supervisor.acquire(assignment()); assert.equal(lease.sessionId, 'assignment-assignment-1'); assert.equal( @@ -73,3 +60,165 @@ test('endpoint runtime supervisor gives stateless work an ephemeral session rout 'http://127.0.0.1:2000/sessions/assignment-assignment-1/api/v2', ); }); + +test('docker runtime supervisor creates a networkless stateful runtime and executes through its loopback', async () => { + const calls: string[][] = []; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') { + throw new Error('No such container'); + } + if (args[0] === 'run') return 'container-id\n'; + if (args[0] === 'exec' && args.some(value => value.includes('/api/v2/health'))) return '200'; + if (args[0] === 'exec' && args.some(value => value.includes('/api/v2/execute'))) { + const writeOut = args[args.indexOf('--write-out') + 1] ?? ''; + return `{"session_id":"run-1"}${writeOut.replace('%{http_code}', '200')}`; + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + client, + }); + + const lease = await supervisor.acquire(assignment('rt-user-1')); + + const result = await lease.execute?.({ body: '{}', headers: { 'X-Test': '1' } }); + assert.equal(result?.status, 200); + assert.equal(result?.body, '{"session_id":"run-1"}'); + assert.equal(lease.sessionId, 'rt-user-1'); + const run = calls.find(args => args[0] === 'run'); + assert.deepEqual(run?.slice(0, 10), [ + 'run', + '--detach', + '--name', + run?.[3] ?? '', + '--network', + 'none', + '--cap-drop', + 'ALL', + '--security-opt', + 'no-new-privileges:true', + ]); + assert.equal(run?.includes('rt-user-1'), false); + const health = calls.find( + args => args[0] === 'exec' && args.some(value => value.includes('/api/v2/health')), + ); + assert.ok(health?.includes('--max-time')); +}); + +test('docker runtime supervisor rejects malformed runtime response framing', async () => { + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'container' && args[1] === 'inspect') throw new Error('No such container'); + if (args[0] === 'run') return 'container-id\n'; + if (args[0] === 'exec' && args.some(value => value.includes('/api/v2/health'))) return '200'; + if (args[0] === 'exec' && args.some(value => value.includes('/api/v2/execute'))) return 'not framed'; + if (args[0] === 'container' && args[1] === 'rm') return 'removed\n'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ image: 'example/code-runtime:latest', client }); + const lease = await supervisor.acquire(assignment('rt-user-1')); + + const execute = lease.execute; + assert.ok(execute); + await assert.rejects(execute({ body: '{}', headers: {} }), /invalid HTTP response/); +}); + +test('docker runtime supervisor preserves an existing stateful container after a health failure', async () => { + const calls: string[][] = []; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') return 'true\n'; + if (args[0] === 'exec') throw new Error('runner unavailable'); + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + client, + startupTimeoutMs: 1, + }); + + await assert.rejects(supervisor.acquire(assignment('rt-user-1')), /did not become healthy/); + assert.equal(calls.some(args => args[0] === 'container' && args[1] === 'rm'), false); +}); + +test('docker runtime supervisor propagates removal failures and forwards reset cancellation', async () => { + const controller = new AbortController(); + let receivedSignal: AbortSignal | undefined; + const client: ContainerRuntimeClient = { + async run(args, options) { + if (args[0] === 'container' && args[1] === 'rm') { + receivedSignal = options?.signal; + throw new Error('Docker daemon unavailable'); + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ image: 'example/code-runtime:latest', client }); + + await assert.rejects(supervisor.reset('rt-user-1', controller.signal), /Docker daemon unavailable/); + assert.equal(receivedSignal, controller.signal); +}); + +test('docker runtime supervisor cleans up stateless containers after interrupted creation', async () => { + const calls: string[][] = []; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') { + throw new Error('No such container'); + } + if (args[0] === 'run') throw new DOMException('aborted', 'AbortError'); + if (args[0] === 'container' && args[1] === 'rm') return 'removed\n'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ image: 'example/code-runtime:latest', client }); + + await assert.rejects(supervisor.acquire(assignment()), /aborted/); + assert.equal(calls.some(args => args[0] === 'container' && args[1] === 'rm'), true); +}); + +test('docker runtime supervisor ignores only confirmed missing-container removal', async () => { + const client: ContainerRuntimeClient = { + async run() { + throw new Error('Error response from daemon: No such container: runtime'); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ image: 'example/code-runtime:latest', client }); + + await supervisor.reset('rt-user-1'); +}); + +test('docker runtime supervisor destroys stateless and reset stateful runtimes', async () => { + const calls: string[][] = []; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') throw new Error('No such container'); + if (args[0] === 'run') return 'container-id\n'; + if (args[0] === 'exec' && args.some(value => value.includes('/api/v2/health'))) return '200'; + if (args[0] === 'container' && args[1] === 'rm') return 'removed\n'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + client, + }); + + const lease = await supervisor.acquire(assignment()); + await lease.release?.(); + await supervisor.reset('rt-user-1'); + await supervisor.quarantine?.('rt-user-2', 'ambiguous result'); + + const removals = calls.filter(args => args[0] === 'container' && args[1] === 'rm'); + assert.equal(removals.length, 3); + assert.ok(removals.every(args => args[2] === '--force')); +}); diff --git a/packages/code/src/runtime.ts b/packages/code/src/runtime.ts index d65febca..45469a5e 100644 --- a/packages/code/src/runtime.ts +++ b/packages/code/src/runtime.ts @@ -1,13 +1,28 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { spawn } from 'node:child_process'; + import type { BridgeAssignment } from './protocol.js'; export const RUNTIME_SESSION_PLACEHOLDER = '{runtimeSessionId}'; export interface RuntimeLease { - endpoint: string; + endpoint?: string; sessionId?: string; + execute?(request: RuntimeExecutionRequest): Promise; release?(): Promise; } +export interface RuntimeExecutionRequest { + body: string; + headers: Record; + signal?: AbortSignal; +} + +export interface RuntimeExecutionResponse { + status: number; + body: string; +} + export interface RuntimeSupervisor { acquire(assignment: BridgeAssignment, signal?: AbortSignal): Promise; reset(runtimeSessionId: string, signal?: AbortSignal): Promise; @@ -19,6 +34,32 @@ export interface EndpointRuntimeSupervisorOptions { statefulWorkspace: boolean; } +export interface ContainerRuntimeClient { + run(args: string[], options?: ContainerRuntimeRunOptions): Promise; +} + +export interface ContainerRuntimeRunOptions { + input?: string; + signal?: AbortSignal; +} + +export interface DockerRuntimeSupervisorOptions { + image: string; + capabilities?: string[]; + dockerCommand?: string; + runnerPort?: number; + startupTimeoutMs?: number; + healthPath?: string; + client?: ContainerRuntimeClient; +} + +const DEFAULT_RUNNER_PORT = 2000; +const DEFAULT_STARTUP_TIMEOUT_MS = 30_000; +const DEFAULT_HEALTH_PATH = '/api/v2/health'; +const CONTAINER_PREFIX = 'librechat-code-'; +const CAPABILITY_PATTERN = /^[A-Z_]{1,32}$/; +const MAX_DOCKER_COMMAND_OUTPUT_BYTES = 64 * 1024 * 1024; + function normalizedEndpoint(value: string): string { return value.replace(/\/+$/, ''); } @@ -29,6 +70,252 @@ function assignmentSessionId(assignment: BridgeAssignment): string | undefined { return `assignment-${assignment.assignmentId}`; } +function containerSuffix(runtimeSessionId: string): string { + return createHash('sha256').update(runtimeSessionId).digest('hex').slice(0, 24); +} + +function containerName(runtimeSessionId: string): string { + return `${CONTAINER_PREFIX}${containerSuffix(runtimeSessionId)}`; +} + +function isMissingContainerError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + return /(?:no such container|no such object)/i.test(error.message); +} + +class DockerCliClient implements ContainerRuntimeClient { + private readonly command: string; + + constructor(command = 'docker') { + this.command = command; + } + + async run(args: string[], options: ContainerRuntimeRunOptions = {}): Promise { + return await new Promise((resolve, reject) => { + const process = spawn(this.command, args, { + stdio: ['pipe', 'pipe', 'pipe'], + signal: options.signal, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let outputBytes = 0; + const append = (chunks: Buffer[], chunk: Buffer): void => { + outputBytes += chunk.length; + if (outputBytes > MAX_DOCKER_COMMAND_OUTPUT_BYTES) { + process.kill('SIGKILL'); + reject(new Error('Docker runtime command exceeded its output limit')); + return; + } + chunks.push(chunk); + }; + process.stdout.on('data', (chunk: Buffer) => append(stdout, chunk)); + process.stderr.on('data', (chunk: Buffer) => append(stderr, chunk)); + process.stdin.once('error', reject); + process.once('error', reject); + process.once('close', (code) => { + if (code === 0) { + resolve(Buffer.concat(stdout).toString('utf8')); + return; + } + const detail = Buffer.concat(stderr).toString('utf8').slice(0, 4096); + reject(new Error(`Docker runtime command exited ${code ?? 'unknown'}${detail ? `: ${detail}` : ''}`)); + }); + process.stdin.end(options.input); + }); + } +} + +/** + * Local OCI runtime adapter. The supervisor, not the sandbox container, talks + * to Docker. Each runtime has no network, and the trusted worker reaches its + * loopback API through Docker exec; stateful containers survive leases until + * reset or quarantine. + */ +export class DockerRuntimeSupervisor implements RuntimeSupervisor { + private readonly client: ContainerRuntimeClient; + private readonly runnerPort: number; + private readonly startupTimeoutMs: number; + private readonly healthPath: string; + private readonly capabilities: string[]; + + constructor(private readonly options: DockerRuntimeSupervisorOptions) { + if (options.image.trim().length === 0) { + throw new Error('Docker runtime image is required'); + } + if (options.capabilities?.some((capability) => !CAPABILITY_PATTERN.test(capability))) { + throw new Error('Docker runtime capabilities must be uppercase capability names'); + } + this.client = options.client ?? new DockerCliClient(options.dockerCommand); + this.runnerPort = options.runnerPort ?? DEFAULT_RUNNER_PORT; + this.startupTimeoutMs = options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; + this.healthPath = options.healthPath ?? DEFAULT_HEALTH_PATH; + this.capabilities = options.capabilities ?? []; + } + + async acquire(assignment: BridgeAssignment, signal?: AbortSignal): Promise { + const sessionId = assignmentSessionId(assignment); + if (sessionId == null) throw new Error('Runtime assignment ID is required'); + const name = containerName(sessionId); + let created = false; + try { + created = await this.ensureContainer(name, sessionId, signal); + await this.waitForHealth(name, signal); + return { + sessionId, + execute: async (request) => this.execute(name, request), + release: assignment.runtimeSessionId == null ? async () => this.remove(name) : undefined, + }; + } catch (error) { + if (created || assignment.runtimeSessionId == null) await this.remove(name); + throw error; + } + } + + async reset(runtimeSessionId: string, signal?: AbortSignal): Promise { + await this.remove(containerName(runtimeSessionId), signal); + } + + async quarantine( + runtimeSessionId: string, + _reason: string, + _cause?: unknown, + ): Promise { + await this.remove(containerName(runtimeSessionId)); + } + + private async ensureContainer( + name: string, + runtimeSessionId: string, + signal?: AbortSignal, + ): Promise { + const running = await this.containerRunning(name, signal); + if (running) return false; + if (running === false) { + await this.client.run(['start', name], { signal }); + return false; + } + await this.client.run( + [ + 'run', + '--detach', + '--name', + name, + '--network', + 'none', + '--cap-drop', + 'ALL', + ...this.capabilities.flatMap((capability) => ['--cap-add', capability]), + '--security-opt', + 'no-new-privileges:true', + '--label', + 'com.librechat.code.runtime=true', + '--label', + `com.librechat.code.runtime-hash=${containerSuffix(runtimeSessionId)}`, + '--env', + 'SANDBOX_SESSION_WORKSPACE_ENABLED=true', + this.options.image, + ], + { signal }, + ); + return true; + } + + private async containerRunning(name: string, signal?: AbortSignal): Promise { + try { + const value = await this.client.run( + ['container', 'inspect', '--format', '{{.State.Running}}', name], + { signal }, + ); + return value.trim() === 'true'; + } catch (error) { + if (isMissingContainerError(error)) return undefined; + throw error; + } + } + + private async waitForHealth(name: string, signal?: AbortSignal): Promise { + const deadline = Date.now() + this.startupTimeoutMs; + let lastError: unknown; + while (Date.now() < deadline) { + if (signal?.aborted) throw signal.reason ?? new DOMException('aborted', 'AbortError'); + try { + const remainingMs = Math.max(1, deadline - Date.now()); + const status = await this.client.run( + [ + 'exec', + name, + 'curl', + '--silent', + '--show-error', + '--max-time', + (remainingMs / 1000).toFixed(3), + '--output', + '/dev/null', + '--write-out', + '%{http_code}', + `http://127.0.0.1:${this.runnerPort}${this.healthPath}`, + ], + { signal }, + ); + if (status.trim() === '200') return; + lastError = new Error(`Runtime health check returned HTTP ${status.trim() || 'unknown'}`); + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error( + `Docker runtime did not become healthy within ${this.startupTimeoutMs}ms${ + lastError instanceof Error ? `: ${lastError.message}` : '' + }`, + ); + } + + private async execute( + name: string, + request: RuntimeExecutionRequest, + ): Promise { + if (Object.entries(request.headers).some(([name, value]) => name.includes('\r') || name.includes('\n') || value.includes('\r') || value.includes('\n'))) { + throw new Error('Runtime request headers cannot contain line breaks'); + } + const marker = randomBytes(32).toString('hex'); + const output = await this.client.run( + [ + 'exec', + '--interactive', + name, + 'curl', + '--silent', + '--show-error', + '--request', + 'POST', + ...Object.entries(request.headers).flatMap(([name, value]) => ['--header', `${name}: ${value}`]), + '--data-binary', + '@-', + '--write-out', + `\n${marker}%{http_code}`, + `http://127.0.0.1:${this.runnerPort}/api/v2/execute`, + ], + { input: request.body, signal: request.signal }, + ); + const suffix = new RegExp(`\\n${marker}(\\d{3})$`); + const match = output.match(suffix); + if (match?.[1] == null) throw new Error('Docker runtime returned an invalid HTTP response'); + return { + status: Number(match[1]), + body: output.slice(0, -match[0].length), + }; + } + + private async remove(name: string, signal?: AbortSignal): Promise { + try { + await this.client.run(['container', 'rm', '--force', name], { signal }); + } catch (error) { + if (!isMissingContainerError(error)) throw error; + } + } +} + /** * Compatibility adapter for an already-running loopback sandbox supervisor. * New runtime adapters own provisioning and return the same lease shape. diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index b4afe1b6..f4abfeba 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -593,7 +593,6 @@ export class BridgeWorker { if (executionController.signal.aborted) { throw executionController.signal.reason ?? new DOMException('aborted', 'AbortError'); } - const sandboxExecuteUrl = `${runtimeLease.endpoint.replace(/\/+$/, '')}/execute`; const headers = { ...assignment.request.headers, ...(runtimeLease.sessionId @@ -607,28 +606,25 @@ export class BridgeWorker { ); } sandboxStarted = true; - const response = await this.fetchImpl( - sandboxExecuteUrl, + const response = await this.executeRuntime( + runtimeLease, + sandboxRequestBody, { - method: 'POST', - headers: { - ...headers, - 'Content-Type': 'application/json', - }, - body: sandboxRequestBody, - signal: executionController.signal, + ...headers, + 'Content-Type': 'application/json', }, + executionController.signal, ); let payload: object = {}; try { - payload = (await response.json()) as object; + payload = JSON.parse(response.body) as object; } catch (error) { - if (response.ok) throw error; + if (response.status >= 200 && response.status < 300) throw error; } if (credentialMaintenanceError != null) { throw credentialMaintenanceError; } - if (!response.ok) { + if (response.status < 200 || response.status >= 300) { sandboxRejectedExecution = response.status >= 400 && response.status < 500 && @@ -740,6 +736,28 @@ export class BridgeWorker { return Math.max(0, Date.parse(assignment.expiresAt) - Date.now()); } + private async executeRuntime( + lease: RuntimeLease, + body: string, + headers: Record, + signal: AbortSignal, + ): Promise<{ status: number; body: string }> { + if (lease.execute != null) { + return await lease.execute({ body, headers, signal }); + } + if (lease.endpoint == null) { + throw new BridgeProtocolError('Runtime lease does not provide an execution transport'); + } + const endpoint = lease.endpoint.replace(/\/+$/, ''); + const response = await this.fetchImpl(`${endpoint}/execute`, { + method: 'POST', + headers, + body, + signal, + }); + return { status: response.status, body: await response.text() }; + } + private async releaseRuntimeLease( lease: RuntimeLease | undefined, assignment: BridgeAssignment, From 5739d26aca0b27706e0b1fbe07b53a210f280cee Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 02:35:14 -0400 Subject: [PATCH 013/116] =?UTF-8?q?=F0=9F=A7=B0=20feat:=20Add=20Docker=20S?= =?UTF-8?q?upervisor=20CLI=20Mode=20(#74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Docker supervisor CLI mode * fix: allow image-free Docker workspace reset --- packages/code/README.md | 15 +++++++++++ packages/code/src/cli.test.ts | 43 +++++++++++++++++++++++++++++++ packages/code/src/cli.ts | 35 ++++++++++++++++++++----- packages/code/src/runtime.test.ts | 12 +++++++++ packages/code/src/runtime.ts | 10 ++++--- 5 files changed, 104 insertions(+), 11 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index a4f78283..3bbea73c 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -55,6 +55,21 @@ an arbitrary image into a supported security boundary. Image-specific Linux capabilities must be explicitly configured by the trusted launcher; the default grants none. +To enable it from the bundled CLI, the host must give the worker access to its +local Docker daemon and explicitly select a known runtime image: + +```bash +LIBRECHAT_CODE_RUNTIME_SUPERVISOR=docker \ +LIBRECHAT_CODE_RUNTIME_IMAGE=ghcr.io/librechat-ai/code-interpreter-runtime:tag \ +LIBRECHAT_CODE_STATEFUL_WORKSPACE=true \ +librechat-code run +``` + +The image reference above is illustrative until the corresponding published +runtime image ships. Docker mode never binds a runner port on the VM. Do not +mount the Docker socket into the sandbox; only the trusted worker may control +the daemon. + ## Static compatibility mode Non-hardened development deployments may still run with a static token: diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 179bf813..4684775f 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -47,3 +47,46 @@ test('CLI rejects invalid advertised capabilities before registration', () => { /LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid/, ); }); + +test('CLI rejects an unknown runtime supervisor before entering the run loop', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'podman', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be either endpoint or docker/, + ); +}); + +test('CLI requires a runtime image for Docker supervision', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /LIBRECHAT_CODE_RUNTIME_IMAGE is required/); +}); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index b93374fc..f6fb3fb1 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -8,7 +8,7 @@ import { saveBridgeIdentity, } from './storage.js'; import { BridgeWorker } from './worker.js'; -import { EndpointRuntimeSupervisor } from './runtime.js'; +import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -86,10 +86,21 @@ async function run(runtimeSessionId?: string): Promise { const statefulWorkspace = process.env.LIBRECHAT_CODE_STATEFUL_WORKSPACE?.trim().toLowerCase() === 'true'; + const runtimeMode = + process.env.LIBRECHAT_CODE_RUNTIME_SUPERVISOR?.trim().toLowerCase() ?? 'endpoint'; + if (runtimeMode !== 'endpoint' && runtimeMode !== 'docker') { + throw new Error( + 'LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be either endpoint or docker', + ); + } const sandboxEndpoint = process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? 'http://127.0.0.1:2000/api/v2'; - if (statefulWorkspace && !sandboxEndpoint.includes('{runtimeSessionId}')) { + if ( + runtimeMode === 'endpoint' && + statefulWorkspace && + !sandboxEndpoint.includes('{runtimeSessionId}') + ) { throw new Error( 'LIBRECHAT_CODE_STATEFUL_WORKSPACE requires LIBRECHAT_CODE_SANDBOX_ENDPOINT to contain {runtimeSessionId}', ); @@ -103,7 +114,9 @@ async function run(runtimeSessionId?: string): Promise { : undefined; const capabilities = { statefulWorkspace, - sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? 'nsjail', + sandboxProfile: + process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? + (runtimeMode === 'docker' ? 'oci-docker' : 'nsjail'), runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), policyDigest: createHash('sha256').update(policy).digest('hex'), }; @@ -120,10 +133,18 @@ async function run(runtimeSessionId?: string): Promise { token: configuredToken, identity: workerIdentity, workerId, - runtimeSupervisor: new EndpointRuntimeSupervisor({ - endpoint: sandboxEndpoint, - statefulWorkspace, - }), + runtimeSupervisor: + runtimeMode === 'docker' + ? new DockerRuntimeSupervisor({ + image: + runtimeSessionId == null + ? required('LIBRECHAT_CODE_RUNTIME_IMAGE') + : process.env.LIBRECHAT_CODE_RUNTIME_IMAGE?.trim(), + }) + : new EndpointRuntimeSupervisor({ + endpoint: sandboxEndpoint, + statefulWorkspace, + }), capabilities, onIdentityChange: pairedIdentity && identityPath diff --git a/packages/code/src/runtime.test.ts b/packages/code/src/runtime.test.ts index b5ba6075..9963efc1 100644 --- a/packages/code/src/runtime.test.ts +++ b/packages/code/src/runtime.test.ts @@ -196,6 +196,18 @@ test('docker runtime supervisor ignores only confirmed missing-container removal await supervisor.reset('rt-user-1'); }); +test('docker runtime supervisor resets a workspace without a configured image', async () => { + const client: ContainerRuntimeClient = { + async run(args) { + assert.deepEqual(args.slice(0, 3), ['container', 'rm', '--force']); + return 'removed\n'; + }, + }; + const supervisor = new DockerRuntimeSupervisor({ client }); + + await supervisor.reset('rt-user-1'); +}); + test('docker runtime supervisor destroys stateless and reset stateful runtimes', async () => { const calls: string[][] = []; const client: ContainerRuntimeClient = { diff --git a/packages/code/src/runtime.ts b/packages/code/src/runtime.ts index 45469a5e..89eaca95 100644 --- a/packages/code/src/runtime.ts +++ b/packages/code/src/runtime.ts @@ -44,7 +44,7 @@ export interface ContainerRuntimeRunOptions { } export interface DockerRuntimeSupervisorOptions { - image: string; + image?: string; capabilities?: string[]; dockerCommand?: string; runnerPort?: number; @@ -139,8 +139,8 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { private readonly capabilities: string[]; constructor(private readonly options: DockerRuntimeSupervisorOptions) { - if (options.image.trim().length === 0) { - throw new Error('Docker runtime image is required'); + if (options.image != null && options.image.trim().length === 0) { + throw new Error('Docker runtime image cannot be empty'); } if (options.capabilities?.some((capability) => !CAPABILITY_PATTERN.test(capability))) { throw new Error('Docker runtime capabilities must be uppercase capability names'); @@ -188,6 +188,8 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { runtimeSessionId: string, signal?: AbortSignal, ): Promise { + const image = this.options.image?.trim(); + if (!image) throw new Error('Docker runtime image is required for acquisition'); const running = await this.containerRunning(name, signal); if (running) return false; if (running === false) { @@ -213,7 +215,7 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { `com.librechat.code.runtime-hash=${containerSuffix(runtimeSessionId)}`, '--env', 'SANDBOX_SESSION_WORKSPACE_ENABLED=true', - this.options.image, + image, ], { signal }, ); From 0640e7202b4c250a19dcee2e46e203627da5fc60 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 10:43:15 -0400 Subject: [PATCH 014/116] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20feat:=20Add=20t?= =?UTF-8?q?agged=20releases=20(#77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployments currently have to track main, which advances whenever an internal snapshot is merged. Cut versioned tags instead, each carrying the packaged Helm chart so a deployment can pin one. The tag is the app version and must match helm/codeapi/Chart.yaml appVersion, so a deployed chart cannot report a version no release ever carried. `latest` moves only for the highest stable tag, and the chart is packaged before the tag is created so a rate-limited subchart pull leaves the version unused and the run retryable. Closes #63 --- .github/release.yml | 31 +++++ .github/workflows/release.yml | 252 ++++++++++++++++++++++++++++++++++ .gitignore | 1 + CONTRIBUTING.md | 8 ++ README.md | 20 +++ docs/RELEASING.md | 71 ++++++++++ 6 files changed, 383 insertions(+) create mode 100644 .github/release.yml create mode 100644 .github/workflows/release.yml create mode 100644 docs/RELEASING.md diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..0a59193a --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,31 @@ +# Categories for the changelog `gh release create --generate-notes` appends to +# every release body (see .github/workflows/release.yml). Labels are matched +# against the merged pull requests in the range; anything unlabelled lands in +# "Other changes" rather than being dropped, which matters here because sync +# pull requests from the internal monorepo usually carry no labels. +changelog: + exclude: + labels: + - duplicate + - invalid + - wontfix + categories: + - title: Security + labels: + - security + - title: Features + labels: + - enhancement + - feature + - title: Fixes + labels: + - bug + - title: Documentation + labels: + - documentation + - title: Dependencies + labels: + - dependencies + - title: Other changes + labels: + - '*' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..55c7a116 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,252 @@ +# Cuts tagged releases for the public code-interpreter repo. Like ci.yml this +# file is inert inside the monorepo — GitHub only runs workflows from the repo +# root — and becomes a root workflow in the published repo. +# +# Two entry points feed one job: +# +# * workflow_dispatch — pick a version in the Actions UI. The chart is +# packaged before the tag is created, so a packaging failure aborts while +# the release is still un-cut and the version is still free to reuse. +# * push of a v* tag — for tags cut locally with `git tag -a … && git push`. +# Tags this workflow pushes itself carry GITHUB_TOKEN, and GitHub does not +# re-trigger workflows for those, so the two paths never double-publish. +# +# `main` accepts no direct pushes (see CONTRIBUTING.md), but the branch +# ruleset does not cover tags, so the job can create them. GITHUB_TOKEN +# defaults to read-only in this repository; the explicit `contents: write` +# below is what lets the tag push and the release upload through. +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version to release, e.g. v2.0.0 or v2.1.0-rc1. Must match helm/codeapi/Chart.yaml appVersion.' + required: true + type: string + draft: + description: 'Publish as a draft so the notes can be edited before going public' + type: boolean + default: false + push: + tags: + - 'v*' + +permissions: + contents: write + +concurrency: + group: release-${{ github.event.inputs.version || github.ref_name }} + cancel-in-progress: false + +jobs: + release: + name: Tag and publish + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + # Full history and tags: resolving whether this release is the newest + # stable one compares it against every other tag in the repository. + fetch-depth: 0 + + - name: Resolve and validate version + id: version + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} + INPUT_DRAFT: ${{ github.event.inputs.draft }} + REF_NAME: ${{ github.ref_name }} + REF_TYPE: ${{ github.ref_type }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + # Releases describe what shipped to main. Dispatching from a topic + # branch would tag a commit that is not on the release line. + if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then + echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" + exit 1 + fi + VERSION="$INPUT_VERSION" + else + VERSION="$REF_NAME" + fi + + # A bare "2.0.0" typed into the dispatch box is accepted; everything + # downstream works with the v-prefixed form the tag actually uses. + case "$VERSION" in + v*) ;; + *) VERSION="v$VERSION" ;; + esac + + if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Release tags must be v.. or v..-rcN, for example v2.0.0 or v2.1.0-rc1 (got '$VERSION')" + exit 1 + fi + + # v2.1.0-rc1 -> 2.1.0. Release candidates carry the version they are + # candidates for, so they compare against the same appVersion. + BASE_VERSION="${VERSION%%-rc*}" + BASE_VERSION="${BASE_VERSION#v}" + + read_chart_field() { + grep -m1 "^$1:" helm/codeapi/Chart.yaml \ + | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" + } + APP_VERSION="$(read_chart_field appVersion)" + CHART_VERSION="$(read_chart_field version)" + + # The tag is the app version. Requiring the bump to have landed on + # main first keeps a deployed chart from reporting a version that no + # release ever carried. + if [ "$APP_VERSION" != "$BASE_VERSION" ]; then + echo "::error::Tag $VERSION does not match helm/codeapi/Chart.yaml appVersion ($APP_VERSION). Land the appVersion bump on main before releasing." + exit 1 + fi + + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." + exit 1 + fi + + case "$VERSION" in + *-rc*) PRERELEASE=true ;; + *) PRERELEASE=false ;; + esac + + # `latest` moves only when this is the highest stable version, so + # re-cutting an older patch cannot drag it backwards. The tag under + # dispatch does not exist yet, hence adding it to the comparison. + LATEST=false + if [ "$PRERELEASE" = "false" ]; then + HIGHEST_STABLE="$( + { + git tag --list 'v[0-9]*' + printf '%s\n' "$VERSION" + } \ + | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ + | sort -V \ + | tail -n 1 + )" + if [ "$HIGHEST_STABLE" = "$VERSION" ]; then + LATEST=true + fi + fi + + DRAFT=false + if [ "$INPUT_DRAFT" = "true" ]; then + DRAFT=true + fi + + { + echo "version=$VERSION" + echo "base_version=$BASE_VERSION" + echo "app_version=$APP_VERSION" + echo "chart_version=$CHART_VERSION" + echo "prerelease=$PRERELEASE" + echo "latest=$LATEST" + echo "draft=$DRAFT" + } >> "$GITHUB_OUTPUT" + + echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" + + # helm is preinstalled on ubuntu-latest, the same way the chart tests in + # ci.yml depend on it. + - name: Package Helm chart + id: chart + run: | + set -euo pipefail + + # Subcharts resolve through the Bitnami OCI mirror on Docker Hub, + # which rate-limits anonymous pulls. A transient 429 should cost a + # retry, not the release. + for attempt in 1 2 3; do + if helm dependency update helm/codeapi; then + break + fi + if [ "$attempt" = 3 ]; then + echo "::error::helm dependency update failed after 3 attempts" + exit 1 + fi + sleep $(( attempt * 15 )) + done + + helm package helm/codeapi --destination dist + + CHART_PATH="$(ls dist/codeapi-*.tgz)" + { + echo "path=$CHART_PATH" + echo "name=$(basename "$CHART_PATH")" + } >> "$GITHUB_OUTPUT" + + - name: Create tag + if: github.event_name == 'workflow_dispatch' + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git tag -a "$VERSION" -m "$VERSION" + git push origin "refs/tags/$VERSION" + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ steps.version.outputs.version }} + APP_VERSION: ${{ steps.version.outputs.app_version }} + CHART_VERSION: ${{ steps.version.outputs.chart_version }} + CHART_PATH: ${{ steps.chart.outputs.path }} + CHART_NAME: ${{ steps.chart.outputs.name }} + PRERELEASE: ${{ steps.version.outputs.prerelease }} + LATEST: ${{ steps.version.outputs.latest }} + DRAFT: ${{ steps.version.outputs.draft }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + run: | + set -euo pipefail + + if gh release view "$VERSION" >/dev/null 2>&1; then + echo "::error::Release $VERSION already exists" + exit 1 + fi + + # Quoted heredoc so the markdown backticks stay literal; the + # placeholders are filled in afterwards. + cat > release-notes.md <<'NOTES' + Pin deployments to this tag instead of tracking `main`: + + ```bash + git clone --branch __VERSION__ --depth 1 __REPO_URL__.git + ``` + + The attached `__CHART_NAME__` is the packaged Helm chart (chart `__CHART_VERSION__`, appVersion `__APP_VERSION__`) with its Redis and MinIO subcharts vendored, so it installs without adding any chart repositories: + + ```bash + helm install codeapi ./__CHART_NAME__ -f my-values.yaml + ``` + + Chart configuration is documented in [helm/codeapi/README.md](__REPO_URL__/blob/__VERSION__/helm/codeapi/README.md). + NOTES + + sed -i \ + -e "s|__VERSION__|$VERSION|g" \ + -e "s|__REPO_URL__|$REPO_URL|g" \ + -e "s|__CHART_NAME__|$CHART_NAME|g" \ + -e "s|__CHART_VERSION__|$CHART_VERSION|g" \ + -e "s|__APP_VERSION__|$APP_VERSION|g" \ + release-notes.md + + # --generate-notes appends the merged-pull-request changelog below + # the body from --notes-file, categorised per .github/release.yml. + gh release create "$VERSION" \ + --title "$VERSION" \ + --notes-file release-notes.md \ + --generate-notes \ + --verify-tag \ + --prerelease="$PRERELEASE" \ + --latest="$LATEST" \ + --draft="$DRAFT" \ + "$CHART_PATH#Helm chart ($CHART_NAME)" diff --git a/.gitignore b/.gitignore index a1a0c6ed..958b3332 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ packages/*/dist/ # Helm artifacts helm/*/charts/*.tgz helm/*/Chart.lock +/dist/ # Local sandbox runtime data (docker volume mount) data/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 441ba55d..45c85202 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,14 @@ Practical consequences: - **History is snapshot-based.** Commits here intentionally do not mirror the internal commit history. +## Releases + +Tagged releases are cut from `main` as `vMAJOR.MINOR.PATCH` (with `-rcN` for +release candidates), and each one carries the packaged Helm chart. The version +comes from `helm/codeapi/Chart.yaml`'s `appVersion`, so a version bump lands on +`main` through the pull request flow above before it can be released. See +[docs/RELEASING.md](docs/RELEASING.md) for the full process. + ## Development See the [README](README.md) for the architecture overview and diff --git a/README.md b/README.md index 384e013a..113cbb0e 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,26 @@ privilege, keep hosts patched, and deploy responsibly. If you believe you have found a vulnerability, please report it privately rather than opening a public issue (see [CONTRIBUTING](CONTRIBUTING.md)). +## Releases + +Deployments should pin a [tagged release](https://github.com/LibreChat-AI/code-interpreter/releases) +rather than track `main`, which moves whenever an internal snapshot is merged: + +```bash +git clone --branch v2.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git +``` + +Every release attaches `codeapi-.tgz`, the packaged Helm chart +with its Redis and MinIO subcharts vendored: + +```bash +helm install codeapi ./codeapi-0.3.0.tgz -f my-values.yaml +``` + +Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as +pre-releases. See [docs/RELEASING.md](docs/RELEASING.md) for how releases are +cut. + ## Local Development ```bash diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 00000000..dd00c96d --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,71 @@ +# Releasing + +Deployments should track a tag, not `main`. This document covers how those +tags are cut. + +## Versioning + +A release is named `vMAJOR.MINOR.PATCH`, optionally with a `-rcN` suffix for a +release candidate — `v2.0.0`, `v2.1.0-rc1`. That version is the **app +version**: `helm/codeapi/Chart.yaml`'s `appVersion` is its source of truth, and +the release workflow refuses any tag that disagrees with it. A release +candidate carries the version it is a candidate for, so `v2.1.0-rc1` also +requires `appVersion: "2.1.0"`. + +Two other version numbers are deliberately independent: + +- `helm/codeapi/Chart.yaml`'s `version` is the **chart** version. Bump it when + the chart's templates or values change, not when the app changes. It names + the packaged chart attached to the release (`codeapi-.tgz`). +- `service/package.json`'s `version` tracks the Lambda service package alone. + +By convention `api/package.json`'s `version` is kept in step with `appVersion`, +so the API package and the tag agree. Nothing enforces it. + +## Cutting a release + +1. Land the `appVersion` bump on `main` first. `main` takes no direct pushes + (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so it arrives through a sync + pull request from the internal monorepo or a community pull request. Bump + the chart `version` too if the chart changed. +2. Run the **Release** workflow from the Actions tab against `main`, entering + the version (`v2.1.0`). Tick *draft* to review the generated notes before + they go public. + +The workflow validates the version, packages the Helm chart, then creates the +annotated tag and publishes the release. Packaging runs before tagging so a +failure — a rate-limited subchart pull, most likely — leaves the version +unused and the run safe to retry. + +A tag pushed by hand works as well, and takes the same path from validation +onward: + +```bash +git checkout main && git pull +git tag -a v2.1.0 -m v2.1.0 +git push origin v2.1.0 +``` + +## What the release contains + +- The tag, so a deployment can pin a commit. +- Notes: a preamble on pinning and installing, followed by the merged + pull requests since the previous tag, categorised per + [.github/release.yml](../.github/release.yml). +- `codeapi-.tgz`, the packaged Helm chart with its Redis and + MinIO subcharts vendored, so it installs without adding chart repositories. + +Release candidates are marked as pre-releases. The *Latest* badge moves only +when the release is stable **and** is the highest stable version in the +repository, so re-cutting an older patch cannot drag it backwards. + +## If a release goes wrong + +Delete the release and its tag, then re-run the workflow: + +```bash +gh release delete v2.1.0 --cleanup-tag --yes +``` + +Republishing the same version is only safe while nobody has deployed it. Once +a tag is public, ship a new patch instead. From 50368e0365ae7add8f8148b3d351563433a7e97a Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 10:45:37 -0400 Subject: [PATCH 015/116] =?UTF-8?q?=F0=9F=9B=96=20feat:=20Add=20Local=20Ns?= =?UTF-8?q?Jail=20Runtime=20Profile=20(#78)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add local NsJail runtime profile * fix: refresh stale local runtimes * fix: surface local runtime state loss --- api/Dockerfile | 17 +++ packages/code/README.md | 46 +++++- packages/code/src/cli.test.ts | 75 +++++++++- packages/code/src/cli.ts | 61 +++++++- packages/code/src/runtime.test.ts | 211 ++++++++++++++++++++++++++- packages/code/src/runtime.ts | 233 +++++++++++++++++++++++++----- 6 files changed, 593 insertions(+), 50 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 3526e98e..61a0bd08 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -171,6 +171,23 @@ ENV PORT=8080 \ EXPOSE 8080/tcp ENTRYPOINT ["/sandbox_api/entrypoint.sh"] +# Local direct-NsJail runtime used by @librechat/code's Docker supervisor. +# Runtime packages are mounted read-only at /pkgs, matching docker-compose.mac. +# This profile shares the Docker Desktop VM kernel and is for operator-trusted +# local/BYOM development; production untrusted execution should retain the +# separate MicroVM boundary described in the repository security guidance. +FROM sandbox-build AS local-oci-runtime + +ENV PORT=2000 \ + SANDBOX_PACKAGES_DIRECTORY=/pkgs \ + SANDBOX_OUTPUT_MAX_SIZE=65536 \ + SANDBOX_SESSION_WORKSPACE_ENABLED=true \ + SANDBOX_USE_CGROUPV2=false \ + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP=false + +EXPOSE 2000/tcp +ENTRYPOINT ["/sandbox_api/entrypoint.sh"] + # ============================================================================ # Stage 3: Build the Rust launcher binary (Fedora for libkrun ABI) # ============================================================================ diff --git a/packages/code/README.md b/packages/code/README.md index 3bbea73c..8e6c63b4 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -46,14 +46,14 @@ sets `no-new-privileges`. The trusted worker invokes the runner only through `docker exec` to `127.0.0.1` inside that container. The sandbox therefore has neither an inbound host port nor network egress. -It is exported for use by a deployment-specific worker launcher. It requires a +It requires a runtime image that provides the Code Interpreter `/api/v2/health` and `/api/v2/execute` endpoints and supports -`SANDBOX_SESSION_WORKSPACE_ENABLED=true`. A dedicated LibreChat runtime image -and CLI selector are the next layer; this adapter intentionally does not turn -an arbitrary image into a supported security boundary. Image-specific Linux -capabilities must be explicitly configured by the trusted launcher; the -default grants none. +`SANDBOX_SESSION_WORKSPACE_ENABLED=true`. The repository's +`local-oci-runtime` target supplies that API for the direct-NsJail macOS +profile. This adapter intentionally does not turn an arbitrary image into a +supported security boundary. Image-specific Linux capabilities must be +explicitly configured by the trusted launcher; the default grants none. To enable it from the bundled CLI, the host must give the worker access to its local Docker daemon and explicitly select a known runtime image: @@ -70,6 +70,40 @@ runtime image ships. Docker mode never binds a runner port on the VM. Do not mount the Docker socket into the sandbox; only the trusted worker may control the daemon. +For local Docker Desktop development, build the direct-NsJail target and use +the same capability and seccomp policy as `docker-compose.mac.yml`: + +```bash +docker build --target local-oci-runtime \ + -t librechat-code-runtime:local -f api/Dockerfile . + +LIBRECHAT_CODE_RUNTIME_SUPERVISOR=docker-macos-nsjail \ +LIBRECHAT_CODE_RUNTIME_IMAGE=librechat-code-runtime:local \ +LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE=./seccomp/nsjail.json \ +LIBRECHAT_CODE_DOCKER_PACKAGES_PATH=./data/pkgs \ +LIBRECHAT_CODE_STATEFUL_WORKSPACE=true \ +librechat-code run +``` + +The packages directory must already be populated using the repository's +package-init workflow. The worker mounts it read-only into each runtime. +Changing the image, package path, capabilities, seccomp contents, or other +confinement settings discards any surviving session container; the current +assignment fails explicitly so the lost workspace is never +presented as continuous state. Likewise, Docker Desktop remounts a fresh tmpfs +when this container restarts, so the profile discards a stopped container and +reports state loss instead of restarting it. The next assignment starts a new +environment. Treat profile changes and Docker restarts as environment resets +and preserve any needed workspace contents first. + +This first local profile supports inline request files. By-reference inputs and +generated-file uploads require a worker-mediated file relay and are not yet +supported; the runtime remains networkless rather than opening general egress +to reach a file server. +Direct NsJail shares the Docker Desktop VM kernel and is suitable for local or +operator-trusted development. Use a separate VM or MicroVM boundary for +internet-facing execution of code from untrusted users. + ## Static compatibility mode Non-hardened development deployments may still run with a static token: diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 4684775f..82701e42 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -67,7 +67,7 @@ test('CLI rejects an unknown runtime supervisor before entering the run loop', ( assert.notEqual(result.status, 0); assert.match( result.stderr, - /LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be either endpoint or docker/, + /LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be endpoint, docker, or docker-macos-nsjail/, ); }); @@ -90,3 +90,76 @@ test('CLI requires a runtime image for Docker supervision', () => { assert.notEqual(result.status, 0); assert.match(result.stderr, /LIBRECHAT_CODE_RUNTIME_IMAGE is required/); }); + +test('CLI requires the macOS NsJail seccomp profile', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-macos-nsjail', + LIBRECHAT_CODE_RUNTIME_IMAGE: 'example/runtime:latest', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE is required/); +}); + +test('CLI requires a package mount for the macOS NsJail profile', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-macos-nsjail', + LIBRECHAT_CODE_RUNTIME_IMAGE: 'example/runtime:latest', + LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE: './seccomp/nsjail.json', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /LIBRECHAT_CODE_DOCKER_PACKAGES_PATH is required/); +}); + +test('CLI reset does not require Docker runtime launch inputs', () => { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'reset-workspace', + 'runtime-session-1', + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-macos-nsjail', + LIBRECHAT_CODE_RUNTIME_IMAGE: undefined, + LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE: undefined, + LIBRECHAT_CODE_DOCKER_PACKAGES_PATH: undefined, + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.doesNotMatch( + result.stderr, + /LIBRECHAT_CODE_(?:RUNTIME_IMAGE|DOCKER_SECCOMP_PROFILE|DOCKER_PACKAGES_PATH) is required/, + ); +}); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index f6fb3fb1..457af16f 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,5 +1,7 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { @@ -29,6 +31,23 @@ function list(value: string | undefined): string[] { ); } +const MACOS_NSJAIL_CAPABILITIES = [ + 'SYS_ADMIN', + 'SYS_CHROOT', + 'SYS_PTRACE', + 'SETUID', + 'SETGID', + 'NET_ADMIN', + 'DAC_OVERRIDE', + 'DAC_READ_SEARCH', + 'CHOWN', + 'FOWNER', + 'FSETID', + 'KILL', + 'SETFCAP', + 'MKNOD', +]; + function option(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index >= 0) return args[index + 1]; @@ -88,9 +107,13 @@ async function run(runtimeSessionId?: string): Promise { 'true'; const runtimeMode = process.env.LIBRECHAT_CODE_RUNTIME_SUPERVISOR?.trim().toLowerCase() ?? 'endpoint'; - if (runtimeMode !== 'endpoint' && runtimeMode !== 'docker') { + if ( + runtimeMode !== 'endpoint' && + runtimeMode !== 'docker' && + runtimeMode !== 'docker-macos-nsjail' + ) { throw new Error( - 'LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be either endpoint or docker', + 'LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be endpoint, docker, or docker-macos-nsjail', ); } const sandboxEndpoint = @@ -116,7 +139,7 @@ async function run(runtimeSessionId?: string): Promise { statefulWorkspace, sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? - (runtimeMode === 'docker' ? 'oci-docker' : 'nsjail'), + (runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), policyDigest: createHash('sha256').update(policy).digest('hex'), }; @@ -134,12 +157,42 @@ async function run(runtimeSessionId?: string): Promise { identity: workerIdentity, workerId, runtimeSupervisor: - runtimeMode === 'docker' + runtimeMode !== 'endpoint' ? new DockerRuntimeSupervisor({ image: runtimeSessionId == null ? required('LIBRECHAT_CODE_RUNTIME_IMAGE') : process.env.LIBRECHAT_CODE_RUNTIME_IMAGE?.trim(), + ...(runtimeMode === 'docker-macos-nsjail' && runtimeSessionId == null + ? (() => { + const seccompProfile = resolve( + required('LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE'), + ); + const packagesPath = resolve( + required('LIBRECHAT_CODE_DOCKER_PACKAGES_PATH'), + ); + return { + capabilities: MACOS_NSJAIL_CAPABILITIES, + securityOptions: [`seccomp=${seccompProfile}`], + profileRevision: createHash('sha256') + .update(readFileSync(seccompProfile)) + .digest('hex'), + restartStoppedContainers: false, + bindMounts: [ + { + source: packagesPath, + target: '/pkgs', + readOnly: true, + }, + ], + httpClient: 'bun', + environment: { + SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', + }, + }; + })() + : {}), }) : new EndpointRuntimeSupervisor({ endpoint: sandboxEndpoint, diff --git a/packages/code/src/runtime.test.ts b/packages/code/src/runtime.test.ts index 9963efc1..113f93bb 100644 --- a/packages/code/src/runtime.test.ts +++ b/packages/code/src/runtime.test.ts @@ -109,6 +109,54 @@ test('docker runtime supervisor creates a networkless stateful runtime and execu assert.ok(health?.includes('--max-time')); }); +test('docker runtime supervisor applies an explicit macOS NsJail confinement profile', async () => { + const calls: string[][] = []; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') throw new Error('No such container'); + if (args[0] === 'run') return 'container-id\n'; + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + client, + capabilities: ['SYS_ADMIN', 'CHOWN'], + securityOptions: ['seccomp=/repo/seccomp/nsjail.json'], + environment: { SANDBOX_USE_CGROUPV2: 'false' }, + bindMounts: [{ source: '/repo/data/pkgs', target: '/pkgs', readOnly: true }], + httpClient: 'bun', + }); + + await supervisor.acquire(assignment('rt-user-1')); + + const run = calls.find(args => args[0] === 'run') ?? []; + assert.ok(run.includes('SYS_ADMIN')); + assert.ok(run.includes('CHOWN')); + assert.ok(run.includes('seccomp=/repo/seccomp/nsjail.json')); + assert.ok(run.includes('SANDBOX_USE_CGROUPV2=false')); + assert.ok(run.includes('type=bind,source=/repo/data/pkgs,target=/pkgs,readonly')); + assert.ok( + run.indexOf('SANDBOX_USE_CGROUPV2=false') < + run.indexOf('SANDBOX_SESSION_WORKSPACE_ENABLED=true'), + ); + const health = calls.find(args => args[0] === 'exec') ?? []; + assert.ok(health.includes('bun')); +}); + +test('docker runtime supervisor rejects relative bind mount paths', () => { + assert.throws( + () => + new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + bindMounts: [{ source: './data/pkgs', target: '/pkgs', readOnly: true }], + }), + /absolute comma-free sources and targets/, + ); +}); + test('docker runtime supervisor rejects malformed runtime response framing', async () => { const client: ContainerRuntimeClient = { async run(args) { @@ -130,10 +178,23 @@ test('docker runtime supervisor rejects malformed runtime response framing', asy test('docker runtime supervisor preserves an existing stateful container after a health failure', async () => { const calls: string[][] = []; + let profileDigest: string | undefined; + let healthChecks = 0; const client: ContainerRuntimeClient = { async run(args) { calls.push(args); - if (args[0] === 'container' && args[1] === 'inspect') return 'true\n'; + if (args[0] === 'container' && args[1] === 'inspect') { + if (!profileDigest) throw new Error('No such container'); + return `true|${profileDigest}|sha256:image-1\n`; + } + if (args[0] === 'image' && args[1] === 'inspect') return 'sha256:image-1\n'; + if (args[0] === 'run') { + profileDigest = args + .find(value => value.startsWith('com.librechat.code.profile-digest=')) + ?.split('=')[1]; + return 'container-id\n'; + } + if (args[0] === 'exec' && healthChecks++ === 0) return '200'; if (args[0] === 'exec') throw new Error('runner unavailable'); throw new Error(`Unexpected Docker command: ${args.join(' ')}`); }, @@ -144,10 +205,158 @@ test('docker runtime supervisor preserves an existing stateful container after a startupTimeoutMs: 1, }); + await supervisor.acquire(assignment('rt-user-1')); await assert.rejects(supervisor.acquire(assignment('rt-user-1')), /did not become healthy/); assert.equal(calls.some(args => args[0] === 'container' && args[1] === 'rm'), false); }); +test('docker runtime supervisor reports state loss before recreating after profile drift', async () => { + const calls: string[][] = []; + let containerExists = true; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') { + if (!containerExists) throw new Error('No such container'); + return 'true|stale-profile|sha256:image-1\n'; + } + if (args[0] === 'image' && args[1] === 'inspect') return 'sha256:image-1\n'; + if (args[0] === 'container' && args[1] === 'rm') { + containerExists = false; + return 'removed\n'; + } + if (args[0] === 'run') { + containerExists = true; + return 'container-id\n'; + } + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + profileRevision: 'seccomp-v2', + client, + }); + + await assert.rejects( + supervisor.acquire(assignment('rt-user-1')), + /workspace was discarded because its confinement profile or image changed/, + ); + await supervisor.acquire(assignment('rt-user-1')); + + const removalIndex = calls.findIndex(args => args[0] === 'container' && args[1] === 'rm'); + const creationIndex = calls.findIndex(args => args[0] === 'run'); + assert.ok(removalIndex >= 0); + assert.ok(creationIndex > removalIndex); + assert.ok( + calls[creationIndex]?.some(value => + value.startsWith('com.librechat.code.profile-digest='), + ), + ); +}); + +test('docker runtime supervisor reports state loss before recreating after an image tag moves', async () => { + const calls: string[][] = []; + let profileDigest: string | undefined; + let containerExists = false; + let currentImageId = 'sha256:image-1'; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') { + if (!containerExists) throw new Error('No such container'); + return `true|${profileDigest}|sha256:image-1\n`; + } + if (args[0] === 'image' && args[1] === 'inspect') { + return `${currentImageId}\n`; + } + if (args[0] === 'container' && args[1] === 'rm') { + containerExists = false; + return 'removed\n'; + } + if (args[0] === 'run') { + profileDigest = args + .find(value => value.startsWith('com.librechat.code.profile-digest=')) + ?.split('=')[1]; + containerExists = true; + return 'container-id\n'; + } + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + client, + }); + + await supervisor.acquire(assignment('rt-user-1')); + currentImageId = 'sha256:image-2'; + await assert.rejects( + supervisor.acquire(assignment('rt-user-1')), + /workspace was discarded because its confinement profile or image changed/, + ); + await supervisor.acquire(assignment('rt-user-1')); + + assert.equal( + calls.filter(args => args[0] === 'container' && args[1] === 'rm').length, + 1, + ); + assert.equal(calls.filter(args => args[0] === 'run').length, 2); +}); + +test('docker runtime supervisor reports state loss instead of restarting tmpfs sessions', async () => { + const calls: string[][] = []; + let profileDigest: string | undefined; + let containerExists = false; + let running = false; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') { + if (!containerExists) throw new Error('No such container'); + return `${running}|${profileDigest}|sha256:image-1\n`; + } + if (args[0] === 'image' && args[1] === 'inspect') return 'sha256:image-1\n'; + if (args[0] === 'container' && args[1] === 'rm') { + containerExists = false; + return 'removed\n'; + } + if (args[0] === 'run') { + profileDigest = args + .find(value => value.startsWith('com.librechat.code.profile-digest=')) + ?.split('=')[1]; + containerExists = true; + running = true; + return 'container-id\n'; + } + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ + image: 'example/code-runtime:latest', + restartStoppedContainers: false, + client, + }); + + await supervisor.acquire(assignment('rt-user-1')); + running = false; + await assert.rejects( + supervisor.acquire(assignment('rt-user-1')), + /workspace was discarded because its container stopped/, + ); + await supervisor.acquire(assignment('rt-user-1')); + + assert.equal(calls.filter(args => args[0] === 'start').length, 0); + assert.equal( + calls.filter(args => args[0] === 'container' && args[1] === 'rm').length, + 1, + ); + assert.equal(calls.filter(args => args[0] === 'run').length, 2); +}); + test('docker runtime supervisor propagates removal failures and forwards reset cancellation', async () => { const controller = new AbortController(); let receivedSignal: AbortSignal | undefined; diff --git a/packages/code/src/runtime.ts b/packages/code/src/runtime.ts index 89eaca95..bde50322 100644 --- a/packages/code/src/runtime.ts +++ b/packages/code/src/runtime.ts @@ -45,7 +45,13 @@ export interface ContainerRuntimeRunOptions { export interface DockerRuntimeSupervisorOptions { image?: string; + profileRevision?: string; + restartStoppedContainers?: boolean; capabilities?: string[]; + securityOptions?: string[]; + environment?: Record; + bindMounts?: DockerRuntimeBindMount[]; + httpClient?: 'curl' | 'bun'; dockerCommand?: string; runnerPort?: number; startupTimeoutMs?: number; @@ -53,6 +59,18 @@ export interface DockerRuntimeSupervisorOptions { client?: ContainerRuntimeClient; } +export interface DockerRuntimeBindMount { + source: string; + target: string; + readOnly?: boolean; +} + +interface DockerContainerState { + running: boolean; + profileDigest?: string; + imageId?: string; +} + const DEFAULT_RUNNER_PORT = 2000; const DEFAULT_STARTUP_TIMEOUT_MS = 30_000; const DEFAULT_HEALTH_PATH = '/api/v2/health'; @@ -83,6 +101,11 @@ function isMissingContainerError(error: unknown): boolean { return /(?:no such container|no such object)/i.test(error.message); } +function isMissingImageError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + return /(?:no such image|no such object)/i.test(error.message); +} + class DockerCliClient implements ContainerRuntimeClient { private readonly command: string; @@ -137,6 +160,11 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { private readonly startupTimeoutMs: number; private readonly healthPath: string; private readonly capabilities: string[]; + private readonly securityOptions: string[]; + private readonly environment: Record; + private readonly bindMounts: DockerRuntimeBindMount[]; + private readonly httpClient: 'curl' | 'bun'; + private readonly restartStoppedContainers: boolean; constructor(private readonly options: DockerRuntimeSupervisorOptions) { if (options.image != null && options.image.trim().length === 0) { @@ -149,7 +177,23 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { this.runnerPort = options.runnerPort ?? DEFAULT_RUNNER_PORT; this.startupTimeoutMs = options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; this.healthPath = options.healthPath ?? DEFAULT_HEALTH_PATH; - this.capabilities = options.capabilities ?? []; + this.capabilities = [...(options.capabilities ?? [])]; + this.securityOptions = [...(options.securityOptions ?? [])]; + this.environment = { ...options.environment }; + this.bindMounts = (options.bindMounts ?? []).map((mount) => ({ ...mount })); + this.httpClient = options.httpClient ?? 'curl'; + this.restartStoppedContainers = options.restartStoppedContainers ?? true; + if ( + this.bindMounts.some( + ({ source, target }) => + !source.startsWith('/') || + !target.startsWith('/') || + source.includes(',') || + target.includes(','), + ) + ) { + throw new Error('Docker runtime bind mounts require absolute comma-free sources and targets'); + } } async acquire(assignment: BridgeAssignment, signal?: AbortSignal): Promise { @@ -158,7 +202,12 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { const name = containerName(sessionId); let created = false; try { - created = await this.ensureContainer(name, sessionId, signal); + created = await this.ensureContainer( + name, + sessionId, + assignment.runtimeSessionId != null, + signal, + ); await this.waitForHealth(name, signal); return { sessionId, @@ -186,13 +235,36 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { private async ensureContainer( name: string, runtimeSessionId: string, + stateful: boolean, signal?: AbortSignal, ): Promise { const image = this.options.image?.trim(); if (!image) throw new Error('Docker runtime image is required for acquisition'); - const running = await this.containerRunning(name, signal); - if (running) return false; - if (running === false) { + const profileDigest = this.profileDigest(image); + let state = await this.containerState(name, signal); + if (state != null) { + const currentImageId = await this.imageId(image, signal); + if ( + state.profileDigest !== profileDigest || + (currentImageId != null && state.imageId !== currentImageId) + ) { + await this.remove(name, signal); + state = undefined; + if (stateful) { + throw new Error( + 'Docker runtime workspace was discarded because its confinement profile or image changed', + ); + } + } + } + if (state?.running) return false; + if (state != null) { + if (!this.restartStoppedContainers) { + await this.remove(name, signal); + throw new Error( + 'Docker runtime workspace was discarded because its container stopped', + ); + } await this.client.run(['start', name], { signal }); return false; } @@ -209,10 +281,18 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { ...this.capabilities.flatMap((capability) => ['--cap-add', capability]), '--security-opt', 'no-new-privileges:true', + ...this.securityOptions.flatMap((option) => ['--security-opt', option]), + ...this.bindMounts.flatMap(({ source, target, readOnly }) => [ + '--mount', + `type=bind,source=${source},target=${target}${readOnly ? ',readonly' : ''}`, + ]), '--label', 'com.librechat.code.runtime=true', '--label', `com.librechat.code.runtime-hash=${containerSuffix(runtimeSessionId)}`, + '--label', + `com.librechat.code.profile-digest=${profileDigest}`, + ...Object.entries(this.environment).flatMap(([name, value]) => ['--env', `${name}=${value}`]), '--env', 'SANDBOX_SESSION_WORKSPACE_ENABLED=true', image, @@ -222,19 +302,73 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { return true; } - private async containerRunning(name: string, signal?: AbortSignal): Promise { + private profileDigest(image: string): string { + return createHash('sha256') + .update( + JSON.stringify({ + version: 1, + image, + profileRevision: this.options.profileRevision ?? null, + restartStoppedContainers: this.restartStoppedContainers, + capabilities: this.capabilities, + securityOptions: this.securityOptions, + environment: Object.entries(this.environment).sort(([left], [right]) => + left.localeCompare(right), + ), + bindMounts: this.bindMounts, + }), + ) + .digest('hex'); + } + + private async containerState( + name: string, + signal?: AbortSignal, + ): Promise { try { const value = await this.client.run( - ['container', 'inspect', '--format', '{{.State.Running}}', name], + [ + 'container', + 'inspect', + '--format', + '{{.State.Running}}|{{index .Config.Labels "com.librechat.code.profile-digest"}}|{{.Image}}', + name, + ], { signal }, ); - return value.trim() === 'true'; + const [running, profileDigest, imageId] = value.trim().split('|'); + if (running !== 'true' && running !== 'false') { + throw new Error( + 'Docker runtime container inspection returned an invalid state', + ); + } + return { + running: running === 'true', + ...(profileDigest && profileDigest !== '' ? { profileDigest } : {}), + ...(imageId ? { imageId } : {}), + }; } catch (error) { if (isMissingContainerError(error)) return undefined; throw error; } } + private async imageId( + image: string, + signal?: AbortSignal, + ): Promise { + try { + const value = await this.client.run( + ['image', 'inspect', '--format', '{{.Id}}', image], + { signal }, + ); + return value.trim() || undefined; + } catch (error) { + if (isMissingImageError(error)) return undefined; + throw error; + } + } + private async waitForHealth(name: string, signal?: AbortSignal): Promise { const deadline = Date.now() + this.startupTimeoutMs; let lastError: unknown; @@ -242,21 +376,32 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { if (signal?.aborted) throw signal.reason ?? new DOMException('aborted', 'AbortError'); try { const remainingMs = Math.max(1, deadline - Date.now()); + const healthUrl = `http://127.0.0.1:${this.runnerPort}${this.healthPath}`; const status = await this.client.run( - [ - 'exec', - name, - 'curl', - '--silent', - '--show-error', - '--max-time', - (remainingMs / 1000).toFixed(3), - '--output', - '/dev/null', - '--write-out', - '%{http_code}', - `http://127.0.0.1:${this.runnerPort}${this.healthPath}`, - ], + this.httpClient === 'bun' + ? [ + 'exec', + name, + 'bun', + '-e', + 'const r=await fetch(process.argv.at(-2),{signal:AbortSignal.timeout(Number(process.argv.at(-1)))});process.stdout.write(String(r.status));', + healthUrl, + String(remainingMs), + ] + : [ + 'exec', + name, + 'curl', + '--silent', + '--show-error', + '--max-time', + (remainingMs / 1000).toFixed(3), + '--output', + '/dev/null', + '--write-out', + '%{http_code}', + healthUrl, + ], { signal }, ); if (status.trim() === '200') return; @@ -281,23 +426,35 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { throw new Error('Runtime request headers cannot contain line breaks'); } const marker = randomBytes(32).toString('hex'); + const executeUrl = `http://127.0.0.1:${this.runnerPort}/api/v2/execute`; const output = await this.client.run( - [ - 'exec', - '--interactive', - name, - 'curl', - '--silent', - '--show-error', - '--request', - 'POST', - ...Object.entries(request.headers).flatMap(([name, value]) => ['--header', `${name}: ${value}`]), - '--data-binary', - '@-', - '--write-out', - `\n${marker}%{http_code}`, - `http://127.0.0.1:${this.runnerPort}/api/v2/execute`, - ], + this.httpClient === 'bun' + ? [ + 'exec', + '--interactive', + name, + 'bun', + '-e', + `const b=await Bun.stdin.text();const r=await fetch(process.argv.at(-2),{method:'POST',headers:JSON.parse(process.argv.at(-1)),body:b});process.stdout.write(await r.text());process.stdout.write('\\n${marker}'+r.status);`, + executeUrl, + JSON.stringify(request.headers), + ] + : [ + 'exec', + '--interactive', + name, + 'curl', + '--silent', + '--show-error', + '--request', + 'POST', + ...Object.entries(request.headers).flatMap(([name, value]) => ['--header', `${name}: ${value}`]), + '--data-binary', + '@-', + '--write-out', + `\n${marker}%{http_code}`, + executeUrl, + ], { input: request.body, signal: request.signal }, ); const suffix = new RegExp(`\\n${marker}(\\d{3})$`); From 6967ba947cae5b38cee196798e5145ad71f76d96 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 10:47:15 -0400 Subject: [PATCH 016/116] =?UTF-8?q?=F0=9F=8E=B4=20fix:=20Rotate=20PTC=20Re?= =?UTF-8?q?play=20Client=20Tokens=20(#82)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(lambda-microvm): give each PTC replay iteration a distinct clientToken PTC replay reuses one executionId across every stateless iteration, but the launch clientToken was derived from that executionId alone. Each iteration relaunches with a changed sandbox payload (a fresh _ptc_history.json), so AWS rejected the second launch with "The provided clientToken was used with different request parameters" and LibreChat surfaced the generic "Bash programmatic execution failed" (#59). Fold the launch inputs and the per-iteration request body into the token so each distinct launch gets a distinct token while an identical retry stays idempotent. Reuses runtimeSessionLaunchRequestFingerprint rather than restating the launch inputs. Reported with a working patch by @snapydziuba. * fix(lambda-microvm): key the stateless launch token to the queued job Addresses codex review on #82. Hashing the request body made the token move between attempts of the same job: workers.ts rebuilds the request on every attempt with a fresh egress grant (random IV and sandbox session id) and a re-signed manifest, so a replacement worker taking over a stalled job would derive a different token, launch a second VM, and leave the accepted one burning capacity until its maximum duration expired. Use the queued job id instead. Each PTC replay iteration is enqueued as its own job, so it is distinct per iteration and stable across attempts of the same job -- and it carries no capability-bearing material. The launch configuration stays in the digest so a worker with a different config cannot reuse another's token. --- .../sandbox-backend/lambda-microvm.test.ts | 54 +++++++++++++++++-- service/src/sandbox-backend/lambda-microvm.ts | 51 +++++++++++++++++- service/src/sandbox-backend/types.ts | 8 +++ service/src/workers.ts | 1 + 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/service/src/sandbox-backend/lambda-microvm.test.ts b/service/src/sandbox-backend/lambda-microvm.test.ts index 6641fa6d..02110edd 100644 --- a/service/src/sandbox-backend/lambda-microvm.test.ts +++ b/service/src/sandbox-backend/lambda-microvm.test.ts @@ -26,6 +26,7 @@ import { runtimeSessionLaunchClientToken, runtimeSessionLaunchFingerprint, runtimeSessionLaunchGenerationSeed, + statelessLaunchClientToken, type LambdaMicrovmBackendConfig, } from './lambda-microvm'; import { SandboxBackendError } from './types'; @@ -301,6 +302,50 @@ describe('runtime session launch tokens', () => { }); }); +describe('statelessLaunchClientToken', () => { + test('is deterministic for an identical relaunch', () => { + expect(statelessLaunchClientToken('exec_42', config(), 420, 'job_1')) + .toBe(statelessLaunchClientToken('exec_42', config(), 420, 'job_1')); + }); + + /* PTC replay reuses one executionId across iterations, each enqueued as its + * own job. A token derived from the executionId alone repeated, and AWS + * rejected the relaunch with "The provided clientToken was used with + * different request parameters". */ + test('differs per replay iteration', () => { + const round1 = statelessLaunchClientToken('exec_42', config(), 420, 'job_1'); + const round2 = statelessLaunchClientToken('exec_42', config(), 420, 'job_2'); + expect(round1).not.toBe(round2); + expect(round1).toMatch(/^exec-exec_42-[0-9a-f]{16}$/); + expect(round2).toMatch(/^exec-exec_42-[0-9a-f]{16}$/); + }); + + /* A replacement worker taking over a stalled job rebuilds the request with a + * fresh egress grant, sandbox session id and re-signed manifest. The token + * must not move with it, or RunMicrovm idempotency cannot recover a launch + * AWS already accepted and the orphaned VM burns capacity until it expires. */ + test('is stable across attempts of the same queued job', () => { + const firstAttempt = statelessLaunchClientToken('exec_42', config(), 420, 'job_1'); + const stalledRetry = statelessLaunchClientToken('exec_42', config(), 420, 'job_1'); + expect(stalledRetry).toBe(firstAttempt); + }); + + test('differs when launch configuration or duration changes', () => { + const base = statelessLaunchClientToken('exec_42', config(), 420, 'job_1'); + expect(statelessLaunchClientToken('exec_42', config({ imageVersion: '4' }), 420, 'job_1')) + .not.toBe(base); + expect(statelessLaunchClientToken('exec_42', config(), 421, 'job_1')).not.toBe(base); + }); + + test('stays within the AWS clientToken budget including the retry suffix', () => { + const token = statelessLaunchClientToken('exec_42', config(), 420, 'job_1'); + expect(`${token}-r1`.length).toBeLessThanOrEqual(128); + expect(() => statelessLaunchClientToken('e'.repeat(200), config(), 420, 'job_1')).toThrow( + 'Stateless launch clientToken exceeds the AWS length limit', + ); + }); +}); + describe('LambdaMicrovmSandboxBackend stateless execution', () => { test('run -> health -> execute -> terminate happy path', async () => { const fake = fakeClient(); @@ -315,7 +360,7 @@ describe('LambdaMicrovmSandboxBackend stateless execution', () => { expect(runCalls).toHaveLength(1); const runArgs = runCalls[0].args as { imageIdentifier: string; clientToken?: string; maximumDurationSeconds: number }; expect(runArgs.imageIdentifier).toBe('arn:aws:lambda:us-east-2:1:microvm-image:codeapi'); - expect(runArgs.clientToken).toBe('exec-exec_42'); + expect(runArgs.clientToken).toMatch(/^exec-exec_42-[0-9a-f]{16}$/); expect(runArgs.maximumDurationSeconds).toBe(Math.ceil(300_000 / 1_000) + 120); const executeReq = captured.find((c) => c.path === '/api/v2/execute'); @@ -509,8 +554,8 @@ describe('LambdaMicrovmSandboxBackend stateless execution', () => { const runCalls = fake.callsFor('runMicrovm'); expect(runCalls).toHaveLength(2); const tokens = runCalls.map((call) => (call.args as { clientToken?: string }).clientToken); - expect(tokens[0]).toBe('exec-exec_42'); - expect(tokens[1]).toBe('exec-exec_42-r1'); + expect(tokens[0]).toMatch(/^exec-exec_42-[0-9a-f]{16}$/); + expect(tokens[1]).toBe(`${tokens[0]}-r1`); }); test('the boot-death retry consumes only the first attempt remaining launch budget', async () => { @@ -533,7 +578,8 @@ describe('LambdaMicrovmSandboxBackend stateless execution', () => { }); const tokens = fake.callsFor('runMicrovm') .map(call => (call.args as { clientToken?: string }).clientToken); - expect(tokens).toEqual(['exec-exec_42', 'exec-exec_42-r1']); + expect(tokens[0]).toMatch(/^exec-exec_42-[0-9a-f]{16}$/); + expect(tokens).toEqual([tokens[0], `${tokens[0]}-r1`]); expect(captured.some(request => request.path === '/api/v2/execute')).toBe(false); expect(fake.callsFor('terminateMicrovm')).toHaveLength(2); }); diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index 1cb8a4dd..d947c76e 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -138,6 +138,48 @@ export function runtimeSessionLaunchGenerationSeed(config: LambdaMicrovmBackendC return RUNTIME_SESSION_NAMESPACED_GENERATION_MIN + offset; } +/** Stateless one-shot launch token. + * + * PTC replay reuses one executionId across every iteration, so a token derived + * from the executionId alone repeats while the launch parameters change with + * each iteration's payload. AWS rejects that with "The provided clientToken was + * used with different request parameters" and the whole execution fails. + * + * The discriminator is the queued job id rather than the request body: the body + * is rebuilt with a fresh egress grant, sandbox session id and re-signed + * manifest on every job attempt, so hashing it would hand a replacement worker + * a different token after a stalled-job takeover and launch a second VM instead + * of recovering the accepted one through RunMicrovm idempotency. The job id is + * distinct per replay iteration and stable across attempts of the same job. + * + * The launch configuration stays in the digest because a worker whose config + * differs must not reuse another worker's token. */ +export function statelessLaunchClientToken( + executionId: string, + config: LambdaMicrovmBackendConfig, + maxDurationSeconds: number, + queuedJobId: string, +): string { + const suffix = createHash('sha256') + .update( + JSON.stringify({ + launchRequest: runtimeSessionLaunchRequestFingerprint(config), + maximumDurationSeconds: maxDurationSeconds, + queuedJobId, + }), + 'utf8', + ) + .digest('hex') + .slice(0, 16); + const token = `exec-${executionId}-${suffix}`; + /* launch() can add "-r1" after a clean boot-time death; reserve those three + * characters so both attempts stay within AWS's 128-byte limit. */ + if (token.length > 125) { + throw new Error('Stateless launch clientToken exceeds the AWS length limit'); + } + return token; +} + export function runtimeSessionLaunchClientToken(runtimeSessionId: string, generation: number): string { if (!Number.isSafeInteger(generation) || generation < 1) { throw new Error('Runtime session generation must be a positive safe integer'); @@ -259,7 +301,14 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { Math.ceil(this.config.jobTimeoutMs / 1_000) + 120, ); const vm = await this.launch(client, ctx, { - clientToken: ctx.executionId !== '' ? `exec-${ctx.executionId}` : `exec-${nanoid()}`, + clientToken: statelessLaunchClientToken( + ctx.executionId !== '' ? ctx.executionId : nanoid(), + this.config, + maxDurationSeconds, + /* No queued job id (direct backend caller): fall back to a fresh value + * so distinct launches never collide on one token. */ + ctx.queuedJobId ?? nanoid(), + ), maxDurationSeconds, }); let terminateReason = 'stateless'; diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index 64e5b6e0..96151dde 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -39,6 +39,14 @@ export interface SandboxExecuteContext { canonicalUserId?: string; /** Trusted API-selected outbound worker. Presence requires a tenant-bound credential. */ bridgeWorkerId?: string; + /** Stable identifier for this queued iteration, used to derive an idempotent + * stateless launch token. PTC replay reuses one executionId across every + * iteration, so the executionId alone cannot separate them; the request body + * can, but it is rebuilt with a fresh egress grant and manifest on every job + * attempt, so hashing it would break RunMicrovm idempotency when BullMQ + * reprocesses a stalled job. The queued job id is distinct per iteration and + * stable across attempts of the same job. */ + queuedJobId?: string; /** Absent ⇒ stateless execution (no runtime session affinity). */ runtimeSessionId?: string; runtimeSessionMode: t.RuntimeSessionMode; diff --git a/service/src/workers.ts b/service/src/workers.ts index d2048dc8..215a4d1b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -141,6 +141,7 @@ async function processJobInner(job: t.ExecuteJob): Promise { }, { executionId: job.data.executionId ?? '', + queuedJobId: job.id != null ? String(job.id) : undefined, language, isSynthetic: isSyntheticJob, signal: controller.signal, From 543bf4e4ad161c627ca1156d5036ca683b664aa6 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 10:49:26 -0400 Subject: [PATCH 017/116] =?UTF-8?q?=F0=9F=93=9B=20fix:=20Preserve=20Upload?= =?UTF-8?q?ed=20Filenames=20Without=20Metadata=20(#79)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: preserve uploaded filenames without s3 metadata * fix: preserve filenames through hardened egress --- api/src/download.test.ts | 23 ++++++++++ api/src/job-helpers.test.ts | 27 ++++++++++++ api/src/job.ts | 17 +++++++- service/src/egress-gateway.test.ts | 24 ++++++++++- service/src/egress-gateway.ts | 17 ++++++-- service/src/file-metadata.test.ts | 68 ++++++++++++++++++++++++++++++ service/src/file-metadata.ts | 62 +++++++++++++++++++++++++++ service/src/file-server.ts | 49 ++++++--------------- 8 files changed, 245 insertions(+), 42 deletions(-) create mode 100644 service/src/file-metadata.test.ts create mode 100644 service/src/file-metadata.ts diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 692957b2..76e21395 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -289,6 +289,29 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(contents).toBe('legacy bytes'); }); + it('writes under the requested name when a legacy server returns an opaque storage filename', async () => { + const file: TFile = { + id: 'opaque-storage-id', + storage_session_id: 'prev-session', + name: 'Sample_-_Superstore.xlsx', + }; + routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { + status: 200, + contentDisposition: "attachment; filename*=UTF-8''opaque-storage-id.xlsx", + body: 'workbook bytes', + }); + + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + const writtenName = await job.downloadAndWriteFile(file); + + expect(writtenName).toBe('Sample_-_Superstore.xlsx'); + expect(await fsp.readFile(path.join(tmpDir, 'Sample_-_Superstore.xlsx'), 'utf8')) + .toBe('workbook bytes'); + expect(await fsp.stat(path.join(tmpDir, 'opaque-storage-id.xlsx')).catch(() => null)).toBeNull(); + }); + it('resolves concurrent header destinations without provisional-name false conflicts', async () => { const renamed: TFile = { id: 'renamed-id', diff --git a/api/src/job-helpers.test.ts b/api/src/job-helpers.test.ts index 3509f9db..0c31f9d3 100644 --- a/api/src/job-helpers.test.ts +++ b/api/src/job-helpers.test.ts @@ -177,6 +177,33 @@ describe('resolveOriginalName', () => { ), ).toBe('nested/file.txt'); }); + + it('keeps the requested name when an old file server advertises the opaque object basename', () => { + expect( + resolveOriginalName( + responseWithHeader("attachment; filename*=UTF-8''storage-id.xlsx"), + { name: 'Sample_-_Superstore.xlsx', id: 'storage-id' }, + ), + ).toBe('Sample_-_Superstore.xlsx'); + }); + + it('keeps the requested name for a legacy opaque filename header', () => { + expect( + resolveOriginalName( + responseWithHeader('attachment; filename="storage-id.csv"'), + { name: 'original.csv', id: 'storage-id' }, + ), + ).toBe('original.csv'); + }); + + it('keeps an authoritative nested filename even when its basename matches the object id', () => { + expect( + resolveOriginalName( + responseWithHeader("attachment; filename*=UTF-8''exports%2Fstorage-id.csv"), + { name: 'original.csv', id: 'storage-id' }, + ), + ).toBe('exports/storage-id.csv'); + }); }); describe('mimeTypeFor', () => { diff --git a/api/src/job.ts b/api/src/job.ts index eceffdfd..e221ae19 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -179,11 +179,24 @@ export function resolveOriginalName(response: Response, file: TFile): string { const header = response.headers.get('content-disposition'); if (!header) return fallback; + const preferRequestedName = (candidate: string): string => { + /* Older file servers advertised path.basename(objectName) when an + * S3-compatible backend omitted original-filename user metadata. That + * basename is ``, so it is a storage identifier rather + * than an authoritative destination. Preserve the caller's requested name + * during rolling upgrades instead of exposing the opaque id in /mnt/data. */ + const opaqueStem = path.basename(candidate, path.extname(candidate)); + const isFlatObjectBasename = candidate === path.basename(candidate); + return file.name && file.id && isFlatObjectBasename && opaqueStem === file.id + ? file.name + : candidate; + }; + const star = header.match(/filename\*=(?:UTF-8'[^']*')?([^;]+)/i); if (star) { const raw = star[1].trim(); try { - return decodeURIComponent(raw); + return preferRequestedName(decodeURIComponent(raw)); } catch { /* Malformed percent-encoding (e.g. `%ZZ`) — fall through to the legacy * forms. The same header may emit both `filename*=` and a legacy @@ -194,7 +207,7 @@ export function resolveOriginalName(response: Response, file: TFile): string { const match = header.match(/filename="([^"]+)"/i) ?? header.match(/filename=([^\s;]+)/i); - return match ? match[1] : fallback; + return match ? preferRequestedName(match[1]) : fallback; } /** diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 9f72da54..99303850 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -632,7 +632,10 @@ describe('egress gateway routes', () => { test('downloads scoped objects by unwrapping handles', async () => { upstreamResponse = new Response('file-body', { status: 200, - headers: { 'Content-Type': 'text/plain' }, + headers: { + 'Content-Type': 'text/plain', + 'Content-Disposition': "attachment; filename*=UTF-8''file_123.csv", + }, }); const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); const object = objectHandle({}); @@ -643,10 +646,29 @@ describe('egress gateway routes', () => { expect(response.status).toBe(200); expect(await response.text()).toBe('file-body'); + expect(response.headers.get('content-disposition')).toBe('attachment'); expect(upstreamCalls[0].url).toBe('http://file-server/sessions/sess_input/objects/file_123'); expect(header(upstreamCalls[0].init, INTERNAL_SERVICE_TOKEN_HEADER)).toBe(INTERNAL_TOKEN); }); + test('preserves an authoritative upstream download filename', async () => { + upstreamResponse = new Response('file-body', { + status: 200, + headers: { + 'Content-Disposition': "attachment; filename*=UTF-8''reports%2Fdata.csv", + }, + }); + const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + + const response = await gatewayFetch(`/sessions/${readSession}/objects/${object}`, { + headers: grantHeader(), + }); + + expect(response.headers.get('content-disposition')) + .toBe("attachment; filename*=UTF-8''reports%2Fdata.csv"); + }); + test('downloads required dirkeep markers without allowing unrelated markers', async () => { upstreamResponse = new Response('marker-body', { status: 200, diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 3499de80..d4d3e713 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -42,6 +42,7 @@ import { isValidId } from './utils'; import logger from './logger'; import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; +import { isOpaqueObjectContentDisposition } from './file-metadata'; export const app: Express = express(); app.disable('x-powered-by'); @@ -347,9 +348,13 @@ function responseHeaders(fetchResponse: globalThis.Response): Record = {}, +): void { res.status(fetchResponse.status); - res.set(responseHeaders(fetchResponse)); + res.set({ ...responseHeaders(fetchResponse), ...headerOverrides }); if (!fetchResponse.body) { res.end(); return; @@ -631,7 +636,13 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { ), { headers: injectTraceHeaders(internalServiceHeaders()) }, ); - return pipeFetchResponse(upstream, res); + const headerOverrides = isOpaqueObjectContentDisposition( + upstream.headers.get('content-disposition'), + object.id, + ) + ? { 'content-disposition': 'attachment' } + : {}; + return pipeFetchResponse(upstream, res, headerOverrides); } catch (error) { return sendEgressError(req, res, error); } diff --git a/service/src/file-metadata.test.ts b/service/src/file-metadata.test.ts new file mode 100644 index 00000000..082a4215 --- /dev/null +++ b/service/src/file-metadata.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'bun:test'; +import { + contentDispositionForOriginalFilename, + decodeOriginalFilename, + isOpaqueObjectContentDisposition, + originalFilenameFromMetadata, +} from './file-metadata'; + +describe('originalFilenameFromMetadata', () => { + it('returns undefined rather than treating an object-key basename as original metadata', () => { + expect(originalFilenameFromMetadata(undefined)).toBeUndefined(); + expect(originalFilenameFromMetadata({ 'content-type': 'application/octet-stream' })).toBeUndefined(); + }); + + it('decodes the base64 filename written by the file server', () => { + expect(originalFilenameFromMetadata({ + 'original-filename': Buffer.from('Sample_-_Superstore.xlsx').toString('base64'), + 'original-filename-encoded': 'base64', + })).toBe('Sample_-_Superstore.xlsx'); + }); + + it('supports legacy plain-text filename metadata', () => { + expect(originalFilenameFromMetadata({ + 'original-filename': 'report.csv', + })).toBe('report.csv'); + }); +}); + +describe('decodeOriginalFilename', () => { + it('retains object-key fallback behavior for listing responses', () => { + expect(decodeOriginalFilename(undefined, 'opaque-id.xlsx')).toBe('opaque-id.xlsx'); + }); +}); + +describe('contentDispositionForOriginalFilename', () => { + it('preserves attachment semantics when filename metadata is unavailable', () => { + expect(contentDispositionForOriginalFilename(undefined)).toBe('attachment'); + }); + + it('encodes a verified original filename', () => { + expect(contentDispositionForOriginalFilename('reports/收益.csv')) + .toBe("attachment; filename*=UTF-8''reports%2F%E6%94%B6%E7%9B%8A.csv"); + }); +}); + +describe('isOpaqueObjectContentDisposition', () => { + it('recognizes extended and legacy storage-id basenames', () => { + expect(isOpaqueObjectContentDisposition( + "attachment; filename*=UTF-8''raw-object-id.xlsx", + 'raw-object-id', + )).toBe(true); + expect(isOpaqueObjectContentDisposition( + 'attachment; filename="raw-object-id.csv"', + 'raw-object-id', + )).toBe(true); + }); + + it('does not replace verified or nested filenames', () => { + expect(isOpaqueObjectContentDisposition( + 'attachment; filename="report.csv"', + 'raw-object-id', + )).toBe(false); + expect(isOpaqueObjectContentDisposition( + "attachment; filename*=UTF-8''exports%2Fraw-object-id.csv", + 'raw-object-id', + )).toBe(false); + }); +}); diff --git a/service/src/file-metadata.ts b/service/src/file-metadata.ts new file mode 100644 index 00000000..13a45623 --- /dev/null +++ b/service/src/file-metadata.ts @@ -0,0 +1,62 @@ +import path from 'path'; + +/** + * Reads a verified original filename from S3 user metadata. Absence remains + * distinct from the object's opaque storage-key basename so callers can avoid + * advertising the latter as an authoritative filename. + */ +export function originalFilenameFromMetadata( + metadata: Record | undefined, +): string | undefined { + const encodedFilename = metadata?.['original-filename']; + if (!encodedFilename) return undefined; + + if (metadata?.['original-filename-encoded'] === 'base64') { + return Buffer.from(encodedFilename, 'base64').toString('utf8'); + } + + return encodedFilename; +} + +export function decodeOriginalFilename( + metadata: Record | undefined, + fallbackName: string, +): string { + return originalFilenameFromMetadata(metadata) ?? fallbackName; +} + +export function contentDispositionForOriginalFilename( + originalFilename: string | undefined, +): string { + if (!originalFilename) return 'attachment'; + return `attachment; filename*=UTF-8''${encodeURIComponent(originalFilename)}`; +} + +function filenameFromContentDisposition(contentDisposition: string | null): string | undefined { + if (!contentDisposition) return undefined; + const star = contentDisposition.match(/filename\*=(?:UTF-8'[^']*')?([^;]+)/i); + if (star) { + try { + return decodeURIComponent(star[1].trim()); + } catch { + // A valid legacy filename may still follow a malformed extended value. + } + } + const legacy = contentDisposition.match(/filename="([^"]+)"/i) + ?? contentDisposition.match(/filename=([^\s;]+)/i); + return legacy?.[1]; +} + +/** + * Detects the legacy file-server fallback ``. The egress + * gateway has the unsealed object id, so it can remove this unverified name + * before forwarding the response to a runner that only sees sealed handles. + */ +export function isOpaqueObjectContentDisposition( + contentDisposition: string | null, + objectId: string, +): boolean { + const candidate = filenameFromContentDisposition(contentDisposition); + if (!candidate || candidate !== path.basename(candidate)) return false; + return path.basename(candidate, path.extname(candidate)) === objectId; +} diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 6d226c06..f9293e48 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -17,6 +17,11 @@ import { shutdownTelemetry, traceHttpRequest } from './telemetry'; import logger from './fileServerLogger'; import { env } from './config'; import { redisKeepAliveOptions } from './redis-options'; +import { + contentDispositionForOriginalFilename, + decodeOriginalFilename, + originalFilenameFromMetadata, +} from './file-metadata'; const { INSTANCE_ID } = env; @@ -458,11 +463,11 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => } const stat: Partial = await minioClient.statObject(bucketName, objectName); - const originalFilename = decodeOriginalFilename(stat.metaData, path.basename(objectName)); + const originalFilename = originalFilenameFromMetadata(stat.metaData); return res.status(200).json({ name: objectName, - originalFilename, + ...(originalFilename ? { originalFilename } : {}), size: stat.size, lastModified: stat.lastModified, etag: stat.etag, @@ -508,17 +513,7 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const stat: Partial = await minioClient.statObject(bucketName, objectName); - let originalFilename = path.basename(objectName); - if (stat.metaData?.['original-filename-encoded'] === 'base64' && stat.metaData['original-filename'] != null) { - try { - originalFilename = Buffer.from(stat.metaData['original-filename'], 'base64').toString('utf8'); - } catch (err) { - logger.warn('Failed to decode filename from metadata, using fallback', { error: err }); - originalFilename = stat.metaData['original-filename'] ?? path.basename(objectName); - } - } else if (stat.metaData?.['original-filename'] != null) { - originalFilename = stat.metaData['original-filename']; - } + const originalFilename = originalFilenameFromMetadata(stat.metaData); logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); @@ -526,8 +521,11 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { res.removeHeader('Transfer-Encoding'); res.removeHeader('Date'); - const encodedFilename = encodeURIComponent(originalFilename); - res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${encodedFilename}`); + /* An object-key basename is only a storage identifier, not an original + * filename. If an S3-compatible backend drops user metadata, retain + * attachment semantics but omit the filename so the runner uses its + * caller-supplied destination. */ + res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilename)); if (stat.metaData?.['content-type'] != null) { res.setHeader('Content-Type', stat.metaData['content-type']); } @@ -573,27 +571,6 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { } }); -/** - * Decodes the original filename from metadata. - * Handles both base64-encoded and plain text filenames for consistency. - */ -function decodeOriginalFilename(metadata: Record | undefined, fallbackName: string): string { - if (!metadata) return fallbackName; - - const encodedFilename = metadata['original-filename']; - const encodingType = metadata['original-filename-encoded']; - - if (encodedFilename && encodingType === 'base64') { - try { - return Buffer.from(encodedFilename, 'base64').toString('utf8'); - } catch { - return encodedFilename; - } - } - - return encodedFilename || fallbackName; -} - /** * Extracts session_id and file_id from object name (format: {session_id}/{file_id}.ext) */ From a1fd45eedc68c666aa689fa40f3e896b95c58f02 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 10:53:00 -0400 Subject: [PATCH 018/116] =?UTF-8?q?=F0=9F=AA=AB=20chore:=20Detect=20Compos?= =?UTF-8?q?e=20Sandbox=20Clock=20Drift=20(#81)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guest clock drift past the 30s execution-manifest tolerance makes every /v1/exec fail with "not_yet_valid" while both health endpoints keep reporting healthy, so the stack looks fine while nothing runs (#37). The healthcheck already detects this, but it stays disabled unless an orchestrator opts in, and the Compose files never did -- only the Helm chart set it. Opt in there too, at the same 10s the chart uses. The 2s probe timeout keeps the check inside both files' healthcheck timeouts (3s and 5s) and leaves headroom under the 30s tolerance. --- docker-compose.local-dev.yml | 5 +++++ docker-compose.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/docker-compose.local-dev.yml b/docker-compose.local-dev.yml index bf58b596..4d8f5a33 100644 --- a/docker-compose.local-dev.yml +++ b/docker-compose.local-dev.yml @@ -42,6 +42,11 @@ services: - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} - SANDBOX_ALLOWED_LOCAL_NETWORK_PORT=3033 - SANDBOX_FORWARD_TARGET=tool_call_server:3033 + # Guest clock drift silently fails every exec with "not_yet_valid" + # once it passes the 30s execution-manifest tolerance (#37). Opt in so + # the healthcheck reports it; set the limit to 0 to disable. + - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-10} + - SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS=${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-2} healthcheck: test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"] interval: 10s diff --git a/docker-compose.yaml b/docker-compose.yaml index ac98d917..00cad657 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -161,6 +161,11 @@ services: - SANDBOX_FORWARD_TARGET=egress_gateway:3190 - SANDBOX_REQUIRE_EGRESS_MANIFEST=${SANDBOX_REQUIRE_EGRESS_MANIFEST:-true} - SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY=${SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY:-MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U=} + # Guest clock drift silently fails every exec with "not_yet_valid" + # once it passes the 30s execution-manifest tolerance (#37). Opt in + # so the healthcheck reports it; set the limit to 0 to disable. + - SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS=${SANDBOX_RUNNER_CLOCK_SKEW_LIVENESS_LIMIT_SECONDS:-10} + - SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS=${SANDBOX_RUNNER_HEALTHCHECK_TIMEOUT_SECONDS:-2} depends_on: egress_gateway: condition: service_healthy From 0eb0f3a30e984f23fa98d0cc48acc8abd21cb184 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 10:53:13 -0400 Subject: [PATCH 019/116] =?UTF-8?q?=F0=9F=9B=B3=20fix:=20Fetch=20Bitnami?= =?UTF-8?q?=20Subcharts=20From=20OCI=20(#83)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(helm): resolve Bitnami subcharts over OCI Bitnami distributes charts OCI-only. The classic charts.bitnami.com index still lists redis 24.1.0 and minio 17.0.21, but resolves them to an oci:// download URL that HTTP-repository getters cannot follow, so FluxCD's source-controller fails dependency resolution outright with 'unsupported protocol scheme "oci"' (#21). Point both dependencies at the OCI registry directly. Requires Helm >= 3.8. Reported by @meroo36. * docs(helm): require Helm >= 3.8 for OCI subchart resolution Addresses codex review on #83. The README's "Helm 3.x" prerequisite and setup-local.sh's existence-only check both allowed 3.0-3.7, where OCI dependency references are not resolved without an experimental flag -- so the documented setup flow would fail at dependency resolution rather than with a clear message. State the real minimum, and reject older Helm in setup-local.sh before it gets that far. Also drop the classic bitnami repo registration, which the OCI references no longer use; verified 'helm dependency update' resolves both subcharts with that repo removed from the local Helm config. * chore(helm): bump chart to 0.3.1 for the dependency source change Addresses codex review on #83. Changing where the subcharts resolve from is a chart-level change, and Chart.yaml's own version comment asks for a bump. Leaving 0.3.0 in place lets consumers reconciling on chart version treat the corrected chart as the already-seen 0.3.0 artifact and keep the broken HTTP dependency metadata. Matches 4b72e9d, which bumped the chart for the same reason. --- helm/codeapi/Chart.yaml | 11 ++++++++--- helm/codeapi/README.md | 4 +++- helm/setup-local.sh | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/helm/codeapi/Chart.yaml b/helm/codeapi/Chart.yaml index 9737e908..4e122bbf 100644 --- a/helm/codeapi/Chart.yaml +++ b/helm/codeapi/Chart.yaml @@ -3,7 +3,7 @@ apiVersion: v2 name: codeapi description: A Helm chart for Code Interpreter API - scalable code execution service type: application -version: 0.3.0 # Chart version (bump this when you change the chart) +version: 0.3.1 # Chart version (bump this when you change the chart) appVersion: "2.0.0" # App version (bump this when you change the app) # Keywords for searching @@ -18,12 +18,17 @@ maintainers: url: https://github.com/danny-avila # Dependencies (we'll use these for Redis and MinIO) +# Bitnami distributes charts OCI-only. The classic charts.bitnami.com/bitnami +# index still lists these versions but resolves them to an oci:// download URL, +# which HTTP-repository getters (notably FluxCD's source-controller) cannot +# follow -- dependency resolution fails outright. Point at the OCI registry +# directly; requires Helm >= 3.8. dependencies: - name: redis version: "24.1.0" - repository: "https://charts.bitnami.com/bitnami" + repository: "oci://registry-1.docker.io/bitnamicharts" condition: redis.enabled - name: minio version: "17.0.21" - repository: "https://charts.bitnami.com/bitnami" + repository: "oci://registry-1.docker.io/bitnamicharts" condition: minio.enabled diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index fcdef01d..29dc894a 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -6,7 +6,9 @@ Deploy the horizontally-scalable Code Interpreter API stack to Kubernetes. - Docker Desktop with Kubernetes enabled, OR - Minikube installed (`brew install minikube` / `choco install minikube`) -- Helm 3.x (`brew install helm` / `choco install kubernetes-helm`) +- Helm >= 3.8 (`brew install helm` / `choco install kubernetes-helm`) — the + redis and minio subcharts are pulled from an OCI registry, which older Helm + releases only support behind an experimental flag - kubectl (`brew install kubectl` / `choco install kubernetes-cli`) ## Execution manifest signing keys (required) diff --git a/helm/setup-local.sh b/helm/setup-local.sh index f71c39bb..f51889d6 100755 --- a/helm/setup-local.sh +++ b/helm/setup-local.sh @@ -32,9 +32,38 @@ check_command() { echo "✓ $1 found" } +# The chart's subchart dependencies are OCI references, which Helm only +# resolves without an experimental flag from 3.8 onward. +check_helm_version() { + local required_major=3 required_minor=8 + local raw major minor + raw=$(helm version --template '{{.Version}}' 2>/dev/null || true) + if [ -z "$raw" ]; then + echo "❌ could not determine the installed Helm version (need >= ${required_major}.${required_minor})." + exit 1 + fi + raw=${raw#v} + major=${raw%%.*} + minor=${raw#*.} + minor=${minor%%.*} + case "$major$minor" in + *[!0-9]*|'') + echo "❌ could not parse the installed Helm version '$raw' (need >= ${required_major}.${required_minor})." + exit 1 + ;; + esac + if [ "$major" -lt "$required_major" ] || + { [ "$major" -eq "$required_major" ] && [ "$minor" -lt "$required_minor" ]; }; then + echo "❌ Helm $raw is too old. The chart's OCI subchart dependencies need >= ${required_major}.${required_minor}." + exit 1 + fi + echo "✓ helm $raw supports OCI dependencies" +} + echo "📋 Checking prerequisites..." check_command docker check_command helm +check_helm_version check_command kubectl check_command "$CLUSTER_TYPE" echo "" @@ -96,8 +125,8 @@ fi # Add Helm repos and update dependencies echo "📚 Setting up Helm dependencies..." -helm repo add bitnami https://charts.bitnami.com/bitnami 2>/dev/null || true -helm repo update +# Subcharts resolve from oci://registry-1.docker.io/bitnamicharts, so no +# classic chart repository needs registering. helm dependency update ./helm/codeapi echo "" From 117c23e76405eee153ec9f07b2a6bc60f9a1ad94 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 12:42:04 -0400 Subject: [PATCH 020/116] =?UTF-8?q?=F0=9F=9B=9C=20feat:=20Add=20Networkles?= =?UTF-8?q?s=20BYOM=20File=20Relay=20(#80)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add networkless BYOM file relay * fix: harden managed file relay * fix: fence relay lifecycle races * fix: order relay handoffs by registration * fix: gate relay workers on runtime readiness * fix: preserve legacy marker listings * fix: recover reclaimed relay staging --- api/src/config.ts | 1 + api/src/download.test.ts | 8 + api/src/job-cleanup.test.ts | 87 ++++ api/src/job.ts | 60 ++- packages/code/README.md | 46 ++- packages/code/src/cli.test.ts | 75 ++++ packages/code/src/cli.ts | 297 ++++++++++---- packages/code/src/protocol.ts | 7 +- packages/code/src/relay-runtime.test.ts | 506 ++++++++++++++++++++++++ packages/code/src/relay-runtime.ts | 425 ++++++++++++++++++++ packages/code/src/relay.test.ts | 383 ++++++++++++++++++ packages/code/src/relay.ts | 276 +++++++++++++ packages/code/src/runtime.test.ts | 39 ++ packages/code/src/runtime.ts | 12 +- packages/code/src/worker.test.ts | 104 +++++ packages/code/src/worker.ts | 36 ++ service/src/bridge/pairing.ts | 5 +- service/src/bridge/router.test.ts | 1 + service/src/bridge/router.ts | 58 ++- service/src/bridge/store.test.ts | 156 +++++++- service/src/bridge/store.ts | 163 +++++++- 21 files changed, 2625 insertions(+), 120 deletions(-) create mode 100644 packages/code/src/relay-runtime.test.ts create mode 100644 packages/code/src/relay-runtime.ts create mode 100644 packages/code/src/relay.test.ts create mode 100644 packages/code/src/relay.ts diff --git a/api/src/config.ts b/api/src/config.ts index 443745f9..d824c0db 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -103,6 +103,7 @@ export const config = { max_input_files: safeInt(process.env.SANDBOX_MAX_INPUT_FILES, 256), prime_concurrency: safeInt(process.env.SANDBOX_PRIME_CONCURRENCY, 8), egress_gateway_url: egressGatewayUrl, + file_relay_token: process.env.SANDBOX_FILE_RELAY_TOKEN ?? '', file_server_url: process.env.FILE_SERVER_URL ?? '', max_nesting_depth: safeInt(process.env.SANDBOX_MAX_NESTING_DEPTH, 10), max_path_length: safeInt(process.env.SANDBOX_MAX_PATH_LENGTH, 256), diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 76e21395..5372a40c 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -110,11 +110,13 @@ let serverPort = 0; const routes = new Map(); let originalFileServerUrl: string; let originalEgressGatewayUrl: string; +let originalFileRelayToken: string; let originalPerJobUids: boolean; beforeAll(() => { originalFileServerUrl = config.file_server_url; originalEgressGatewayUrl = config.egress_gateway_url; + originalFileRelayToken = config.file_relay_token; originalPerJobUids = config.per_job_uids; server = Bun.serve({ port: 0, @@ -151,6 +153,7 @@ beforeAll(() => { afterAll(() => { (config as { file_server_url: string }).file_server_url = originalFileServerUrl; (config as { egress_gateway_url: string }).egress_gateway_url = originalEgressGatewayUrl; + (config as { file_relay_token: string }).file_relay_token = originalFileRelayToken; (config as { per_job_uids: boolean }).per_job_uids = originalPerJobUids; server.stop(true); }); @@ -164,6 +167,7 @@ beforeEach(async () => { afterEach(async () => { (config as { egress_gateway_url: string }).egress_gateway_url = originalEgressGatewayUrl; + (config as { file_relay_token: string }).file_relay_token = originalFileRelayToken; (config as { file_server_url: string }).file_server_url = `http://127.0.0.1:${serverPort}`; (config as { per_job_uids: boolean }).per_job_uids = false; await fsp.rm(tmpDir, { recursive: true, force: true }); @@ -208,6 +212,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { name: 'gateway-fallback.txt', }; let sawGrantHeader = false; + let sawRelayToken = false; let sawInternalHeader = false; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -215,10 +220,12 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { body: 'gateway bytes', onRequest(req) { sawGrantHeader = req.headers.get('x-codeapi-egress-grant') === 'opaque-grant'; + sawRelayToken = req.headers.get('x-librechat-code-relay-token') === 'relay-secret'; sawInternalHeader = req.headers.has('x-codeapi-internal-token'); }, }); (config as { egress_gateway_url: string }).egress_gateway_url = `http://127.0.0.1:${serverPort}`; + (config as { file_relay_token: string }).file_relay_token = 'relay-secret'; (config as { file_server_url: string }).file_server_url = 'http://127.0.0.1:1'; const job = new Job({ @@ -238,6 +245,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(writtenName).toBe('gateway.txt'); expect(sawGrantHeader).toBe(true); + expect(sawRelayToken).toBe(true); expect(sawInternalHeader).toBe(false); expect(await fsp.readFile(path.join(tmpDir, 'gateway.txt'), 'utf8')).toBe('gateway bytes'); }); diff --git a/api/src/job-cleanup.test.ts b/api/src/job-cleanup.test.ts index de4b52e5..0b45a8ab 100644 --- a/api/src/job-cleanup.test.ts +++ b/api/src/job-cleanup.test.ts @@ -22,6 +22,10 @@ interface CleanupInternals { jobIdentity?: SandboxJobIdentity; } +interface MarkerInternals { + autoLoadDirkeep(): Promise; +} + function makeRuntime(): Runtime { return { language: 'bash', @@ -196,4 +200,87 @@ describe('Job cleanup', () => { await fsp.rm(workspace, { recursive: true, force: true }); } }); + + test('bounds inherited marker listing concurrency', async () => { + const files: TFile[] = Array.from( + { length: config.prime_concurrency + 4 }, + (_, index) => ({ + id: `input-${index}`, + name: `input-${index}.txt`, + storage_session_id: `storage-${index}`, + }), + ); + const job = new Job({ + session_id: 'marker-concurrency', + runtime: makeRuntime(), + files, + args: [], + stdin: '', + timeouts: { compile: 5000, run: 5000 }, + cpu_times: { compile: 5000, run: 5000 }, + memory_limits: { compile: 100_000_000, run: 100_000_000 }, + }); + const originalFetch = globalThis.fetch; + let active = 0; + let maxActive = 0; + globalThis.fetch = async () => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise(resolve => setTimeout(resolve, 10)); + active -= 1; + return Response.json([]); + }; + + try { + await (job as unknown as MarkerInternals).autoLoadDirkeep(); + expect(maxActive).toBeLessThanOrEqual(config.prime_concurrency); + expect(maxActive).toBeGreaterThan(1); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test('retries relay backpressure and rejects persistent marker-list failures', async () => { + const makeMarkerJob = (): Job => + new Job({ + session_id: 'marker-backpressure', + runtime: makeRuntime(), + files: [ + { + id: 'input-1', + name: 'input.txt', + storage_session_id: 'storage-1', + }, + ], + args: [], + stdin: '', + timeouts: { compile: 5000, run: 5000 }, + cpu_times: { compile: 5000, run: 5000 }, + memory_limits: { compile: 100_000_000, run: 100_000_000 }, + }); + const originalFetch = globalThis.fetch; + let attempts = 0; + globalThis.fetch = async () => { + attempts += 1; + if (attempts === 1) { + return new Response(null, { + status: 503, + headers: { 'Retry-After': '0' }, + }); + } + return Response.json([]); + }; + + try { + await (makeMarkerJob() as unknown as MarkerInternals).autoLoadDirkeep(); + expect(attempts).toBe(2); + + globalThis.fetch = async () => new Response(null, { status: 502 }); + await expect( + (makeMarkerJob() as unknown as MarkerInternals).autoLoadDirkeep(), + ).rejects.toThrow('HTTP error loading .dirkeep markers: 502'); + } finally { + globalThis.fetch = originalFetch; + } + }); }); diff --git a/api/src/job.ts b/api/src/job.ts index e221ae19..1e78ae75 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -57,6 +57,7 @@ export { } from './validation'; const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; +const AUTO_LOAD_DIRKEEP_RETRIES = 2; /** * Bridges a `fetch` response body to a Node-stream Readable. The types at the @@ -1131,6 +1132,9 @@ export class Job { return injectTraceHeaders({ ...headers, [EGRESS_GRANT_HEADER]: this.egressGrantToken, + ...(config.file_relay_token + ? { 'X-LibreChat-Code-Relay-Token': config.file_relay_token } + : {}), }); } @@ -1143,8 +1147,11 @@ export class Job { .filter(f => !isDirkeep(f.name)) .map(f => f.name); - const fetches = Array.from(sessionIds).map(sid => this.fetchSessionMarkers(sid)); - const results = await Promise.all(fetches); + const results = await mapWithConcurrency( + Array.from(sessionIds), + config.prime_concurrency, + sid => this.fetchSessionMarkers(sid), + ); let added = 0; let hitCap = false; @@ -1166,7 +1173,7 @@ export class Job { /** * Fetches normalized objects for one inherited session and returns the * `.dirkeep` markers belonging to exactly that session. Guards against: - * - non-OK responses (empty list, no throw) + * - legacy 404 responses (no marker support) and transient backpressure * - non-array JSON bodies * - missing/malformed id/name/storage_session_id fields * - MinIO prefix-list leakage (`abc` prefix also matches `abcdef/...`) @@ -1178,20 +1185,43 @@ export class Job { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), AUTO_LOAD_DIRKEEP_TIMEOUT_MS); try { - const res = await fetch( - `${this.fileEgressBaseUrl()}/sessions/${encodeURIComponent(sid)}/objects?detail=normalized`, - { - headers: this.fileEgressHeaders(), - signal: controller.signal, - }, - ); - if (!res.ok) return []; - const data: unknown = await res.json(); - if (!Array.isArray(data)) return []; - return data.filter(isNormalizedObjectForSession(sid)); + for (let attempt = 0; attempt <= AUTO_LOAD_DIRKEEP_RETRIES; attempt += 1) { + const res = await fetch( + `${this.fileEgressBaseUrl()}/sessions/${encodeURIComponent(sid)}/objects?detail=normalized`, + { + headers: this.fileEgressHeaders(), + signal: controller.signal, + }, + ); + if (res.status === 503 && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { + await res.body?.cancel().catch(() => {}); + const retryAfterSeconds = Number(res.headers.get('retry-after')); + await sleep( + Number.isFinite(retryAfterSeconds) + ? Math.min(1000, Math.max(25, retryAfterSeconds * 1000)) + : 100, + controller.signal, + ); + continue; + } + if (res.status === 404) { + await res.body?.cancel().catch(() => {}); + return []; + } + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new Error(`HTTP error loading .dirkeep markers: ${res.status}`); + } + const data: unknown = await res.json(); + if (!Array.isArray(data)) { + throw new Error('Invalid .dirkeep marker response'); + } + return data.filter(isNormalizedObjectForSession(sid)); + } + throw new Error('Exhausted .dirkeep marker retries'); } catch (err) { this.log.warn({ sessionId: sid, err }, 'Failed to auto-load .dirkeep markers'); - return []; + throw err; } finally { clearTimeout(timeout); } diff --git a/packages/code/README.md b/packages/code/README.md index 8e6c63b4..caf129b5 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -96,10 +96,48 @@ reports state loss instead of restarting it. The next assignment starts a new environment. Treat profile changes and Docker restarts as environment resets and preserve any needed workspace contents first. -This first local profile supports inline request files. By-reference inputs and -generated-file uploads require a worker-mediated file relay and are not yet -supported; the runtime remains networkless rather than opening general egress -to reach a file server. +By-reference inputs and generated-file uploads remain disabled unless the +worker-managed file relay is configured. Build the worker image, then point the +relay at the deployment's public egress-gateway base URL: + +```bash +docker build -t librechat-code-worker:local packages/code + +LIBRECHAT_CODE_FILE_RELAY_IMAGE=librechat-code-worker:local \ +LIBRECHAT_CODE_FILE_RELAY_UPSTREAM=https://code.example.com/egress \ +LIBRECHAT_CODE_EXECUTION_MANIFEST_PUBLIC_KEY='' \ +librechat-code run +``` + +The URL is illustrative; it must be the externally reachable HTTPS base URL +for the same Code API deployment's egress-gateway routes. Plain HTTP is accepted +only for loopback and Docker Desktop development hosts. Enabling the relay also +requires signed execution manifests. The worker creates a labeled internal +Docker network for each worker identity, connects the runtime only to that +network, and starts a separate hardened relay container on a labeled, +worker-specific egress network. Reused networks are accepted only when their +internal flag and ownership labels match the required profile. The relay +publishes no host port, accepts only the file-object read, normalized list, and +generated-object write routes, requires both its worker-derived token and the +assignment's scoped egress grant, refuses redirects, and caps request headers, +transfer size, duration, and concurrency. Its upstream is fixed at startup. +Overlapping worker incarnations use separate relay containers; the newly +registered incarnation removes stale relays only after Code API fences the old +incarnation, and orderly shutdown removes its own relay. Relay-capable workers +remain unavailable for dispatch until they activate and health-check the relay, +then confirm readiness for the exact registration incarnation and generation. +Each registration heartbeat revalidates the relay before renewing its +shorter-lived readiness confirmation, so a stopped relay ages out without +creating an availability gap during healthy heartbeats. +Stopped staging containers are reclaimed on the next activation; running +staging containers are reclaimed only after a conservative grace period. + +The trusted runner API can use this relay for file staging. User code still +runs in NsJail's separate network namespace with no interfaces, so it cannot +reach the relay or the public internet. Anyone with access to the Docker daemon +remains inside the trusted worker boundary and can inspect container +configuration and secrets. + Direct NsJail shares the Docker Desktop VM kernel and is suitable for local or operator-trusted development. Use a separate VM or MicroVM boundary for internet-facing execution of code from untrusted users. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 82701e42..00ae78c8 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -163,3 +163,78 @@ test('CLI reset does not require Docker runtime launch inputs', () => { /LIBRECHAT_CODE_(?:RUNTIME_IMAGE|DOCKER_SECCOMP_PROFILE|DOCKER_PACKAGES_PATH) is required/, ); }); + +test('CLI relay requires a fixed upstream URL', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url)), 'relay'], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_FILE_RELAY_UPSTREAM: undefined, + LIBRECHAT_CODE_FILE_RELAY_TOKEN: 'relay-secret', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /LIBRECHAT_CODE_FILE_RELAY_UPSTREAM is required/); +}); + +test('CLI requires manifest verification before enabling the file relay', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-macos-nsjail', + LIBRECHAT_CODE_RUNTIME_IMAGE: 'example/runtime:latest', + LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE: '../../seccomp/nsjail.json', + LIBRECHAT_CODE_DOCKER_PACKAGES_PATH: '.', + LIBRECHAT_CODE_FILE_RELAY_UPSTREAM: 'https://code.example/egress', + LIBRECHAT_CODE_EXECUTION_MANIFEST_PUBLIC_KEY: undefined, + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /LIBRECHAT_CODE_EXECUTION_MANIFEST_PUBLIC_KEY is required/, + ); +}); + +test('CLI treats a whitespace-only file relay upstream as disabled', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + timeout: 500, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-macos-nsjail', + LIBRECHAT_CODE_RUNTIME_IMAGE: 'example/runtime:latest', + LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE: '../../seccomp/nsjail.json', + LIBRECHAT_CODE_DOCKER_PACKAGES_PATH: '.', + LIBRECHAT_CODE_FILE_RELAY_UPSTREAM: ' ', + LIBRECHAT_CODE_EXECUTION_MANIFEST_PUBLIC_KEY: undefined, + LIBRECHAT_CODE_FILE_RELAY_IMAGE: undefined, + }, + }, + ); + + assert.doesNotMatch( + result.stderr, + /LIBRECHAT_CODE_(?:EXECUTION_MANIFEST_PUBLIC_KEY|FILE_RELAY_IMAGE) is required/, + ); +}); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 457af16f..9aa6f589 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,9 +1,11 @@ #!/usr/bin/env node -import { createHash } from 'node:crypto'; +import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { startFileRelay } from './relay.js'; +import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { defaultBridgeIdentityPath, loadBridgeIdentity, @@ -31,6 +33,15 @@ function list(value: string | undefined): string[] { ); } +function positiveInteger(name: string, value: string | undefined, fallback: number): number { + if (value == null || value.trim().length === 0) return fallback; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return parsed; +} + const MACOS_NSJAIL_CAPABILITIES = [ 'SYS_ADMIN', 'SYS_CHROOT', @@ -71,6 +82,41 @@ async function pair(args: string[]): Promise { `Paired worker ${workerId}. Identity saved to ${identityPath}\n`, ); } + +async function relay(): Promise { + const handle = await startFileRelay({ + host: process.env.LIBRECHAT_CODE_FILE_RELAY_HOST?.trim() || '0.0.0.0', + port: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_PORT', + process.env.LIBRECHAT_CODE_FILE_RELAY_PORT, + 3000, + ), + upstreamUrl: required('LIBRECHAT_CODE_FILE_RELAY_UPSTREAM'), + token: required('LIBRECHAT_CODE_FILE_RELAY_TOKEN'), + maxBytes: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_MAX_BYTES', + process.env.LIBRECHAT_CODE_FILE_RELAY_MAX_BYTES, + 16 * 1024 * 1024, + ), + timeoutMs: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_TIMEOUT_MS', + process.env.LIBRECHAT_CODE_FILE_RELAY_TIMEOUT_MS, + 30_000, + ), + maxConcurrentRequests: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_MAX_CONCURRENT_REQUESTS', + process.env.LIBRECHAT_CODE_FILE_RELAY_MAX_CONCURRENT_REQUESTS, + 8, + ), + }); + process.stdout.write(`librechat-code: file relay listening at ${handle.url}\n`); + await new Promise((resolve) => { + process.once('SIGINT', resolve); + process.once('SIGTERM', resolve); + }); + await handle.close(); +} + async function run(runtimeSessionId?: string): Promise { const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); @@ -135,6 +181,12 @@ async function run(runtimeSessionId?: string): Promise { expiresAt: pairedIdentity.expiresAt, } : undefined; + const fileRelayUpstream = + process.env.LIBRECHAT_CODE_FILE_RELAY_UPSTREAM?.trim(); + const fileRelayEnabled = + runtimeMode === 'docker-macos-nsjail' && + runtimeSessionId == null && + (fileRelayUpstream?.length ?? 0) > 0; const capabilities = { statefulWorkspace, sandboxProfile: @@ -142,6 +194,7 @@ async function run(runtimeSessionId?: string): Promise { (runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), policyDigest: createHash('sha256').update(policy).digest('hex'), + ...(fileRelayEnabled ? { requiresReadyConfirmation: true } : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { throw new Error( @@ -151,84 +204,186 @@ async function run(runtimeSessionId?: string): Promise { const controller = new AbortController(); process.once('SIGINT', () => controller.abort()); process.once('SIGTERM', () => controller.abort()); - const worker = new BridgeWorker({ - codeApiUrl, - token: configuredToken, - identity: workerIdentity, - workerId, - runtimeSupervisor: - runtimeMode !== 'endpoint' - ? new DockerRuntimeSupervisor({ - image: - runtimeSessionId == null - ? required('LIBRECHAT_CODE_RUNTIME_IMAGE') - : process.env.LIBRECHAT_CODE_RUNTIME_IMAGE?.trim(), - ...(runtimeMode === 'docker-macos-nsjail' && runtimeSessionId == null - ? (() => { - const seccompProfile = resolve( - required('LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE'), - ); - const packagesPath = resolve( - required('LIBRECHAT_CODE_DOCKER_PACKAGES_PATH'), - ); - return { - capabilities: MACOS_NSJAIL_CAPABILITIES, - securityOptions: [`seccomp=${seccompProfile}`], - profileRevision: createHash('sha256') - .update(readFileSync(seccompProfile)) - .digest('hex'), - restartStoppedContainers: false, - bindMounts: [ - { - source: packagesPath, - target: '/pkgs', - readOnly: true, + const incarnationId = randomBytes(18).toString('base64url'); + const runtimeImage = + runtimeMode !== 'endpoint' + ? runtimeSessionId == null + ? required('LIBRECHAT_CODE_RUNTIME_IMAGE') + : process.env.LIBRECHAT_CODE_RUNTIME_IMAGE?.trim() + : undefined; + const macLaunchProfile = + runtimeMode === 'docker-macos-nsjail' && runtimeSessionId == null + ? (() => { + const seccompProfile = resolve( + required('LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE'), + ); + const packagesPath = resolve( + required('LIBRECHAT_CODE_DOCKER_PACKAGES_PATH'), + ); + return { + seccompProfile, + packagesPath, + profileRevision: createHash('sha256') + .update(readFileSync(seccompProfile)) + .digest('hex'), + }; + })() + : undefined; + const executionManifestPublicKey = fileRelayEnabled + ? required('LIBRECHAT_CODE_EXECUTION_MANIFEST_PUBLIC_KEY') + : undefined; + const fileRelayLimits = fileRelayEnabled + ? { + maxBytes: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_MAX_BYTES', + process.env.LIBRECHAT_CODE_FILE_RELAY_MAX_BYTES, + 16 * 1024 * 1024, + ), + timeoutMs: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_TIMEOUT_MS', + process.env.LIBRECHAT_CODE_FILE_RELAY_TIMEOUT_MS, + 30_000, + ), + maxConcurrentRequests: positiveInteger( + 'LIBRECHAT_CODE_FILE_RELAY_MAX_CONCURRENT_REQUESTS', + process.env.LIBRECHAT_CODE_FILE_RELAY_MAX_CONCURRENT_REQUESTS, + 8, + ), + } + : undefined; + const fileRelaySupervisor = + fileRelayEnabled && fileRelayUpstream + ? new DockerFileRelaySupervisor({ + workerId, + incarnationId, + image: required('LIBRECHAT_CODE_FILE_RELAY_IMAGE'), + upstreamUrl: fileRelayUpstream, + ...fileRelayLimits, + token: createHmac( + 'sha256', + pairedIdentity?.privateKey ?? + required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken), + ) + .update('librechat-code-file-relay-v1') + .digest('hex'), + }) + : undefined; + const fileRelayProfile = await fileRelaySupervisor?.prepare( + controller.signal, + ); + try { + const worker = new BridgeWorker({ + codeApiUrl, + token: configuredToken, + identity: workerIdentity, + workerId, + incarnationId, + runtimeSupervisor: + runtimeMode !== 'endpoint' + ? new DockerRuntimeSupervisor({ + image: runtimeImage, + ...(runtimeMode === 'docker-macos-nsjail' && runtimeSessionId == null + ? (() => { + const { seccompProfile, packagesPath, profileRevision } = + macLaunchProfile!; + return { + capabilities: MACOS_NSJAIL_CAPABILITIES, + securityOptions: [`seccomp=${seccompProfile}`], + profileRevision, + restartStoppedContainers: false, + ...(fileRelayProfile + ? { network: fileRelayProfile.network } + : {}), + bindMounts: [ + { + source: packagesPath, + target: '/pkgs', + readOnly: true, + }, + ], + httpClient: 'bun', + environment: { + SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', + ...(fileRelayProfile + ? { + EGRESS_GATEWAY_URL: fileRelayProfile.url, + SANDBOX_PRIME_CONCURRENCY: String( + fileRelayLimits!.maxConcurrentRequests, + ), + SANDBOX_UPLOAD_CONCURRENCY: String( + fileRelayLimits!.maxConcurrentRequests, + ), + SANDBOX_FILE_RELAY_TOKEN: fileRelayProfile.token, + SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: + executionManifestPublicKey!, + } + : {}), }, - ], - httpClient: 'bun', - environment: { - SANDBOX_USE_CGROUPV2: 'false', - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', - }, - }; - })() - : {}), - }) - : new EndpointRuntimeSupervisor({ - endpoint: sandboxEndpoint, - statefulWorkspace, - }), - capabilities, - onIdentityChange: - pairedIdentity && identityPath - ? async (identity) => { - await saveBridgeIdentity(identityPath, { - ...pairedIdentity, - credential: identity.credential, - expiresAt: identity.expiresAt, - }); + }; + })() + : {}), + }) + : new EndpointRuntimeSupervisor({ + endpoint: sandboxEndpoint, + statefulWorkspace, + }), + capabilities, + onIdentityChange: + pairedIdentity && identityPath + ? async (identity) => { + await saveBridgeIdentity(identityPath, { + ...pairedIdentity, + credential: identity.credential, + expiresAt: identity.expiresAt, + }); + } + : undefined, + onRegistered: fileRelaySupervisor + ? async (registration) => { + if ( + registration.registrationGeneration == null || + !Number.isSafeInteger(registration.registrationGeneration) || + registration.registrationGeneration < 1 + ) { + throw new Error( + 'Code API does not support registration-ordered file relay activation', + ); + } + await fileRelaySupervisor.activate( + registration.registrationGeneration, + controller.signal, + ); } : undefined, - onError: (error) => { - const message = - error instanceof Error ? error.message : 'unknown bridge error'; - process.stderr.write(`librechat-code: reconnecting after ${message}\n`); - }, - }); - if (runtimeSessionId !== undefined) { - await worker.refreshCredential(controller.signal); - await worker.register(controller.signal); - await worker.resetWorkspace(runtimeSessionId, controller.signal); - process.stdout.write( - `librechat-code: reset acknowledged for ${runtimeSessionId}\n`, - ); - return; + onError: (error) => { + const message = + error instanceof Error ? error.message : 'unknown bridge error'; + process.stderr.write(`librechat-code: reconnecting after ${message}\n`); + }, + }); + if (runtimeSessionId !== undefined) { + await worker.refreshCredential(controller.signal); + await worker.register(controller.signal); + await worker.resetWorkspace(runtimeSessionId, controller.signal); + process.stdout.write( + `librechat-code: reset acknowledged for ${runtimeSessionId}\n`, + ); + return; + } + await worker.run(controller.signal); + } finally { + await fileRelaySupervisor?.stop(); } - await worker.run(controller.signal); } async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'relay') { + await relay(); + return; + } if (args[0] === 'pair') { await pair(args); return; diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index b2027e1c..a9ff5649 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -11,6 +11,7 @@ export interface BridgeWorkerCapabilities { sandboxProfile: string; runtimes: string[]; policyDigest?: string; + requiresReadyConfirmation?: boolean; } export interface BridgeWorkerRegistration { @@ -24,6 +25,8 @@ export interface BridgeWorkerRegistrationResponse { protocolVersion: BridgeProtocolVersion; workerId: string; incarnationId: string; + /** Monotonic per-worker generation allocated when the active incarnation changes. */ + registrationGeneration?: number; registeredAt: string; leaseTtlMs: number; } @@ -138,6 +141,8 @@ export function isValidBridgeWorkerCapabilities( ) && (capabilities.policyDigest === undefined || (typeof capabilities.policyDigest === 'string' && - /^[a-f0-9]{64}$/.test(capabilities.policyDigest))) + /^[a-f0-9]{64}$/.test(capabilities.policyDigest))) && + (capabilities.requiresReadyConfirmation === undefined || + typeof capabilities.requiresReadyConfirmation === 'boolean') ); } diff --git a/packages/code/src/relay-runtime.test.ts b/packages/code/src/relay-runtime.test.ts new file mode 100644 index 00000000..bdbcf5f1 --- /dev/null +++ b/packages/code/src/relay-runtime.test.ts @@ -0,0 +1,506 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { DockerFileRelaySupervisor } from './relay-runtime.js'; + +import type { ContainerRuntimeClient } from './runtime.js'; + +test('Docker file relay prepares a private runtime network and hardened dual-homed relay', async () => { + const calls: string[][] = []; + let staleRelay = ''; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'network' && args[1] === 'inspect') { + throw new Error('network not found'); + } + if (args[0] === 'container' && args[1] === 'rm') { + throw new Error('No such container'); + } + if (args[0] === 'container' && args[1] === 'ls') return staleRelay; + if (args[0] === 'container' && args[1] === 'rename') return ''; + if (args[0] === 'network' && args[1] === 'create') return 'network-id\n'; + if (args[0] === 'run') return 'relay-id\n'; + if (args[0] === 'network' && args[1] === 'connect') return ''; + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'incarnation-one', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + maxBytes: 20_000_000, + timeoutMs: 45_000, + maxConcurrentRequests: 4, + client, + }); + + const profile = await supervisor.prepare(); + + assert.match(profile.network, /^librechat-code-relay-/); + assert.equal(profile.url, 'http://relay:3000'); + assert.equal(profile.token, 'relay-secret'); + const networkCreates = calls.filter( + (args) => args[0] === 'network' && args[1] === 'create', + ); + assert.equal(networkCreates.length, 2); + assert.equal(networkCreates.filter((args) => args.includes('--internal')).length, 1); + const run = calls.find((args) => args[0] === 'run') ?? []; + const stagingRelay = run[run.indexOf('--name') + 1] ?? ''; + assert.match(stagingRelay, /^librechat-code-relay-.+-staging-[a-f0-9]{12}$/); + assert.match(run[run.indexOf('--network') + 1] ?? '', /^librechat-code-egress-/); + assert.ok(run.includes('LIBRECHAT_CODE_FILE_RELAY_MAX_BYTES=20000000')); + assert.ok(run.includes('LIBRECHAT_CODE_FILE_RELAY_TIMEOUT_MS=45000')); + assert.ok(run.includes('LIBRECHAT_CODE_FILE_RELAY_MAX_CONCURRENT_REQUESTS=4')); + assert.ok(run.includes('ALL')); + assert.ok(run.includes('no-new-privileges:true')); + assert.equal(run.includes('--publish'), false); + assert.equal( + calls.some((args) => args[0] === 'network' && args[1] === 'connect'), + false, + ); + staleRelay = stagingRelay.replace('-staging-', '-g1-'); + await supervisor.activate(2); + const rename = calls.find( + (args) => args[0] === 'container' && args[1] === 'rename', + ); + assert.match(rename?.at(-1) ?? '', /-g2-[a-f0-9]{12}$/); + const connect = calls.find( + (args) => args[0] === 'network' && args[1] === 'connect', + ); + assert.ok(connect?.includes('--alias')); + assert.ok(connect?.includes('relay')); + assert.ok(connect?.includes(profile.network)); + const health = calls.find((args) => args[0] === 'exec') ?? []; + assert.match(health.at(-1) ?? '', /^\d+$/); + assert.ok(calls.indexOf(health) < calls.indexOf(connect ?? [])); + + assert.ok( + calls.some( + (args) => + args[0] === 'container' && + args[1] === 'rm' && + args.includes(staleRelay), + ), + ); + const removalsBeforeStop = calls.filter( + (args) => args[0] === 'container' && args[1] === 'rm', + ).length; + await supervisor.stop(); + assert.equal( + calls.filter((args) => args[0] === 'container' && args[1] === 'rm').length, + removalsBeforeStop + 1, + ); +}); + +test('Docker file relay fails closed when a reused runtime network is not internal', async () => { + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'network' && args[1] === 'inspect') { + return 'false|true|runtime\n'; + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'incarnation-two', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + client, + }); + + await assert.rejects(supervisor.prepare(), /does not match its required profile/); +}); + +test('Docker file relay validates a network created by a concurrent incarnation', async () => { + let runtimeInspections = 0; + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'network' && args[1] === 'inspect') { + const isRuntime = args.at(-1)?.includes('-relay-') === true; + if (!isRuntime) return 'false|true|egress\n'; + runtimeInspections += 1; + if (runtimeInspections === 1) throw new Error('network not found'); + return 'true|true|runtime\n'; + } + if (args[0] === 'network' && args[1] === 'create') { + throw new Error('network with name already exists'); + } + if (args[0] === 'container' && args[1] === 'rm') { + throw new Error('No such container'); + } + if (args[0] === 'run') return 'relay-id\n'; + if (args[0] === 'network' && args[1] === 'connect') return ''; + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'incarnation-three', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + client, + }); + + await supervisor.prepare(); + + assert.equal(runtimeInspections, 2); +}); + +test('Docker file relay rejects a delayed lower registration generation', async () => { + const calls: string[][] = []; + let currentRelay = ''; + let newerRelay = ''; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'network' && args[1] === 'inspect') { + return args.at(-1)?.includes('-egress-') === true + ? 'false|true|egress\n' + : 'true|true|runtime\n'; + } + if (args[0] === 'container' && args[1] === 'rm') { + return 'removed\n'; + } + if (args[0] === 'run') { + currentRelay = args[args.indexOf('--name') + 1] ?? ''; + newerRelay = currentRelay.replace( + /-staging-[a-f0-9]{12}$/, + '-g2-ffffffffffff', + ); + return 'relay-id\n'; + } + if (args[0] === 'container' && args[1] === 'rename') return ''; + if (args[0] === 'network' && args[1] === 'connect') return ''; + if (args[0] === 'exec') return '200'; + if (args[0] === 'container' && args[1] === 'ls') return `${newerRelay}\n`; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'older-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + client, + }); + await supervisor.prepare(); + + await assert.rejects( + supervisor.activate(1), + /registration was superseded before activation/, + ); + + assert.ok( + calls.some( + (args) => + args[0] === 'container' && + args[1] === 'rm' && + args.includes(currentRelay.replace('-staging-', '-g1-')), + ), + ); + assert.equal( + calls.some( + (args) => args[0] === 'container' && args[1] === 'rm' && args.includes(newerRelay), + ), + false, + ); + assert.equal( + calls.some((args) => args[0] === 'network' && args[1] === 'connect'), + false, + ); +}); + +test('Docker file relay handoff follows registration order instead of creation order', async () => { + const containers = new Set(); + const connected: string[] = []; + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'network' && args[1] === 'inspect') { + return args.at(-1)?.includes('-egress-') === true + ? 'false|true|egress\n' + : 'true|true|runtime\n'; + } + if (args[0] === 'container' && args[1] === 'rm') { + const name = args.at(-1) ?? ''; + if (!containers.delete(name)) throw new Error('No such container'); + return 'removed\n'; + } + if (args[0] === 'run') { + containers.add(args[args.indexOf('--name') + 1] ?? ''); + return 'relay-id\n'; + } + if (args[0] === 'exec') return '200'; + if (args[0] === 'container' && args[1] === 'rename') { + const previous = args.at(-2) ?? ''; + const next = args.at(-1) ?? ''; + if (!containers.delete(previous)) throw new Error('No such container'); + containers.add(next); + return ''; + } + if (args[0] === 'container' && args[1] === 'ls') { + return `${Array.from(containers) + .map((name) => `${name}|running|1000`) + .join('\n')}\n`; + } + if (args[0] === 'network' && args[1] === 'connect') { + const name = args.at(-1) ?? ''; + if (!containers.has(name)) throw new Error('No such container'); + connected.push(name); + return ''; + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const olderCreated = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'older-created-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + now: () => 1_000, + client, + }); + const newerCreated = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'newer-created-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + now: () => 1_000, + client, + }); + + await olderCreated.prepare(); + await newerCreated.prepare(); + await newerCreated.activate(1); + await olderCreated.activate(2); + + assert.equal(containers.size, 1); + assert.match(Array.from(containers)[0] ?? '', /-g2-[a-f0-9]{12}$/); + assert.match(connected[0] ?? '', /-g1-[a-f0-9]{12}$/); + assert.match(connected[1] ?? '', /-g2-[a-f0-9]{12}$/); +}); + +test('Docker file relay reclaims stopped and expired staging containers', async () => { + const removed: string[] = []; + let staging = ''; + let listCalls = 0; + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'network' && args[1] === 'inspect') { + return args.at(-1)?.includes('-egress-') === true + ? 'false|true|egress\n' + : 'true|true|runtime\n'; + } + if (args[0] === 'container' && args[1] === 'rm') { + removed.push(args.at(-1) ?? ''); + return 'removed\n'; + } + if (args[0] === 'run') { + staging = args[args.indexOf('--name') + 1] ?? ''; + return 'relay-id\n'; + } + if (args[0] === 'exec') return '200'; + if (args[0] === 'container' && args[1] === 'rename') return ''; + if (args[0] === 'network' && args[1] === 'connect') return ''; + if (args[0] === 'container' && args[1] === 'ls') { + listCalls += 1; + if (listCalls === 1) return ''; + const prefix = staging.replace(/-staging-[a-f0-9]{12}$/, '-staging-'); + return [ + `${prefix}111111111111|exited|99000`, + `${prefix}222222222222|running|1`, + `${prefix}333333333333|running|99000`, + ].join('\n'); + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'current-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + stagingGraceMs: 10_000, + now: () => 100_000, + client, + }); + + await supervisor.prepare(); + await supervisor.activate(1); + await supervisor.activate(1); + + assert.ok(removed.some((name) => name.endsWith('111111111111'))); + assert.ok(removed.some((name) => name.endsWith('222222222222'))); + assert.equal(removed.some((name) => name.endsWith('333333333333')), false); +}); + +test('Docker file relay relaunches an unhealthy active generation', async () => { + let relay = ''; + let launches = 0; + let healthChecks = 0; + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'network' && args[1] === 'inspect') { + return args.at(-1)?.includes('-egress-') === true + ? 'false|true|egress\n' + : 'true|true|runtime\n'; + } + if (args[0] === 'container' && args[1] === 'rm') return 'removed\n'; + if (args[0] === 'run') { + launches += 1; + relay = args[args.indexOf('--name') + 1] ?? ''; + return 'relay-id\n'; + } + if (args[0] === 'exec') { + healthChecks += 1; + if (healthChecks === 2) throw new Error('container is not running'); + return '200'; + } + if (args[0] === 'container' && args[1] === 'rename') { + relay = args.at(-1) ?? ''; + return ''; + } + if (args[0] === 'container' && args[1] === 'ls') { + return `${relay}|running|1000\n`; + } + if (args[0] === 'network' && args[1] === 'connect') return ''; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'recovering-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + client, + }); + + await supervisor.prepare(); + await supervisor.activate(1); + await supervisor.activate(1); + + assert.equal(launches, 2); + assert.ok(healthChecks >= 3); +}); + +test('Docker file relay relaunches when its live staging container was reclaimed', async () => { + let container = ''; + let launches = 0; + let renameAttempts = 0; + let connected = false; + const client: ContainerRuntimeClient = { + async run(args) { + if (args[0] === 'network' && args[1] === 'inspect') { + return args.at(-1)?.includes('-egress-') === true + ? 'false|true|egress\n' + : 'true|true|runtime\n'; + } + if (args[0] === 'container' && args[1] === 'rm') { + const name = args.at(-1) ?? ''; + if (name !== container || container === '') { + throw new Error('No such container'); + } + container = ''; + return 'removed\n'; + } + if (args[0] === 'run') { + launches += 1; + container = args[args.indexOf('--name') + 1] ?? ''; + return 'relay-id\n'; + } + if (args[0] === 'exec') return '200'; + if (args[0] === 'container' && args[1] === 'rename') { + renameAttempts += 1; + if (renameAttempts === 1) { + container = ''; + throw new Error('No such container'); + } + container = args.at(-1) ?? ''; + return ''; + } + if (args[0] === 'container' && args[1] === 'ls') { + return `${container}|running|1000\n`; + } + if (args[0] === 'network' && args[1] === 'connect') { + connected = true; + return ''; + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'delayed-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + client, + }); + + await supervisor.prepare(); + await supervisor.activate(2); + + assert.equal(launches, 2); + assert.equal(renameAttempts, 2); + assert.equal(connected, true); + assert.match(container, /-g2-[a-f0-9]{12}$/); +}); + +test('Docker file relay rolls back startup with a fresh signal after abort', async () => { + const controller = new AbortController(); + let currentRelay = ''; + let cleanupCalls = 0; + let cleanupSignal: AbortSignal | undefined; + const client: ContainerRuntimeClient = { + async run(args, options) { + if (args[0] === 'network' && args[1] === 'inspect') { + return args.at(-1)?.includes('-egress-') === true + ? 'false|true|egress\n' + : 'true|true|runtime\n'; + } + if (args[0] === 'container' && args[1] === 'rm') { + if (args.includes(currentRelay)) { + cleanupCalls += 1; + cleanupSignal = options?.signal; + } + return 'removed\n'; + } + if (args[0] === 'run') { + currentRelay = args[args.indexOf('--name') + 1] ?? ''; + return 'relay-id\n'; + } + if (args[0] === 'network' && args[1] === 'connect') return ''; + if (args[0] === 'exec') { + controller.abort(new Error('shutdown')); + throw controller.signal.reason; + } + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerFileRelaySupervisor({ + workerId: 'engineering-vm', + incarnationId: 'aborted-incarnation', + image: 'librechat-code-worker:local', + upstreamUrl: 'https://code.example/egress', + token: 'relay-secret', + startupTimeoutMs: 1, + client, + }); + + await assert.rejects(supervisor.prepare(controller.signal), /shutdown/); + + assert.equal(cleanupCalls, 1); + assert.equal(cleanupSignal?.aborted ?? false, false); +}); diff --git a/packages/code/src/relay-runtime.ts b/packages/code/src/relay-runtime.ts new file mode 100644 index 00000000..889028a3 --- /dev/null +++ b/packages/code/src/relay-runtime.ts @@ -0,0 +1,425 @@ +import { createHash } from 'node:crypto'; + +import { validateFileRelayUpstream } from './relay.js'; +import { DockerCliClient } from './runtime.js'; + +import type { ContainerRuntimeClient } from './runtime.js'; + +export interface DockerFileRelaySupervisorOptions { + workerId: string; + incarnationId: string; + image: string; + upstreamUrl: string; + token: string; + maxBytes?: number; + timeoutMs?: number; + maxConcurrentRequests?: number; + dockerCommand?: string; + startupTimeoutMs?: number; + stagingGraceMs?: number; + now?: () => number; + client?: ContainerRuntimeClient; +} + +export interface DockerFileRelayProfile { + network: string; + url: string; + token: string; +} + +const DEFAULT_STARTUP_TIMEOUT_MS = 30_000; +const DEFAULT_STAGING_GRACE_MS = 5 * 60_000; +const TERMINAL_CONTAINER_STATES = new Set(['dead', 'exited']); + +function suffix(workerId: string): string { + return createHash('sha256').update(workerId).digest('hex').slice(0, 20); +} + +function missingNetwork(error: unknown): boolean { + return ( + error instanceof Error && + /(?:network(?: .*?)? not found|no such network)/i.test(error.message) + ); +} + +function existingNetwork(error: unknown): boolean { + return error instanceof Error && /network .* already exists/i.test(error.message); +} + +function missingContainer(error: unknown): boolean { + return ( + error instanceof Error && + /(?:no such container|no such object)/i.test(error.message) + ); +} + +export class DockerFileRelaySupervisor { + private readonly client: ContainerRuntimeClient; + private readonly network: string; + private readonly egressNetwork: string; + private readonly containerPrefix: string; + private readonly incarnationHash: string; + private readonly workerHash: string; + private container: string; + private prepared = false; + private activeGeneration?: number; + + constructor(private readonly options: DockerFileRelaySupervisorOptions) { + if (!options.image.trim()) { + throw new Error('Docker file relay image is required'); + } + if (!options.token.trim()) { + throw new Error('Docker file relay token is required'); + } + validateFileRelayUpstream(options.upstreamUrl); + for (const [name, value] of [ + ['maxBytes', options.maxBytes], + ['timeoutMs', options.timeoutMs], + ['maxConcurrentRequests', options.maxConcurrentRequests], + ['stagingGraceMs', options.stagingGraceMs], + ] as const) { + if (value != null && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`Docker file relay ${name} must be a positive integer`); + } + } + this.workerHash = suffix(options.workerId); + this.incarnationHash = suffix(options.incarnationId).slice(0, 12); + this.network = `librechat-code-relay-${this.workerHash}`; + this.egressNetwork = `librechat-code-egress-${this.workerHash}`; + this.containerPrefix = `librechat-code-relay-${this.workerHash}`; + this.container = this.stagingContainer(); + this.client = options.client ?? new DockerCliClient(options.dockerCommand); + } + + async prepare(signal?: AbortSignal): Promise { + await this.ensureNetwork(this.network, true, 'runtime', signal); + await this.ensureNetwork(this.egressNetwork, false, 'egress', signal); + await this.launchRelay(signal); + return { + network: this.network, + url: 'http://relay:3000', + token: this.options.token, + }; + } + + async stop(signal?: AbortSignal): Promise { + await this.removeRelay(signal); + } + + async activate( + registrationGeneration: number, + signal?: AbortSignal, + ): Promise { + if ( + !Number.isSafeInteger(registrationGeneration) || + registrationGeneration < 1 + ) { + throw new Error('Docker file relay registration generation is invalid'); + } + let activeAndHealthy = false; + if (this.activeGeneration === registrationGeneration) { + try { + await this.checkHealth(signal); + activeAndHealthy = true; + } catch { + await this.removeRelay(); + } + } + if (!this.prepared) await this.launchRelay(signal); + const activeContainer = `${this.containerPrefix}-g${registrationGeneration}-${this.incarnationHash}`; + if (this.container !== activeContainer) { + try { + await this.renameRelay(activeContainer, signal); + } catch (error) { + if (!missingContainer(error)) throw error; + this.resetPreparedState(); + await this.launchRelay(signal); + try { + await this.renameRelay(activeContainer, signal); + } catch (retryError) { + if (missingContainer(retryError)) this.resetPreparedState(); + throw retryError; + } + } + this.container = activeContainer; + } + const containers = await this.client.run( + [ + 'container', + 'ls', + '--all', + '--filter', + 'label=com.librechat.code.file-relay=true', + '--filter', + `label=com.librechat.code.worker-hash=${this.workerHash}`, + '--format', + '{{.Names}}|{{.State}}|{{.Label "com.librechat.code.file-relay-staged-at"}}', + ], + { signal }, + ); + const olderContainers: string[] = []; + for (const record of containers.split('\n').map((value) => value.trim())) { + const [name, state = '', stagedAtValue = ''] = record.split('|'); + if (!name || name === this.container) continue; + if (name.startsWith(`${this.containerPrefix}-staging-`)) { + const stagedAt = Number(stagedAtValue); + const staleStage = + Number.isSafeInteger(stagedAt) && + stagedAt > 0 && + (this.options.now?.() ?? Date.now()) - stagedAt >= + (this.options.stagingGraceMs ?? DEFAULT_STAGING_GRACE_MS); + if ( + TERMINAL_CONTAINER_STATES.has(state.toLowerCase()) || + staleStage + ) { + olderContainers.push(name); + } + continue; + } + const candidateGeneration = this.registrationGeneration(name); + if (candidateGeneration == null) { + throw new Error('Docker returned an invalid stale file relay name'); + } + if (candidateGeneration > registrationGeneration) { + await this.removeRelay(); + throw new Error( + 'Docker file relay registration was superseded before activation', + ); + } + if (candidateGeneration === registrationGeneration) { + throw new Error('Docker returned conflicting file relay registrations'); + } + olderContainers.push(name); + } + for (const name of olderContainers) { + try { + await this.client.run(['container', 'rm', '--force', name], { signal }); + } catch (error) { + if (!missingContainer(error)) throw error; + } + } + if (activeAndHealthy) return; + try { + await this.client.run( + [ + 'network', + 'connect', + '--alias', + 'relay', + this.network, + this.container, + ], + { signal }, + ); + } catch (error) { + if (missingContainer(error)) this.resetPreparedState(); + throw error; + } + this.activeGeneration = registrationGeneration; + } + + private registrationGeneration(name: string): number | undefined { + const match = new RegExp( + `^${this.containerPrefix}-g([1-9][0-9]*)-[a-f0-9]{12}$`, + ).exec(name); + if (match == null) return undefined; + const generation = Number(match[1]); + return Number.isSafeInteger(generation) ? generation : undefined; + } + + private stagingContainer(): string { + return `${this.containerPrefix}-staging-${this.incarnationHash}`; + } + + private async renameRelay( + activeContainer: string, + signal?: AbortSignal, + ): Promise { + await this.client.run( + ['container', 'rename', this.container, activeContainer], + { signal }, + ); + } + + private async launchRelay(signal?: AbortSignal): Promise { + this.container = this.stagingContainer(); + await this.removeRelay(signal); + this.container = this.stagingContainer(); + try { + await this.client.run( + [ + 'run', + '--detach', + '--name', + this.container, + '--network', + this.egressNetwork, + '--cap-drop', + 'ALL', + '--security-opt', + 'no-new-privileges:true', + '--read-only', + '--tmpfs', + '/tmp:rw,noexec,nosuid,size=16m', + '--label', + 'com.librechat.code.file-relay=true', + '--label', + `com.librechat.code.worker-hash=${this.workerHash}`, + '--label', + `com.librechat.code.file-relay-staged-at=${this.options.now?.() ?? Date.now()}`, + '--env', + `LIBRECHAT_CODE_FILE_RELAY_UPSTREAM=${this.options.upstreamUrl}`, + '--env', + `LIBRECHAT_CODE_FILE_RELAY_TOKEN=${this.options.token}`, + ...(this.options.maxBytes != null + ? [ + '--env', + `LIBRECHAT_CODE_FILE_RELAY_MAX_BYTES=${this.options.maxBytes}`, + ] + : []), + ...(this.options.timeoutMs != null + ? [ + '--env', + `LIBRECHAT_CODE_FILE_RELAY_TIMEOUT_MS=${this.options.timeoutMs}`, + ] + : []), + ...(this.options.maxConcurrentRequests != null + ? [ + '--env', + `LIBRECHAT_CODE_FILE_RELAY_MAX_CONCURRENT_REQUESTS=${this.options.maxConcurrentRequests}`, + ] + : []), + this.options.image, + 'relay', + ], + { signal }, + ); + await this.waitForHealth(signal); + this.prepared = true; + } catch (error) { + await this.removeRelay(); + throw error; + } + } + + private async ensureNetwork( + name: string, + internal: boolean, + role: 'runtime' | 'egress', + signal?: AbortSignal, + ): Promise { + try { + await this.validateNetwork(name, internal, role, signal); + } catch (error) { + if (!missingNetwork(error)) throw error; + try { + await this.client.run( + [ + 'network', + 'create', + ...(internal ? ['--internal'] : []), + '--label', + 'com.librechat.code.file-relay=true', + '--label', + `com.librechat.code.network-role=${role}`, + name, + ], + { signal }, + ); + } catch (createError) { + if (!existingNetwork(createError)) throw createError; + await this.validateNetwork(name, internal, role, signal); + } + } + } + + private async validateNetwork( + name: string, + internal: boolean, + role: 'runtime' | 'egress', + signal?: AbortSignal, + ): Promise { + const profile = await this.client.run( + [ + 'network', + 'inspect', + '--format', + '{{.Internal}}|{{index .Labels "com.librechat.code.file-relay"}}|{{index .Labels "com.librechat.code.network-role"}}', + name, + ], + { signal }, + ); + const expected = `${String(internal)}|true|${role}`; + if (profile.trim() !== expected) { + throw new Error( + `Docker file relay network ${name} does not match its required profile`, + ); + } + } + + private async removeRelay(signal?: AbortSignal): Promise { + try { + await this.client.run(['container', 'rm', '--force', this.container], { + signal, + }); + } catch (error) { + if (!missingContainer(error)) throw error; + } + this.resetPreparedState(); + } + + private resetPreparedState(): void { + this.container = this.stagingContainer(); + this.prepared = false; + this.activeGeneration = undefined; + } + + private async waitForHealth(signal?: AbortSignal): Promise { + const deadline = + Date.now() + + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS); + let lastError: unknown; + while (Date.now() < deadline) { + if (signal?.aborted) { + throw signal.reason ?? new DOMException('aborted', 'AbortError'); + } + try { + const remainingMs = Math.max(1, deadline - Date.now()); + await this.checkHealth(signal, remainingMs); + return; + } catch (error) { + if (signal?.aborted) { + throw signal.reason ?? new DOMException('aborted', 'AbortError'); + } + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error('Docker file relay did not become healthy', { + cause: lastError, + }); + } + + private async checkHealth( + signal?: AbortSignal, + timeoutMs = Math.max( + 1, + this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS, + ), + ): Promise { + const status = await this.client.run( + [ + 'exec', + this.container, + 'node', + '-e', + "fetch('http://127.0.0.1:3000/health',{headers:{'X-LibreChat-Code-Relay-Token':process.env.LIBRECHAT_CODE_FILE_RELAY_TOKEN},signal:AbortSignal.timeout(Number(process.argv.at(-1)))}).then(r=>process.stdout.write(String(r.status)))", + String(timeoutMs), + ], + { signal }, + ); + if (status.trim() !== '200') { + throw new Error(`File relay health check returned HTTP ${status.trim()}`); + } + } +} diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts new file mode 100644 index 00000000..c0814408 --- /dev/null +++ b/packages/code/src/relay.test.ts @@ -0,0 +1,383 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import test from 'node:test'; + +import { startFileRelay } from './relay.js'; + +import type { AddressInfo } from 'node:net'; + +async function listen( + server: ReturnType, +): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +test('file relay streams an authorized object download from its fixed upstream', async () => { + const upstream = createServer((req, res) => { + assert.equal(req.method, 'GET'); + assert.equal(req.url, '/sessions/storage-1/objects/file-1'); + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant-1'); + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'Content-Disposition': "attachment; filename*=UTF-8''canonical.txt", + 'X-Read-Only': 'true', + }); + res.end('input-data'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/storage-1/objects/file-1`, + { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }, + ); + + assert.equal(response.status, 200); + assert.equal(response.headers.get('x-read-only'), 'true'); + assert.equal( + response.headers.get('content-disposition'), + "attachment; filename*=UTF-8''canonical.txt", + ); + assert.equal(await response.text(), 'input-data'); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay forwards an authorized generated-file upload', async () => { + const upstream = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + assert.equal(req.method, 'PUT'); + assert.equal(req.url, '/sessions/output-1/objects/generated-1'); + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant-2'); + assert.equal(req.headers['x-original-filename'], 'report.txt'); + assert.equal(Buffer.concat(chunks).toString('utf8'), 'generated-data'); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end('{"stored":true}'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/output-1/objects/generated-1`, + { + method: 'PUT', + headers: { + 'Content-Type': 'text/plain', + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-2', + 'X-Original-Filename': 'report.txt', + }, + body: 'generated-data', + }, + ); + + assert.equal(response.status, 200); + assert.equal(await response.text(), '{"stored":true}'); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay permits only the normalized session object listing', async () => { + const upstream = createServer((req, res) => { + assert.equal(req.method, 'GET'); + assert.equal(req.url, '/sessions/storage-1/objects?detail=normalized'); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end('[]'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/storage-1/objects?detail=normalized`, + { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }, + ); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), []); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay health is available only with its private relay token', async () => { + const upstream = createServer((_req, res) => res.writeHead(500).end()); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const unauthorized = await fetch(`${relay.url}/health`); + assert.equal(unauthorized.status, 401); + const healthy = await fetch(`${relay.url}/health`, { + headers: { 'X-LibreChat-Code-Relay-Token': 'relay-secret' }, + }); + assert.equal(healthy.status, 200); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay rejects object traffic without an execution grant before contacting upstream', async () => { + let upstreamRequests = 0; + const upstream = createServer((_req, res) => { + upstreamRequests += 1; + res.writeHead(200).end('should-not-run'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/storage-1/objects/file-1`, + { headers: { 'X-LibreChat-Code-Relay-Token': 'relay-secret' } }, + ); + assert.equal(response.status, 403); + assert.equal(upstreamRequests, 0); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay never follows upstream redirects', async () => { + let upstreamRequests = 0; + const upstream = createServer((req, res) => { + upstreamRequests += 1; + if (req.url === '/admin') { + res.writeHead(200).end('sensitive'); + return; + } + res.writeHead(302, { Location: '/admin' }).end(); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/storage-1/objects/file-1`, + { + redirect: 'manual', + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }, + ); + assert.equal(response.status, 302); + assert.equal(upstreamRequests, 1); + assert.equal(response.headers.get('location'), null); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay bounds concurrent upstream transfers', async () => { + let releaseFirst: (() => void) | undefined; + const firstStarted = new Promise((resolve) => { + releaseFirst = resolve; + }); + let seen = 0; + const upstream = createServer(async (_req, res) => { + seen += 1; + if (seen === 1) await firstStarted; + res.writeHead(200).end('ok'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + maxConcurrentRequests: 1, + }); + const headers = { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }; + + try { + const first = fetch(`${relay.url}/sessions/storage-1/objects/file-1`, { + headers, + }); + while (seen === 0) await new Promise((resolve) => setTimeout(resolve, 1)); + const second = await fetch( + `${relay.url}/sessions/storage-1/objects/file-2`, + { headers }, + ); + assert.equal(second.status, 503); + releaseFirst?.(); + assert.equal((await first).status, 200); + assert.equal(seen, 1); + } finally { + releaseFirst?.(); + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay rejects an oversized chunked upstream response', async () => { + const upstream = createServer((_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/octet-stream' }); + res.write(Buffer.alloc(768)); + res.end(Buffer.alloc(768)); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/storage-1/objects/file-1`, + { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }, + ); + assert.equal(response.status, 502); + assert.equal(await response.text(), ''); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay accepts a valid scoped grant larger than Node defaults', async () => { + const grant = `grant-${'a'.repeat(32 * 1024)}`; + const upstream = createServer({ maxHeaderSize: 512 * 1024 }, (req, res) => { + assert.equal(req.headers['x-codeapi-egress-grant'], grant); + res.writeHead(200).end('ok'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl, + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }); + + try { + const response = await fetch( + `${relay.url}/sessions/storage-1/objects/file-1`, + { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': grant, + }, + }, + ); + assert.equal(response.status, 200); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test('file relay rejects plaintext remote upstreams', async () => { + await assert.rejects( + startFileRelay({ + host: '127.0.0.1', + port: 0, + upstreamUrl: 'http://code.example/egress', + token: 'relay-secret', + maxBytes: 1024, + timeoutMs: 1_000, + }), + /HTTPS unless it is a local development host/, + ); +}); diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts new file mode 100644 index 00000000..255845e9 --- /dev/null +++ b/packages/code/src/relay.ts @@ -0,0 +1,276 @@ +import { createHash, timingSafeEqual } from 'node:crypto'; +import { createServer } from 'node:http'; + +import type { IncomingMessage } from 'node:http'; +import type { AddressInfo } from 'node:net'; + +export interface FileRelayOptions { + host: string; + port: number; + upstreamUrl: string; + token: string; + maxBytes: number; + timeoutMs: number; + maxConcurrentRequests?: number; +} + +export interface FileRelayHandle { + url: string; + close(): Promise; +} + +const OBJECT_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+$/; +const OBJECT_LIST_PATH = /^\/sessions\/[^/]+\/objects$/; +const MAX_RELAY_HEADER_BYTES = 512 * 1024; +const LOCAL_HTTP_HOSTS = new Set([ + '127.0.0.1', + '[::1]', + 'localhost', + 'host.docker.internal', + 'gateway.docker.internal', +]); + +class RelayPayloadTooLargeError extends Error {} +class UpstreamPayloadTooLargeError extends Error {} + +export function validateFileRelayUpstream(value: string): URL { + const upstream = new URL(value); + if ( + (upstream.protocol !== 'http:' && upstream.protocol !== 'https:') || + upstream.username || + upstream.password || + upstream.search || + upstream.hash + ) { + throw new Error( + 'File relay upstream must be an HTTP URL without credentials, query, or fragment', + ); + } + if ( + upstream.protocol !== 'https:' && + !LOCAL_HTTP_HOSTS.has(upstream.hostname.toLowerCase()) + ) { + throw new Error( + 'File relay upstream must use HTTPS unless it is a local development host', + ); + } + return upstream; +} + +function positiveInteger(name: string, value: number): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + +function tokenMatches( + expected: string, + supplied: string | string[] | undefined, +): boolean { + if (typeof supplied !== 'string') return false; + return timingSafeEqual( + createHash('sha256').update(expected).digest(), + createHash('sha256').update(supplied).digest(), + ); +} + +async function readRequestBody( + request: IncomingMessage, + maxBytes: number, +): Promise { + const declaredLength = request.headers['content-length']; + if ( + typeof declaredLength === 'string' && + (!/^\d+$/.test(declaredLength) || Number(declaredLength) > maxBytes) + ) { + throw new RelayPayloadTooLargeError(); + } + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.from(chunk); + bytes += buffer.length; + if (bytes > maxBytes) throw new RelayPayloadTooLargeError(); + chunks.push(buffer); + } + return Buffer.concat(chunks); +} + +async function readResponseBody( + response: Response, + maxBytes: number, +): Promise { + const declaredLength = response.headers.get('content-length'); + if ( + declaredLength != null && + (!/^\d+$/.test(declaredLength) || Number(declaredLength) > maxBytes) + ) { + await response.body?.cancel(); + throw new UpstreamPayloadTooLargeError(); + } + if (response.body == null) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let bytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > maxBytes) throw new UpstreamPayloadTooLargeError(); + chunks.push(Buffer.from(value)); + } + } catch (error) { + await reader.cancel().catch(() => undefined); + throw error; + } + return Buffer.concat(chunks); +} + +export async function startFileRelay( + options: FileRelayOptions, +): Promise { + const upstream = validateFileRelayUpstream(options.upstreamUrl); + if (!options.token.trim()) throw new Error('File relay token is required'); + positiveInteger('File relay maxBytes', options.maxBytes); + positiveInteger('File relay timeoutMs', options.timeoutMs); + const maxConcurrentRequests = positiveInteger( + 'File relay maxConcurrentRequests', + options.maxConcurrentRequests ?? 8, + ); + let activeRequests = 0; + const server = createServer( + { maxHeaderSize: MAX_RELAY_HEADER_BYTES }, + async (request, response) => { + let admitted = false; + try { + if ( + !tokenMatches( + options.token, + request.headers['x-librechat-code-relay-token'], + ) + ) { + response.writeHead(401).end(); + return; + } + const requestUrl = new URL(request.url ?? '/', 'http://relay.invalid'); + if (request.method === 'GET' && requestUrl.pathname === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end('{"status":"ok"}'); + return; + } + const objectRequest = + OBJECT_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; + const normalizedListRequest = + request.method === 'GET' && + OBJECT_LIST_PATH.test(requestUrl.pathname) && + requestUrl.searchParams.size === 1 && + requestUrl.searchParams.get('detail') === 'normalized'; + if ( + (request.method !== 'GET' && request.method !== 'PUT') || + (!objectRequest && !normalizedListRequest) + ) { + response.writeHead(404).end(); + return; + } + const grant = request.headers['x-codeapi-egress-grant']; + if (typeof grant !== 'string' || grant.length === 0) { + response.writeHead(403).end(); + return; + } + if (activeRequests >= maxConcurrentRequests) { + response.writeHead(503, { 'Retry-After': '1' }).end(); + return; + } + activeRequests += 1; + admitted = true; + const target = new URL(upstream); + target.pathname = `${upstream.pathname.replace(/\/$/, '')}${ + requestUrl.pathname + }`; + target.search = requestUrl.search; + const requestBody = + request.method === 'PUT' + ? await readRequestBody(request, options.maxBytes) + : undefined; + const upstreamResponse = await fetch(target, { + method: request.method, + headers: { + ...(typeof grant === 'string' + ? { 'X-CodeAPI-Egress-Grant': grant } + : {}), + ...(request.method === 'PUT' + ? { + 'Content-Length': String(requestBody?.length ?? 0), + ...(typeof request.headers['content-type'] === 'string' + ? { + 'Content-Type': request.headers['content-type'], + } + : {}), + ...(typeof request.headers['x-original-filename'] === 'string' + ? { + 'X-Original-Filename': + request.headers['x-original-filename'], + } + : {}), + } + : {}), + }, + body: requestBody ? new Uint8Array(requestBody) : undefined, + redirect: 'manual', + signal: AbortSignal.timeout(options.timeoutMs), + }); + const body = await readResponseBody(upstreamResponse, options.maxBytes); + response.writeHead(upstreamResponse.status, { + ...(upstreamResponse.headers.get('content-type') + ? { + 'Content-Type': upstreamResponse.headers.get('content-type')!, + } + : {}), + ...(upstreamResponse.headers.get('x-read-only') + ? { + 'X-Read-Only': upstreamResponse.headers.get('x-read-only')!, + } + : {}), + ...(upstreamResponse.headers.get('content-disposition') + ? { + 'Content-Disposition': upstreamResponse.headers.get( + 'content-disposition', + )!, + } + : {}), + 'Content-Length': String(body.length), + }); + response.end(body); + } catch (error) { + if (error instanceof RelayPayloadTooLargeError) { + if (!response.headersSent) response.writeHead(413); + response.end(); + return; + } + if (!response.headersSent) response.writeHead(502); + response.end(); + } finally { + if (admitted) activeRequests -= 1; + } + }, + ); + server.requestTimeout = options.timeoutMs; + server.headersTimeout = options.timeoutMs; + server.keepAliveTimeout = Math.min(options.timeoutMs, 5_000); + server.maxRequestsPerSocket = 100; + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(options.port, options.host, resolve); + }); + const address = server.address() as AddressInfo; + return { + url: `http://${options.host}:${address.port}`, + close: async () => { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + }, + }; +} diff --git a/packages/code/src/runtime.test.ts b/packages/code/src/runtime.test.ts index 113f93bb..9ee67fae 100644 --- a/packages/code/src/runtime.test.ts +++ b/packages/code/src/runtime.test.ts @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import test from 'node:test'; import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; @@ -109,6 +110,42 @@ test('docker runtime supervisor creates a networkless stateful runtime and execu assert.ok(health?.includes('--max-time')); }); +test('docker runtime supervisor preserves the legacy profile digest for the default network', async () => { + const image = 'example/code-runtime:latest'; + const legacyDigest = createHash('sha256') + .update( + JSON.stringify({ + version: 1, + image, + profileRevision: null, + restartStoppedContainers: true, + capabilities: [], + securityOptions: [], + environment: [], + bindMounts: [], + }), + ) + .digest('hex'); + const calls: string[][] = []; + const client: ContainerRuntimeClient = { + async run(args) { + calls.push(args); + if (args[0] === 'container' && args[1] === 'inspect') { + return `true|${legacyDigest}|sha256:image-1\n`; + } + if (args[0] === 'image' && args[1] === 'inspect') return 'sha256:image-1\n'; + if (args[0] === 'exec') return '200'; + throw new Error(`Unexpected Docker command: ${args.join(' ')}`); + }, + }; + const supervisor = new DockerRuntimeSupervisor({ image, client }); + + await supervisor.acquire(assignment('existing-workspace')); + + assert.equal(calls.some((args) => args[0] === 'container' && args[1] === 'rm'), false); + assert.equal(calls.some((args) => args[0] === 'run'), false); +}); + test('docker runtime supervisor applies an explicit macOS NsJail confinement profile', async () => { const calls: string[][] = []; const client: ContainerRuntimeClient = { @@ -123,6 +160,7 @@ test('docker runtime supervisor applies an explicit macOS NsJail confinement pro const supervisor = new DockerRuntimeSupervisor({ image: 'example/code-runtime:latest', client, + network: 'librechat-code-worker', capabilities: ['SYS_ADMIN', 'CHOWN'], securityOptions: ['seccomp=/repo/seccomp/nsjail.json'], environment: { SANDBOX_USE_CGROUPV2: 'false' }, @@ -134,6 +172,7 @@ test('docker runtime supervisor applies an explicit macOS NsJail confinement pro const run = calls.find(args => args[0] === 'run') ?? []; assert.ok(run.includes('SYS_ADMIN')); + assert.equal(run[run.indexOf('--network') + 1], 'librechat-code-worker'); assert.ok(run.includes('CHOWN')); assert.ok(run.includes('seccomp=/repo/seccomp/nsjail.json')); assert.ok(run.includes('SANDBOX_USE_CGROUPV2=false')); diff --git a/packages/code/src/runtime.ts b/packages/code/src/runtime.ts index bde50322..cf6eea48 100644 --- a/packages/code/src/runtime.ts +++ b/packages/code/src/runtime.ts @@ -47,6 +47,7 @@ export interface DockerRuntimeSupervisorOptions { image?: string; profileRevision?: string; restartStoppedContainers?: boolean; + network?: string; capabilities?: string[]; securityOptions?: string[]; environment?: Record; @@ -76,6 +77,7 @@ const DEFAULT_STARTUP_TIMEOUT_MS = 30_000; const DEFAULT_HEALTH_PATH = '/api/v2/health'; const CONTAINER_PREFIX = 'librechat-code-'; const CAPABILITY_PATTERN = /^[A-Z_]{1,32}$/; +const NETWORK_PATTERN = /^(?:none|[A-Za-z0-9][A-Za-z0-9_.-]{0,127})$/; const MAX_DOCKER_COMMAND_OUTPUT_BYTES = 64 * 1024 * 1024; function normalizedEndpoint(value: string): string { @@ -106,7 +108,7 @@ function isMissingImageError(error: unknown): boolean { return /(?:no such image|no such object)/i.test(error.message); } -class DockerCliClient implements ContainerRuntimeClient { +export class DockerCliClient implements ContainerRuntimeClient { private readonly command: string; constructor(command = 'docker') { @@ -165,6 +167,7 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { private readonly bindMounts: DockerRuntimeBindMount[]; private readonly httpClient: 'curl' | 'bun'; private readonly restartStoppedContainers: boolean; + private readonly network: string; constructor(private readonly options: DockerRuntimeSupervisorOptions) { if (options.image != null && options.image.trim().length === 0) { @@ -173,6 +176,9 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { if (options.capabilities?.some((capability) => !CAPABILITY_PATTERN.test(capability))) { throw new Error('Docker runtime capabilities must be uppercase capability names'); } + if (options.network != null && !NETWORK_PATTERN.test(options.network)) { + throw new Error('Docker runtime network name is invalid'); + } this.client = options.client ?? new DockerCliClient(options.dockerCommand); this.runnerPort = options.runnerPort ?? DEFAULT_RUNNER_PORT; this.startupTimeoutMs = options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; @@ -183,6 +189,7 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { this.bindMounts = (options.bindMounts ?? []).map((mount) => ({ ...mount })); this.httpClient = options.httpClient ?? 'curl'; this.restartStoppedContainers = options.restartStoppedContainers ?? true; + this.network = options.network ?? 'none'; if ( this.bindMounts.some( ({ source, target }) => @@ -275,7 +282,7 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { '--name', name, '--network', - 'none', + this.network, '--cap-drop', 'ALL', ...this.capabilities.flatMap((capability) => ['--cap-add', capability]), @@ -310,6 +317,7 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { image, profileRevision: this.options.profileRevision ?? null, restartStoppedContainers: this.restartStoppedContainers, + ...(this.network !== 'none' ? { network: this.network } : {}), capabilities: this.capabilities, securityOptions: this.securityOptions, environment: Object.entries(this.environment).sort(([left], [right]) => diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index f66074a3..68d95b28 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -12,6 +12,110 @@ import type { RuntimeSupervisor } from './runtime.js'; const incarnationId = 'incarnation-00000001'; +test('worker invokes lifecycle hooks only after its incarnation registers', async () => { + const registered: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async () => + Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + onRegistered: async (registration) => { + registered.push(registration.incarnationId); + }, + }); + + await worker.register(); + + assert.deepEqual(registered, [incarnationId]); +}); + +test('worker confirms readiness only after local registration activation succeeds', async () => { + const events: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/ready')) { + events.push('ready'); + return Response.json({ protocolVersion: 1, ready: true }); + } + events.push('registered'); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registrationGeneration: 3, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }, + onRegistered: async () => { + events.push('activated'); + }, + }); + + await worker.register(); + + assert.deepEqual(events, ['registered', 'activated', 'ready']); +}); + +test('worker does not confirm readiness when local activation fails', async () => { + let readinessRequests = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/ready')) readinessRequests += 1; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registrationGeneration: 1, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + }, + onRegistered: async () => { + throw new Error('relay activation failed'); + }, + }); + + await assert.rejects(worker.register(), /relay activation failed/); + assert.equal(readinessRequests, 0); +}); + test('worker forwards a fenced assignment to the sandbox and settles the result', async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; const fetchImpl: typeof fetch = async (input, init) => { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index f4abfeba..81a2d91e 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -44,6 +44,9 @@ export interface BridgeWorkerOptions { fetchImpl?: typeof fetch; onError?: (error: unknown) => void; onIdentityChange?: (identity: BridgeWorkerIdentity) => void | Promise; + onRegistered?: ( + registration: BridgeWorkerRegistrationResponse, + ) => void | Promise; incarnationId?: string; } @@ -202,10 +205,43 @@ export class BridgeWorker { this.serverClockOffsetMs = registeredAtMs - registrationStartedAtMs; } this.registrationTtlMs = registration.leaseTtlMs; + await this.options.onRegistered?.(registration); + if (this.options.capabilities.requiresReadyConfirmation === true) { + await this.confirmReady(registration, signal); + } this.lastRegisteredAtMs = registrationStartedAtMs; return registration; } + private async confirmReady( + registration: BridgeWorkerRegistrationResponse, + signal?: AbortSignal, + ): Promise { + const registrationGeneration = registration.registrationGeneration; + if ( + !Number.isSafeInteger(registrationGeneration) || + (registrationGeneration ?? 0) < 1 + ) { + throw new BridgeProtocolError( + 'Code API does not support explicit worker readiness confirmation', + ); + } + await this.timedRequest( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/ready`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId: this.incarnationId, + registrationGeneration, + }, + Math.max( + 1, + this.options.registrationTransportTimeoutMs ?? + DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS, + ), + signal, + ); + } + async resetWorkspace( runtimeSessionId: string, signal?: AbortSignal, diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 74c75b7c..9eb6d76c 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -82,7 +82,7 @@ if credential then redis.call('DEL', KEYS[3]) redis.call('DEL', ARGV[1] .. credential) end -redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6]) +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7]) if activeIncarnation then redis.call('SET', ARGV[2] .. activeIncarnation .. ':fenced', '1') end @@ -446,13 +446,14 @@ export class RedisBridgePairingStore { // that linearizes afterward installs a distinct generation and code. await this.redis.eval( REVOKE_PAIRING_SCRIPT, - 6, + 7, workerPairingIndexKey(workerId), workerPairingGenerationKey(workerId), workerIdentityKey(workerId), workerStableIdentityKey(workerId), `${PREFIX}:worker:${encodeURIComponent(workerId)}`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, + `${PREFIX}:worker:${encodeURIComponent(workerId)}:ready`, `${PREFIX}:credential:`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, ); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 2b3baa10..7e6e0fc5 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -278,6 +278,7 @@ describe('paired bridge HTTP API', () => { await expect(registrationResponse.json()).resolves.toMatchObject({ workerId: 'vm-1', incarnationId: 'incarnation-00000001', + registrationGeneration: 1, }); const crossDeploymentRevoke = await fetch( diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index e41336d4..996dd16b 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -362,10 +362,59 @@ router.post( : {}), }; try { - await options.store.register( + const registrationGeneration = await options.store.register( trustedRegistration, authorization, ); + res.json({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: registration.workerId, + incarnationId: registration.incarnationId, + registrationGeneration, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + } catch (error) { + if (error instanceof BridgeStoreError) { + sendStoreError(error, res); + return; + } + throw error; + } + }), +); + +router.post( + '/workers/:workerId/ready', + workerAuth, + asyncRoute(async (req, res) => { + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !validIncarnationId(body.incarnationId) || + !Number.isSafeInteger(body.registrationGeneration) || + Number(body.registrationGeneration) < 1 + ) { + res.status(400).json({ + error: 'Invalid bridge worker readiness confirmation', + }); + return; + } + if (!configuredWorker(workerId)) { + res.status(403).json({ + error: 'Worker is not authorized for this Code API deployment', + }); + return; + } + try { + await options.store.confirmReady( + workerId, + body.incarnationId, + Number(body.registrationGeneration), + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ready: true }); } catch (error) { if (error instanceof BridgeStoreError) { sendStoreError(error, res); @@ -373,13 +422,6 @@ router.post( } throw error; } - res.json({ - protocolVersion: BRIDGE_PROTOCOL_VERSION, - workerId: registration.workerId, - incarnationId: registration.incarnationId, - registeredAt: new Date().toISOString(), - leaseTtlMs: 60_000, - }); }), ); diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts index 23b04067..e08273f9 100644 --- a/service/src/bridge/store.test.ts +++ b/service/src/bridge/store.test.ts @@ -6,6 +6,8 @@ import type * as t from '../types'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; import { RedisBridgeStore } from './store'; +import type { RegisteredBridgeWorker } from './store'; + const redis = new RedisMock() as unknown as Redis; const store = new RedisBridgeStore(redis); const incarnationId = 'incarnation-00000001'; @@ -13,12 +15,14 @@ const redisEval = redis.eval.bind(redis); const redisDel = redis.del.bind(redis); const redisLpop = redis.lpop.bind(redis); const redisGet = redis.get.bind(redis); +const redisMget = redis.mget.bind(redis); afterEach(async () => { redis.eval = redisEval as Redis['eval']; redis.del = redisDel as Redis['del']; redis.lpop = redisLpop as Redis['lpop']; redis.get = redisGet as Redis['get']; + redis.mget = redisMget as Redis['mget']; await redis.flushall(); }); @@ -72,7 +76,149 @@ describe('RedisBridgeStore', () => { runtimes: ['bash'], }, }, 'old-authenticated-credential-digest'), - ).resolves.toBeUndefined(); + ).resolves.toBe(1); + }); + + test('allocates registration generations only when the active incarnation changes', async () => { + const registration: RegisteredBridgeWorker = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'generation-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + }; + + await expect(store.register(registration)).resolves.toBe(1); + await expect(store.register(registration)).resolves.toBe(1); + await expect( + store.register({ + ...registration, + incarnationId: 'incarnation-00000002', + }), + ).resolves.toBe(2); + }); + + test('dispatches an explicitly gated worker only after exact-generation readiness', async () => { + const workerId = 'ready-worker'; + const registration: RegisteredBridgeWorker = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }, + }; + const generation = await store.register(registration); + + await expect( + store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_OFFLINE' }); + + await store.confirmReady(workerId, incarnationId, generation); + const controller = new AbortController(); + const completion = store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }); + await expect(store.lease(workerId, incarnationId, 1_000)).resolves.toBeDefined(); + controller.abort(); + await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + + await store.register(registration); + const secondController = new AbortController(); + const secondCompletion = store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 5_000, + signal: secondController.signal, + }); + await expect(store.lease(workerId, incarnationId, 1_000)).resolves.toBeDefined(); + secondController.abort(); + await expect(secondCompletion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + }); + + test('rejects readiness from a replaced registration generation', async () => { + const workerId = 'replaced-ready-worker'; + const capabilities = { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }; + const staleGeneration = await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities, + }); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId: 'incarnation-00000002', + capabilities, + }); + + await expect( + store.confirmReady(workerId, incarnationId, staleGeneration), + ).rejects.toMatchObject({ code: 'WORKER_FENCED' }); + }); + + test('does not enqueue after readiness is withdrawn during dispatch', async () => { + const workerId = 'readiness-race-worker'; + const generation = await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + let withdrewReadiness = false; + redis.eval = (async (...args: Parameters) => { + if (!withdrewReadiness && String(args[0]).includes('ARGV[7]')) { + withdrewReadiness = true; + await redis.del(`codeapi:bridge:v1:worker:${workerId}:ready`); + } + return redisEval(...args); + }) as Redis['eval']; + + await expect( + store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_OFFLINE' }); + expect(withdrewReadiness).toBe(true); + await expect( + redis.llen( + `codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`, + ), + ).resolves.toBe(0); }); test('rejects a dynamic worker lease outside its bound tenant', async () => { @@ -1174,7 +1320,7 @@ describe('RedisBridgeStore', () => { runtimes: [], }, }), - ).resolves.toBeUndefined(); + ).resolves.toBe(2); }); test('recovers only the assignment owner after registration expiry', async () => { @@ -1227,7 +1373,7 @@ describe('RedisBridgeStore', () => { runtimes: [], }, }), - ).resolves.toBeUndefined(); + ).resolves.toBe(1); controller.abort(); await expect(completion).rejects.toMatchObject({ @@ -1304,7 +1450,7 @@ describe('RedisBridgeStore', () => { runtimes: [], }, }); - redis.get = (() => new Promise(() => {})) as Redis['get']; + redis.mget = (() => new Promise<(string | null)[]>(() => {})) as Redis['mget']; await expect( timedStore.dispatch({ @@ -2072,6 +2218,6 @@ describe('RedisBridgeStore', () => { runtimes: [], }, }), - ).resolves.toBeUndefined(); + ).resolves.toBe(1); }); }); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 3a43298b..afc8a72b 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -68,6 +68,25 @@ function workerIncarnationKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`; } +function workerRegistrationGenerationKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:registration-generation`; +} + +function workerRegistrationGenerationIncarnationKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:registration-generation-incarnation`; +} + +function workerReadyKey(workerId: string): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:ready`; +} + +function workerReadyToken( + incarnationId: string, + registrationGeneration: number, +): string { + return `${incarnationId}:${registrationGeneration}`; +} + function incarnationFenceKey(workerId: string, incarnationId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:fenced`; } @@ -240,7 +259,7 @@ export class RedisBridgeStore { pairingGeneration?: number; activeCredentialId?: string; }, - ): Promise { + ): Promise { const authorizationObject = typeof authorization === 'object' ? authorization : undefined; const expectedActiveCredentialId = @@ -278,15 +297,24 @@ export class RedisBridgeStore { ' redis.call(\'SET\', ARGV[4] .. current .. \':fenced\', \"1\")', ' end', 'end', + 'local registrationGeneration = tonumber(redis.call(\'GET\', KEYS[10]) or \"0\")', + 'local registrationGenerationIncarnation = redis.call(\'GET\', KEYS[11])', + 'local registrationGenerationChanged = false', + 'if registrationGeneration < 1 or registrationGenerationIncarnation ~= ARGV[1] then', + ' registrationGeneration = redis.call(\'INCR\', KEYS[10])', + ' redis.call(\'SET\', KEYS[11], ARGV[1])', + ' registrationGenerationChanged = true', + 'end', 'redis.call(\'SET\', KEYS[1], ARGV[2], \"EX\", ARGV[3])', 'redis.call(\'SET\', KEYS[4], ARGV[1], \"EX\", ARGV[3])', - 'return 1', + 'if ARGV[9] == "1" and registrationGenerationChanged then redis.call(\'DEL\', KEYS[12]) end', + 'return registrationGeneration', ].join('\n'); const result = Number( await boundedCommand( this.redis.eval( script, - 9, + 12, workerKey(registration.workerId), incarnationFenceKey(registration.workerId, registration.incarnationId), quarantineKey(registration.workerId, registration.incarnationId), @@ -296,6 +324,9 @@ export class RedisBridgeStore { `${PREFIX}:pairing-generation:${registration.workerId}`, `${PREFIX}:stable-identity:${registration.workerId}`, `${PREFIX}:identity:${registration.workerId}`, + workerRegistrationGenerationKey(registration.workerId), + workerRegistrationGenerationIncarnationKey(registration.workerId), + workerReadyKey(registration.workerId), registration.incarnationId, JSON.stringify(registration), String(this.workerTtlSeconds), @@ -306,6 +337,7 @@ export class RedisBridgeStore { authorizationObject?.identityId ?? '', expectedActiveCredentialId ?? '', registration.identityId ?? '', + registration.capabilities.requiresReadyConfirmation === true ? '1' : '0', ), this.redisCommandTimeoutMs, 'Bridge worker registration', @@ -341,6 +373,73 @@ export class RedisBridgeStore { 'Bridge worker authorization was revoked before registration completed', ); } + if (!Number.isSafeInteger(result) || result < 1) { + throw new Error('Bridge worker registration returned an invalid generation'); + } + return result; + } + + async confirmReady( + workerId: string, + incarnationId: string, + registrationGeneration: number, + ): Promise { + const result = Number( + await boundedCommand( + this.redis.eval( + [ + 'if redis.call(\'EXISTS\', KEYS[1]) == 0 then return -1 end', + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return -2 end', + 'if redis.call(\'GET\', KEYS[3]) ~= ARGV[2] then return -2 end', + 'if redis.call(\'GET\', KEYS[4]) ~= ARGV[1] then return -2 end', + 'if redis.call(\'EXISTS\', KEYS[5]) == 1 then return -2 end', + 'if redis.call(\'EXISTS\', KEYS[6]) == 1 then return -3 end', + 'redis.call(\'SET\', KEYS[7], ARGV[3], "EX", ARGV[4])', + 'return 1', + ].join('\n'), + 7, + workerKey(workerId), + workerIncarnationKey(workerId), + workerRegistrationGenerationKey(workerId), + workerRegistrationGenerationIncarnationKey(workerId), + incarnationFenceKey(workerId, incarnationId), + quarantineKey(workerId, incarnationId), + workerReadyKey(workerId), + incarnationId, + String(registrationGeneration), + workerReadyToken(incarnationId, registrationGeneration), + String( + Math.min( + this.workerTtlSeconds, + Math.ceil(this.workerTtlSeconds / 2) + 5, + ), + ), + ), + this.redisCommandTimeoutMs, + 'Bridge worker readiness confirmation', + ), + ); + if (result === -1) { + throw new BridgeStoreError( + 'WORKER_OFFLINE', + 'Bridge worker registration expired before readiness confirmation', + ); + } + if (result === -2) { + throw new BridgeStoreError( + 'WORKER_FENCED', + 'Bridge worker readiness confirmation is stale', + ); + } + if (result === -3) { + throw new BridgeStoreError( + 'WORKER_QUARANTINED', + 'Bridge worker incarnation is quarantined', + ); + } + if (result !== 1) { + throw new Error('Bridge worker readiness confirmation failed'); + } } async dispatch(args: { @@ -357,17 +456,18 @@ export class RedisBridgeStore { ) => Promise; }): Promise { this.assertDispatchActive(args.signal, args.deadlineAtMs); - let registration = await this.dispatchCommand( - () => this.registration(args.workerId), + const dispatchable = await this.dispatchCommand( + () => this.dispatchableRegistration(args.workerId), args, 'Bridge worker registration read', ); - if (registration == null) { + if (dispatchable == null) { throw new BridgeStoreError( 'WORKER_OFFLINE', `Bridge worker ${args.workerId} is offline`, ); } + let { registration, readyToken } = dispatchable; if ( (args.requireTenantBinding === true && registration.binding == null) || (registration.binding != null && @@ -459,13 +559,18 @@ export class RedisBridgeStore { this.assertDispatchActive(args.signal, args.deadlineAtMs); assignment.incarnationId = registration.incarnationId; queued = await this.dispatchCommand( - () => this.enqueueForActiveIncarnation(assignment!, ttlSeconds), + () => + this.enqueueForActiveIncarnation( + assignment!, + ttlSeconds, + readyToken, + ), args, 'Bridge assignment enqueue', ); if (queued) break; const replacement = await this.dispatchCommand( - () => this.registration(args.workerId), + () => this.dispatchableRegistration(args.workerId), args, 'Bridge replacement registration read', ); @@ -477,14 +582,15 @@ export class RedisBridgeStore { } if ( args.runtimeSessionId !== undefined && - replacement.capabilities.statefulWorkspace !== true + replacement.registration.capabilities.statefulWorkspace !== true ) { throw new BridgeStoreError( 'WORKER_MISMATCH', `Bridge worker ${args.workerId} does not provide a stateful workspace`, ); } - registration = replacement; + registration = replacement.registration; + readyToken = replacement.readyToken; } if (!queued) { throw new BridgeStoreError( @@ -1103,6 +1209,35 @@ export class RedisBridgeStore { return raw == null ? undefined : (JSON.parse(raw) as RegisteredBridgeWorker); } + private async dispatchableRegistration( + workerId: string, + ): Promise< + | { registration: RegisteredBridgeWorker; readyToken?: string } + | undefined + > { + const [raw, ready, generation, generationIncarnation] = await this.redis.mget( + workerKey(workerId), + workerReadyKey(workerId), + workerRegistrationGenerationKey(workerId), + workerRegistrationGenerationIncarnationKey(workerId), + ); + if (raw == null) return undefined; + const registration = JSON.parse(raw) as RegisteredBridgeWorker; + if (registration.capabilities.requiresReadyConfirmation !== true) { + return { registration }; + } + const registrationGeneration = Number(generation); + if ( + !Number.isSafeInteger(registrationGeneration) || + registrationGeneration < 1 || + generationIncarnation !== registration.incarnationId || + ready !== workerReadyToken(registration.incarnationId, registrationGeneration) + ) { + return undefined; + } + return { registration, readyToken: ready }; + } + private assertDispatchActive( signal: AbortSignal, deadlineAtMs: number, @@ -1201,16 +1336,18 @@ export class RedisBridgeStore { private async enqueueForActiveIncarnation( assignment: StoredAssignment, ttlSeconds: number, + readyToken?: string, ): Promise { const script = [ 'if redis.call(\'GET\', KEYS[1]) ~= ARGV[1] then return 0 end', - 'if #KEYS == 6 and redis.call(\'EXISTS\', KEYS[6]) == 1 then return -1 end', + 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', + 'if #KEYS == 7 and redis.call(\'EXISTS\', KEYS[7]) == 1 then return -1 end', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', 'redis.call(\'RPUSH\', KEYS[3], ARGV[4])', 'redis.call(\'EXPIRE\', KEYS[3], ARGV[3])', 'redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])', 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', - 'if #KEYS == 6 then redis.call(\'SET\', KEYS[6], ARGV[4]) end', + 'if #KEYS == 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', 'return 1', ].join('\n'); const keys = [ @@ -1219,6 +1356,7 @@ export class RedisBridgeStore { queueKey(assignment.workerId, assignment.incarnationId), lockIncarnationKey(assignment.workerId), assignmentDeadlineKey(assignment.assignmentId), + workerReadyKey(assignment.workerId), ]; if (assignment.runtimeSessionId !== undefined) { keys.push( @@ -1238,6 +1376,7 @@ export class RedisBridgeStore { assignment.assignmentId, String(ttlSeconds * 1000), String(Date.parse(assignment.expiresAt)), + readyToken ?? '', ); if (Number(result) === -1) { throw new BridgeStoreError( From 0739f3d6397083c7d76a8779522f758020136d14 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 12:49:15 -0400 Subject: [PATCH 021/116] =?UTF-8?q?=F0=9F=97=91=EF=B8=8F=20fix:=20Make=20C?= =?UTF-8?q?ode=20Environment=20File=20Deletion=20Work=20(#85)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🗑️ fix: Make Code Environment File Deletion Work Object deletion has never removed anything, and the failure was silent at every layer. The client (LibreChat `deleteCodeEnvFile`) issues DELETE against `/v1/sessions/:session_id/objects/:fileId`, the file-server's own path, which is not exposed on `/v1` — only GET is mounted there. Every deletion 404'd, and a 404 is indistinguishable from "already gone", so the caller cleared its state and the bucket only ever grew (13 GiB / 29k objects on a six-week-old deployment, per danny-avila/LibreChat#15511). Mount DELETE on that path as an alias of `/v1/files/:session_id/:fileId`, so deployments running a client older than LibreChat v0.8.6 — before the fallback to `/files/...` landed — delete successfully. Pass the file-server's 404 through instead of collapsing it into a 500: a 500 reads as retryable, and a client sweeping its retention window re-issues the same DELETE hourly, forever, for an object that no longer exists. Correcting the route is not sufficient on its own. `sessionAuth` authorizes deletion against `session:`, whose `SESSION_CACHE_TTL` is 24h and is not refreshed by use, so an object was deletable only for the day following upload and stranded permanently after that — unreadable, unusable as an execution input, and undeletable through every route. Clients are typically far outside that window when they get there; LibreChat's default retention is 30 days. Record ownership twice: `session:` stays the hot-path cache bounding read access, and a durable `session-owner:` record (`SESSION_OWNER_TTL`, 90 days, never shorter than the cache TTL) backs deletion once the cache key has lapsed. The fallback applies to DELETE only — reads keep the window they have always had — and a live cache key naming a different owner remains authoritative, so a re-registered session is never deletable by its previous owner. The recovery script restores both records, so a rehydrated session stays deletable rather than stranding again a day later. * fix: Close Codex review findings on session ownership Four P2 findings from the review of daae56c: - The 404 deletion path cleared the upload key with a bare `await` inside the catch block. A Redis failure there rejects with no handler above it, and Express 4 does not forward async rejections, so the request would hang instead of answering 404. Make the cleanup best effort and log it. - The blocking PTC path discarded the registration promise with `void`, preserving the previous fire-and-forget behavior. That now spans two keys: a partial write (cache key stored, durable record refused by a Redis ACL scoped to `session:*`) would produce exactly the undeletable files this change exists to prevent. Await it; the caller turns a rejection into a 500 before anything is enqueued. - Recovery treated a durable owner record naming someone else as a log line while still counting the session as restored or matching, so an apply could exit 0 having recovered nothing usable. Reconcile the owner record before touching the cache key and report the disagreement as a conflict, in dry run as well as apply. The cache key is no longer restored for those sessions either — the manifest's claimant should not get a day of access the service never granted it. - `SET NX` cannot extend an expiry, so a matching owner record could carry less remaining TTL than the cache key being restored and lapse first, stranding the session again just as recovery reported success. Top up the expiry when it is shorter than the target, leaving longer ones alone. * fix: Settle session ownership before recovery writes anything Two findings from the review of 816dbb5: - Reconciling the durable owner record first meant creating it before the live cache key had been consulted. For a session whose durable record was absent and whose cache key named a different owner, recovery wrote a durable record for the manifest's claimant, then reported the cache conflict and moved on — leaving the record behind. It outlives the cache key by design, so once that expired, `sessionAuth` would authorize the manifest owner to delete the real owner's files. Split the read from the write. The durable record is now inspected read-only up front, where a disagreement still settles the session before anything is written, and is created or extended only once the cache key has been confirmed to name the same owner. - `/exec` registered ownership before entering the route's `try`. Express 4 does not forward a rejected async handler to the error middleware, so a Redis failure there would hang the request rather than answering. Guard it and return a controlled 500. * fix: Harden recovery's durable owner handling Three findings from the review of 47b36e0, all in the recovery script: - A `SET NX` that lost the race to a key which then expired before the follow-up read left no record and no conflict, and the session was reported as recovered while its durable half did not exist. Retry once, and report anything past that as missing so an apply exits nonzero instead of claiming success. - When the owner commit conflicted on a session whose cache key this run had just created, the cache key stayed. That grant authorizes reads and deletes for its full TTL while the durable record names somebody else, so roll it back. A durable record that merely could not be created is left alone: the session is no worse off than before the run, and removing the grant would leave the operator with nothing. - A dry run reported a session whose cache key already matched as `matching` even when its owner record was absent or short-lived, hiding the work an apply would do and inviting operators to skip it. Pending owner repairs now count as missing. * fix: Answer Redis failures instead of hanging on them Two findings from the review of d1de664: - `sessionAuth` awaited the ownership lookup unguarded. Express 4 does not forward a rejected async middleware, so an unavailable Redis — or an ACL granting `session:*` but not `session-owner:*` — would hang a DELETE rather than answering it. Catch and return a controlled 500. - The recovery script's TTL top-up read, extended and returned across three round trips, reporting success on evidence it had not rechecked. The record can lapse in between, in which case it is now created fresh, or name somebody else, in which case the session is a conflict. Extending a record that turns out to belong to another owner prolongs a claim the service itself wrote and grants nothing new, but reporting the session as recovered on that basis would not be true. --- service/scripts/rehydrate-session-cache.ts | 226 ++++++++++++- service/src/config.ts | 10 + service/src/middleware/auth.ts | 32 +- service/src/rehydrate-session-cache.test.ts | 337 +++++++++++++++++++- service/src/service/programmatic-router.ts | 12 +- service/src/service/router.ts | 62 +++- service/src/session-ownership.test.ts | 165 ++++++++++ service/src/session-ownership.ts | 126 ++++++++ 8 files changed, 950 insertions(+), 20 deletions(-) create mode 100644 service/src/session-ownership.test.ts create mode 100644 service/src/session-ownership.ts diff --git a/service/scripts/rehydrate-session-cache.ts b/service/scripts/rehydrate-session-cache.ts index c21afa92..819bb250 100644 --- a/service/scripts/rehydrate-session-cache.ts +++ b/service/scripts/rehydrate-session-cache.ts @@ -16,10 +16,18 @@ import { redisKeepAliveOptions } from '../src/redis-options'; * {"type":"source","environment":"example","region":"region-1","namespace":"codeapi","query_start_utc":"2026-01-01T00:00:00Z","query_end_utc":"2026-01-02T00:00:00Z"} * {"session_id":"<21-character id>","expected_session_key":""} * - * The apply path uses SET NX and never replaces an existing owner. + * The apply path uses SET NX and never replaces an existing owner. Both the + * `session:` cache key and the durable `session-owner:` record are + * restored, so recovered sessions stay deletable past SESSION_CACHE_TTL + * rather than stranding again a day later. A durable owner record naming a + * different owner is reported as a conflict and the session is skipped + * entirely, in both dry-run and apply. */ const DEFAULT_SESSION_CACHE_TTL_SECONDS = 86400; +const DEFAULT_SESSION_OWNER_TTL_SECONDS = 90 * 86400; +/** Redis `TTL` reply for a key that does not exist. */ +const KEY_ABSENT_TTL = -2; const MAX_RECOVERY_CONTEXT_LENGTH = 128; const MAX_RECOVERY_SESSION_KEY_LENGTH = 512; const MAX_RECONNECT_ATTEMPTS = 5; @@ -48,6 +56,9 @@ export interface RecoveryStore { ttlSeconds: number, condition: 'NX', ): Promise<'OK' | null>; + ttl(key: string): Promise; + expire(key: string, ttlSeconds: number): Promise; + del(key: string): Promise; } export interface RecoverySummary { @@ -78,6 +89,7 @@ interface Options extends RecoveryScope { apply: boolean; inputPath?: string; ttlSeconds: number; + ownerTtlSeconds: number; } function usage(): string { @@ -97,8 +109,15 @@ Options: SESSION_RECOVERY_REGION. --namespace Expected source namespace. Defaults to SESSION_RECOVERY_NAMESPACE. - --ttl-seconds Redis TTL for restored keys. Defaults to - SESSION_CACHE_TTL or ${DEFAULT_SESSION_CACHE_TTL_SECONDS}. + --ttl-seconds Redis TTL for restored session cache keys. + Defaults to SESSION_CACHE_TTL or + ${DEFAULT_SESSION_CACHE_TTL_SECONDS}. + --owner-ttl-seconds + Redis TTL for the durable session-owner records + restored alongside them. Defaults to + SESSION_OWNER_TTL or + ${DEFAULT_SESSION_OWNER_TTL_SECONDS}, and is never + shorter than --ttl-seconds. --help Show this help. Keep recovery manifests outside the repository because expected_session_key @@ -154,6 +173,10 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv = process.en let ttlSeconds = configuredTtl != null && configuredTtl !== '' ? parsePositiveInteger(configuredTtl, 'SESSION_CACHE_TTL') : DEFAULT_SESSION_CACHE_TTL_SECONDS; + const configuredOwnerTtl = env.SESSION_OWNER_TTL?.trim(); + let ownerTtlSeconds = configuredOwnerTtl != null && configuredOwnerTtl !== '' + ? parsePositiveInteger(configuredOwnerTtl, 'SESSION_OWNER_TTL') + : DEFAULT_SESSION_OWNER_TTL_SECONDS; let environment = env.SESSION_RECOVERY_ENVIRONMENT; let region = env.SESSION_RECOVERY_REGION; let namespace = env.SESSION_RECOVERY_NAMESPACE; @@ -189,6 +212,13 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv = process.en ); index += 1; break; + case '--owner-ttl-seconds': + ownerTtlSeconds = parsePositiveInteger( + optionValue(args, index, '--owner-ttl-seconds'), + '--owner-ttl-seconds', + ); + index += 1; + break; case '--help': break; default: @@ -203,6 +233,9 @@ export function parseOptions(args: string[], env: NodeJS.ProcessEnv = process.en region: parseRecoveryContext(region, 'Recovery region'), namespace: parseRecoveryContext(namespace, 'Recovery namespace'), ttlSeconds, + /* The durable record is what keeps a recovered session deletable + * beyond the cache TTL, so it can never be the shorter of the two. */ + ownerTtlSeconds: Math.max(ownerTtlSeconds, ttlSeconds), }; } @@ -320,11 +353,141 @@ export function parseRecoveryManifest( return { source, records: [...bySessionId.values()] }; } +type OwnerInspection = + | { status: 'agrees'; existing: string | null } + | { status: 'conflict' }; + +/** + * Reads the durable owner record without writing. A record naming a + * different owner is the strongest ownership signal available, so it + * settles the session before anything is restored. + */ +async function inspectOwnerRecord( + store: RecoveryStore, + record: RecoveryRecord, +): Promise { + const existing = await store.get(`session-owner:${record.session_id}`); + if (existing !== null && existing !== record.expected_session_key) { + return { status: 'conflict' }; + } + return { status: 'agrees', existing }; +} + +/** + * Extends the durable record when it would lapse before the cache key this + * run just restored. `SET NX` cannot do it: a record near the end of its + * life would otherwise strand the session again the moment recovery + * reported success. + * + * The read, the extension and the confirmation are three round trips, so + * the record is re-read afterwards rather than assumed: it can expire in + * between (the caller then creates it fresh) or name somebody else (a + * conflict). Extending a record that turns out to belong to another owner + * prolongs a claim the service itself wrote and grants nothing new, but + * reporting the session as recovered on that basis would be a lie. + */ +async function refreshOwnerTtl( + store: RecoveryStore, + ownerKey: string, + record: RecoveryRecord, + ownerTtlSeconds: number, +): Promise<'ready' | 'conflict' | 'vanished'> { + const remaining = await store.ttl(ownerKey); + if (remaining === KEY_ABSENT_TTL) { + return 'vanished'; + } + if (remaining >= 0 && remaining < ownerTtlSeconds && await store.expire(ownerKey, ownerTtlSeconds) === 0) { + return 'vanished'; + } + + const confirmed = await store.get(ownerKey); + if (confirmed === null) { + return 'vanished'; + } + return confirmed === record.expected_session_key ? 'ready' : 'conflict'; +} + +/** + * Creates or extends the durable owner record for a session whose cache + * key has just been confirmed to name the same owner. Runs only after that + * confirmation: writing it earlier would leave a record behind for a + * manifest owner the live cache key contradicts, and `sessionAuth` would + * later authorize deletion through it. + */ +async function ensureOwnerRecord( + store: RecoveryStore, + record: RecoveryRecord, + ownerTtlSeconds: number, + existing: string | null, +): Promise<'ready' | 'conflict' | 'unresolved'> { + const ownerKey = `session-owner:${record.session_id}`; + + if (existing !== null) { + const refreshed = await refreshOwnerTtl(store, ownerKey, record, ownerTtlSeconds); + if (refreshed !== 'vanished') { + return refreshed; + } + /* Expired between the inspection and the refresh. Fall through and + * create it as though it had never been there. */ + } + + /* `SET NX` can lose to a writer whose key then expires before the + * follow-up read, leaving no record and no conflict to report. One retry + * settles that; anything past it is reported rather than assumed. */ + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = await store.set( + ownerKey, + record.expected_session_key, + 'EX', + ownerTtlSeconds, + 'NX', + ); + if (result === 'OK') { + return 'ready'; + } + const raced = await store.get(ownerKey); + if (raced === record.expected_session_key) { + const refreshed = await refreshOwnerTtl(store, ownerKey, record, ownerTtlSeconds); + if (refreshed !== 'vanished') { + return refreshed; + } + continue; + } + if (raced !== null) { + return 'conflict'; + } + } + return 'unresolved'; +} + +/** + * Whether an apply would still have durable-record work to do. Keeps the + * dry run honest: a session whose cache key already matches can still need + * its owner record created or extended, and reporting it as fully in place + * invites operators to skip the apply. + */ +async function ownerRepairPending( + store: RecoveryStore, + record: RecoveryRecord, + ownerTtlSeconds: number, + existing: string | null, +): Promise { + if (existing === null) { + return true; + } + const remaining = await store.ttl(`session-owner:${record.session_id}`); + return remaining >= 0 && remaining < ownerTtlSeconds; +} + export async function recoverSessionCache( records: RecoveryRecord[], store: RecoveryStore, - options: Pick, + options: Pick & { ownerTtlSeconds?: number }, ): Promise { + const ownerTtlSeconds = Math.max( + options.ownerTtlSeconds ?? DEFAULT_SESSION_OWNER_TTL_SECONDS, + options.ttlSeconds, + ); const summary: RecoverySummary = { input: records.length, missing: 0, @@ -335,10 +498,59 @@ export async function recoverSessionCache( for (const record of records) { try { + /* Inspected first, and read-only: a durable owner that disagrees + * with the manifest settles the session before anything is written, + * including the cache key — restoring that would hand the manifest's + * claimant a day of access the service never granted it. */ + const inspection = await inspectOwnerRecord(store, record); + if (inspection.status === 'conflict') { + summary.conflicts += 1; + // eslint-disable-next-line no-console + console.error(`Conflict: ${record.session_id} has a durable owner record for a different owner`); + continue; + } + const redisKey = `session:${record.session_id}`; + + /* Deferred until the cache key agrees. The durable record outlives + * the cache key by design, so creating one for an owner the live + * cache key contradicts would outlast the evidence against it. + * Returns the bucket the record lands in when the durable half did + * not settle, or null to keep the cache-key bucket. */ + const settleOwnerRecord = async ( + restoredCacheKey: boolean, + ): Promise<'conflicts' | 'missing' | null> => { + if (!options.apply) { + const pending = await ownerRepairPending(store, record, ownerTtlSeconds, inspection.existing); + return pending ? 'missing' : null; + } + + const outcome = await ensureOwnerRecord(store, record, ownerTtlSeconds, inspection.existing); + if (outcome === 'ready') { + return null; + } + if (outcome === 'conflict') { + if (restoredCacheKey) { + /* Undo the grant just made: left in place it would authorize + * the manifest owner to read and delete for `ttlSeconds` while + * the durable record names somebody else. */ + await store.del(redisKey); + } + // eslint-disable-next-line no-console + console.error(`Conflict: ${record.session_id} durable owner record changed during recovery`); + return 'conflicts'; + } + /* The cache key stays: a session without a durable record is no + * worse off than before this run, and removing the grant would + * leave the operator with nothing at all. */ + // eslint-disable-next-line no-console + console.error(`Missing: ${record.session_id} durable owner record could not be created`); + return 'missing'; + }; + const current = await store.get(redisKey); if (current === record.expected_session_key) { - summary.matching += 1; + summary[await settleOwnerRecord(false) ?? 'matching'] += 1; continue; } if (current !== null) { @@ -361,13 +573,13 @@ export async function recoverSessionCache( 'NX', ); if (result === 'OK') { - summary.restored += 1; + summary[await settleOwnerRecord(true) ?? 'restored'] += 1; continue; } const racedValue = await store.get(redisKey); if (racedValue === record.expected_session_key) { - summary.matching += 1; + summary[await settleOwnerRecord(false) ?? 'matching'] += 1; } else if (racedValue === null) { summary.missing += 1; // eslint-disable-next-line no-console diff --git a/service/src/config.ts b/service/src/config.ts index f0be8ba4..8a8208f9 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -336,6 +336,16 @@ export const env = { FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute // Redis Key Cache Config SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, + /** TTL for the durable `session-owner:` record that backs + * deletion after `SESSION_CACHE_TTL` has lapsed (see + * `session-ownership.ts`). Sized to outlive a client's retention + * window — LibreChat sweeps expired files at 30 days by default, and a + * shorter value here reinstates the leak it exists to close. Clamped + * so it can never be tighter than the cache TTL. */ + SESSION_OWNER_TTL: Math.max( + Number(process.env.SESSION_OWNER_TTL) || 90 * 86400, + Number(process.env.SESSION_CACHE_TTL) || 86400, + ), /** Strict tenant isolation. When true, sessionKey resolution fails closed * (500) on requests whose auth context lacks `tenantId`, instead of * silently falling back to the `'legacy'` tenant prefix. Default OFF in diff --git a/service/src/middleware/auth.ts b/service/src/middleware/auth.ts index b402e346..f8e66835 100644 --- a/service/src/middleware/auth.ts +++ b/service/src/middleware/auth.ts @@ -4,6 +4,7 @@ import { connection } from '../queue'; import { isValidId } from '../utils'; import { env } from '../config'; import { resolveSessionKey, parseUploadSessionKeyInput, SessionKeyResolutionError } from '../session-key'; +import { authorizeSessionOwnership } from '../session-ownership'; import { LibreChatJwtAuthProvider, CodeApiJwtAuthError } from '../auth/librechat-jwt'; import { applyPrincipal, type CodeApiPrincipal } from '../auth/principal'; import { applyLocalPrincipal } from '../auth/local'; @@ -238,11 +239,36 @@ export const sessionAuth = async (req: AuthenticatedRequest, res: Response, next } throw err; } - const cachedSessionKey = await connection.get(`session:${session_id}`); - if (cachedSessionKey !== sessionKey) { - logger.error(`Unauthorized download: Cached session key: ${cachedSessionKey} | Expected session key: ${sessionKey} | Session ID: ${session_id} | File ID: ${fileId}`); + + /* A delete may fall back to the durable owner record once the session + * cache key has expired; reads keep the `SESSION_CACHE_TTL` window they + * have always had. Without the fallback an object is deletable only for + * the 24h following upload and is stranded in the bucket forever after + * that — see `session-ownership.ts`. */ + const isDelete = req.method === 'DELETE'; + let ownership: Awaited>; + try { + ownership = await authorizeSessionOwnership(connection, { + /* Narrowed by the `isValidId` guard above, which is not a type + * predicate. */ + session_id: session_id as string, + expectedSessionKey: sessionKey, + allowExpiredCache: isDelete, + }); + } catch (err) { + /* Express 4 does not forward a rejected async middleware, so an + * unavailable Redis — or an ACL that grants `session:*` but not + * `session-owner:*` — would hang the request instead of answering. */ + logger.error(`Session ownership lookup failed - Session ID: ${session_id} | File ID: ${fileId}`, err); + return res.status(500).json({ error: 'Internal server error' }); + } + if (!ownership.authorized) { + logger.error(`Unauthorized ${isDelete ? 'delete' : 'download'}: Cached session key: ${ownership.cachedSessionKey} | Expected session key: ${sessionKey} | Session ID: ${session_id} | File ID: ${fileId} | Reason: ${ownership.reason}`); return res.status(403).json({ error: 'Unauthorized' }); } + if (ownership.source === 'owner') { + logger.info(`Delete authorized from durable owner record - Session ID: ${session_id} | File ID: ${fileId}`); + } req.sessionKey = sessionKey; next(); diff --git a/service/src/rehydrate-session-cache.test.ts b/service/src/rehydrate-session-cache.test.ts index 22e391f8..4b9a0b5c 100644 --- a/service/src/rehydrate-session-cache.test.ts +++ b/service/src/rehydrate-session-cache.test.ts @@ -25,6 +25,7 @@ const SOURCE = { class MemoryStore implements RecoveryStore { readonly values = new Map(); + readonly ttls = new Map(); async get(key: string): Promise { return this.values.get(key) ?? null; @@ -34,15 +35,60 @@ class MemoryStore implements RecoveryStore { key: string, value: string, _expiryMode: 'EX', - _ttlSeconds: number, + ttlSeconds: number, _condition: 'NX', ): Promise<'OK' | null> { if (this.values.has(key)) { return null; } this.values.set(key, value); + this.ttls.set(key, ttlSeconds); return 'OK'; } + + async ttl(key: string): Promise { + if (!this.values.has(key)) { + return -2; + } + return this.ttls.get(key) ?? -1; + } + + async expire(key: string, ttlSeconds: number): Promise { + if (!this.values.has(key)) { + return 0; + } + this.ttls.set(key, ttlSeconds); + return 1; + } + + async del(key: string): Promise { + this.ttls.delete(key); + return this.values.delete(key) ? 1 : 0; + } +} + +/** Loses every `SET NX` on the durable owner key, optionally planting a + * different owner for the follow-up read to find. */ +class RacingOwnerStore extends MemoryStore { + constructor(private readonly ownerAfterRace: string | null) { + super(); + } + + async set( + key: string, + value: string, + expiryMode: 'EX', + ttlSeconds: number, + condition: 'NX', + ): Promise<'OK' | null> { + if (!key.startsWith('session-owner:')) { + return super.set(key, value, expiryMode, ttlSeconds, condition); + } + if (this.ownerAfterRace !== null) { + this.values.set(key, this.ownerAfterRace); + } + return null; + } } describe('rehydrate-session-cache', () => { @@ -156,6 +202,272 @@ describe('rehydrate-session-cache', () => { expect(() => parseOptions([], {})).toThrow('Recovery environment'); }); + it('defaults the owner TTL past the cache TTL and never below it', () => { + expect(parseOptions([], { + SESSION_RECOVERY_ENVIRONMENT: 'configured-env', + SESSION_RECOVERY_REGION: 'configured-region', + SESSION_RECOVERY_NAMESPACE: 'configured-namespace', + })).toMatchObject({ ttlSeconds: 86400, ownerTtlSeconds: 90 * 86400 }); + + expect(parseOptions(['--owner-ttl-seconds', '604800'], { + SESSION_RECOVERY_ENVIRONMENT: 'configured-env', + SESSION_RECOVERY_REGION: 'configured-region', + SESSION_RECOVERY_NAMESPACE: 'configured-namespace', + })).toMatchObject({ ownerTtlSeconds: 604800 }); + + /* A shorter owner TTL would re-strand the session it just recovered. */ + expect(parseOptions([ + '--ttl-seconds', '86400', + '--owner-ttl-seconds', '600', + ], { + SESSION_RECOVERY_ENVIRONMENT: 'configured-env', + SESSION_RECOVERY_REGION: 'configured-region', + SESSION_RECOVERY_NAMESPACE: 'configured-namespace', + })).toMatchObject({ ttlSeconds: 86400, ownerTtlSeconds: 86400 }); + }); + + it('restores the durable owner record alongside the cache key', async () => { + const store = new MemoryStore(); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toMatchObject({ restored: 1, conflicts: 0 }); + expect(store.values.get(`session:${SESSION_ID}`)).toBe(SESSION_KEY); + expect(store.ttls.get(`session:${SESSION_ID}`)).toBe(86400); + expect(store.values.get(`session-owner:${SESSION_ID}`)).toBe(SESSION_KEY); + expect(store.ttls.get(`session-owner:${SESSION_ID}`)).toBe(7776000); + }); + + it('backfills the owner record for a session whose cache key is still live', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, SESSION_KEY); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toMatchObject({ matching: 1, restored: 0, conflicts: 0 }); + expect(store.values.get(`session-owner:${SESSION_ID}`)).toBe(SESSION_KEY); + }); + + it('reports a conflicting owner record and restores nothing for that session', async () => { + const store = new MemoryStore(); + store.values.set(`session-owner:${SESSION_ID}`, 'tenant-id:user:someone-else'); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 0, conflicts: 1 }); + expect(store.values.get(`session-owner:${SESSION_ID}`)).toBe('tenant-id:user:someone-else'); + /* The manifest's claimant must not get a day of access the service + * never granted it. */ + expect(store.values.has(`session:${SESSION_ID}`)).toBe(false); + }); + + it('creates no durable owner record when the live cache key names another owner', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, 'tenant-id:user:someone-else'); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 0, conflicts: 1 }); + /* A durable record written here would outlive the cache key that + * contradicts it, and `sessionAuth` would then authorize the manifest + * owner to delete the real owner's files. */ + expect(store.values.has(`session-owner:${SESSION_ID}`)).toBe(false); + }); + + it('rolls back a restored cache key when the owner record turns out to be another owner', async () => { + const store = new RacingOwnerStore('tenant-id:user:someone-else'); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 0, conflicts: 1 }); + /* Left in place, the grant would authorize the manifest owner for a + * full ttlSeconds against a session the durable record says is not + * theirs. */ + expect(store.values.has(`session:${SESSION_ID}`)).toBe(false); + }); + + it('reports an owner record that cannot be created as missing', async () => { + const store = new RacingOwnerStore(null); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + /* `missing` during an apply is what drives the nonzero exit — the run + * must not look like a completed recovery. */ + expect(summary).toEqual({ input: 1, missing: 1, restored: 0, matching: 0, conflicts: 0 }); + expect(store.values.has(`session-owner:${SESSION_ID}`)).toBe(false); + /* The cache key stays: no durable record is where this session already + * was, and removing it would leave the operator with nothing. */ + expect(store.values.get(`session:${SESSION_ID}`)).toBe(SESSION_KEY); + }); + + it('reports pending owner work during a dry run', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, SESSION_KEY); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: false, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + /* Counting this as `matching` would tell an operator the session is + * fully in place and invite them to skip the apply that creates its + * durable record. */ + expect(summary).toEqual({ input: 1, missing: 1, restored: 0, matching: 0, conflicts: 0 }); + expect(store.values.has(`session-owner:${SESSION_ID}`)).toBe(false); + }); + + it('reports a short-lived owner record as pending work during a dry run', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, SESSION_KEY); + store.values.set(`session-owner:${SESSION_ID}`, SESSION_KEY); + store.ttls.set(`session-owner:${SESSION_ID}`, 600); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: false, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 1, restored: 0, matching: 0, conflicts: 0 }); + expect(store.ttls.get(`session-owner:${SESSION_ID}`)).toBe(600); + }); + + it('counts a fully in-place session as matching during a dry run', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, SESSION_KEY); + store.values.set(`session-owner:${SESSION_ID}`, SESSION_KEY); + store.ttls.set(`session-owner:${SESSION_ID}`, 7776000); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: false, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 1, conflicts: 0 }); + }); + + it('creates the owner record when it expires between inspection and refresh', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, SESSION_KEY); + store.values.set(`session-owner:${SESSION_ID}`, SESSION_KEY); + store.ttls.set(`session-owner:${SESSION_ID}`, 600); + const realTtl = store.ttl.bind(store); + let ttlReads = 0; + store.ttl = async (key: string): Promise => { + ttlReads += 1; + if (ttlReads === 1) { + /* Lapses in the window between the inspection read and the + * extension it was about to perform. */ + store.values.delete(`session-owner:${SESSION_ID}`); + store.ttls.delete(`session-owner:${SESSION_ID}`); + return -2; + } + return realTtl(key); + }; + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 1, conflicts: 0 }); + expect(store.values.get(`session-owner:${SESSION_ID}`)).toBe(SESSION_KEY); + expect(store.ttls.get(`session-owner:${SESSION_ID}`)).toBe(7776000); + }); + + it('reports an owner record replaced during the refresh as a conflict', async () => { + const store = new MemoryStore(); + store.values.set(`session:${SESSION_ID}`, SESSION_KEY); + store.values.set(`session-owner:${SESSION_ID}`, SESSION_KEY); + store.ttls.set(`session-owner:${SESSION_ID}`, 600); + const realExpire = store.expire.bind(store); + store.expire = async (key: string, ttlSeconds: number): Promise => { + const result = await realExpire(key, ttlSeconds); + store.values.set(`session-owner:${SESSION_ID}`, 'tenant-id:user:someone-else'); + return result; + }; + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + /* The extension itself is harmless — it prolongs a claim the service + * wrote — but the session must not be reported as recovered. */ + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 0, conflicts: 1 }); + }); + + it('surfaces a conflicting owner record during a dry run', async () => { + const store = new MemoryStore(); + store.values.set(`session-owner:${SESSION_ID}`, 'tenant-id:user:someone-else'); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: false, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toEqual({ input: 1, missing: 0, restored: 0, matching: 0, conflicts: 1 }); + expect(store.values.size).toBe(1); + }); + + it('extends a matching owner record that would expire before the restored cache key', async () => { + const store = new MemoryStore(); + store.values.set(`session-owner:${SESSION_ID}`, SESSION_KEY); + store.ttls.set(`session-owner:${SESSION_ID}`, 600); + + const summary = await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(summary).toMatchObject({ restored: 1, conflicts: 0 }); + expect(store.ttls.get(`session-owner:${SESSION_ID}`)).toBe(7776000); + }); + + it('leaves a longer-lived owner record alone', async () => { + const store = new MemoryStore(); + store.values.set(`session-owner:${SESSION_ID}`, SESSION_KEY); + store.ttls.set(`session-owner:${SESSION_ID}`, 9999999); + + await recoverSessionCache( + [{ session_id: SESSION_ID, expected_session_key: SESSION_KEY }], + store, + { apply: true, ttlSeconds: 86400, ownerTtlSeconds: 7776000 }, + ); + + expect(store.ttls.get(`session-owner:${SESSION_ID}`)).toBe(9999999); + }); + it('does not write during a dry run', async () => { const store = new MemoryStore(); const summary = await recoverSessionCache( @@ -189,9 +501,12 @@ describe('rehydrate-session-cache', () => { it('preserves partial counts when a Redis operation fails', async () => { let reads = 0; const store: RecoveryStore = { + /* Three reads to settle the first record: the durable owner, the + * cache key, then the owner re-read that confirms the refresh. The + * second record fails on its first read. */ async get(): Promise { reads += 1; - if (reads === 1) { + if (reads <= 3) { return SESSION_KEY; } throw new Error('Redis unavailable'); @@ -199,6 +514,15 @@ describe('rehydrate-session-cache', () => { async set(): Promise<'OK' | null> { throw new Error('unexpected set'); }, + async ttl(): Promise { + return 7776000; + }, + async expire(): Promise { + throw new Error('unexpected expire'); + }, + async del(): Promise { + throw new Error('unexpected del'); + }, }; try { @@ -231,6 +555,15 @@ describe('rehydrate-session-cache', () => { async set(): Promise { return null; }, + async ttl(): Promise { + return -2; + }, + async expire(): Promise { + return 0; + }, + async del(): Promise { + return 0; + }, }; const summary = await recoverSessionCache( diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index baa350e6..4536270b 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -40,6 +40,7 @@ import { } from '../sandbox-egress'; import { findUnregisteredToolCall } from '../tool-scope'; import { summarizeRequestedFiles } from '../execution-log'; +import { clearSessionOwnership, recordSessionOwnership } from '../session-ownership'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; import { buildReplayExecutionState, @@ -562,7 +563,7 @@ async function handleReplayInitial( code.includes('import matplotlib') || code.includes('import seaborn') ); - await connection.set(`session:${session_id}`, sessionKey, 'EX', env.SESSION_CACHE_TTL); + await recordSessionOwnership(connection, session_id, sessionKey); const state = buildReplayExecutionState({ executionId: execution_id, @@ -605,7 +606,7 @@ async function handleReplayInitial( bytes: err.bytes, cap: err.cap, }); - await connection.del(`session:${session_id}`).catch(() => {}); + await clearSessionOwnership(connection, session_id).catch(() => {}); ptcReplayStateOversize.inc(); res.status(413).json({ error: `Request too large: serialized execution state is ${err.bytes} bytes (max ${err.cap}). Reduce the size of "code", "tools", or "files".`, @@ -1315,7 +1316,12 @@ async function handleBlocking( const execution_id = nanoid(); const identity = getExecutionIdentity(req, userId); - connection.set(`session:${session_id}`, sessionKey, 'EX', env.SESSION_CACHE_TTL); + /* Awaited: a partial registration (cache key written, durable record + * refused — a Redis ACL scoped to `session:*` would do it) would let the + * job write files that become undeletable once `SESSION_CACHE_TTL` + * lapses. The caller turns a rejection into a 500 before anything is + * enqueued. */ + await recordSessionOwnership(connection, session_id, sessionKey); const executionState: ExecutionState = { execution_id, diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 971bb7fd..f1a840be 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -24,6 +24,7 @@ import { captureTraceCarrier, withSpan } from '../telemetry'; import { Jobs, Languages } from '../enum'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; import { createUploadSessionRegistrar } from './upload-session'; +import { recordSessionOwnership } from '../session-ownership'; import { prepareSandboxJobSecurity } from '../sandbox-egress'; import { BridgeWorkerSelectionError, @@ -219,7 +220,16 @@ router.post('/exec', executionLimiter, async (req: t.AuthenticatedRequest, res) * sandbox invocation." */ const session_id = nanoid(); const execution_id = nanoid(); - await connection.set(`session:${session_id}`, sessionKey, 'EX', env.SESSION_CACHE_TTL); + /* Guarded: registration runs before the route's `try`, and Express 4 + * does not forward a rejected async handler to the error middleware — + * an unavailable Redis, or an ACL that permits `session:*` but not + * `session-owner:*`, would hang the request instead of answering. */ + try { + await recordSessionOwnership(connection, session_id, sessionKey); + } catch (error) { + logger.error(`[${INSTANCE_ID}] Error registering session ownership - Session ID: ${session_id}:`, error); + return res.status(500).json({ error: 'Internal server error' }); + } try { if (!isSyntheticRequest) { @@ -478,7 +488,7 @@ router.post('/upload', uploadLimiter, async (req: t.AuthenticatedRequest, res: R if (readOnly) { putHeaders['X-Read-Only'] = 'true'; } - connection.set(`session:${session_id}`, sessionKey, 'EX', env.SESSION_CACHE_TTL) + recordSessionOwnership(connection, session_id, sessionKey) .then(() => { logger.info(`[${INSTANCE_ID}] Upload: Session ID: ${session_id} | User ID: ${userId} | Session key: ${sessionKey}`); return axios.put( @@ -600,7 +610,7 @@ router.post('/upload/batch', uploadLimiter, async (req: t.AuthenticatedRequest, const ensureSessionRegistered = createUploadSessionRegistrar((sessionKey) => { logger.info(`[${INSTANCE_ID}] Batch upload: Session ID: ${session_id} | User ID: ${userId} | Session key: ${sessionKey}`); - return connection.set(`session:${session_id}`, sessionKey, 'EX', env.SESSION_CACHE_TTL); + return recordSessionOwnership(connection, session_id, sessionKey); }); const planFileSize = planLimits[req.planId ?? '']?.max_file_size ?? planLimits.default.max_file_size; @@ -904,7 +914,15 @@ router.get('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, a } }); -router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, async (req: t.AuthenticatedRequest, res: Response) => { +/** + * Remove a session object. + * + * Mounted on two paths (see the registrations below); both are gated by + * `sessionAuth`, so the caller has to own the `(session_id, entity_id)` + * pair the object was stored under, and both proxy the same file-server + * route. + */ +const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) => { const { session_id, fileId } = req.params; try { @@ -917,12 +935,46 @@ router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, async (re logger.info(`[${INSTANCE_ID}] File deleted: Session ID: ${session_id} | File ID: ${fileId}`); return res.status(200).json(response.data); } catch (error) { + /* The file-server answers 404 when the object is already gone. Pass + * that through instead of collapsing it into a 500: a client sweeping + * expired files can retire the reference on 404, whereas a 500 reads + * as retryable and has it re-issuing the same DELETE for an object + * that no longer exists on every subsequent pass. */ + if (axios.isAxiosError(error) && error.response?.status === 404) { + /* Best effort: this runs inside the catch block, where a rejection + * has no handler above it — Express 4 does not forward async + * rejections, so it would hang the request instead of answering. + * The key expires on its own, and the object is already gone. */ + await connection.del(`upload:${req.sessionKey}${session_id}${fileId}`).catch((err: unknown) => { + logger.warn(`[${INSTANCE_ID}] Failed to clear upload key for absent file - Session ID: ${session_id} | File ID: ${fileId}:`, err); + }); + logger.info(`[${INSTANCE_ID}] File already absent: Session ID: ${session_id} | File ID: ${fileId}`); + return res.status(404).json({ error: 'File not found' }); + } const errorDetails = getAxiosErrorDetails(error); logger.error(`[${INSTANCE_ID}] Error deleting file - Session ID: ${session_id} | File ID: ${fileId}:`, errorDetails); return res.status(500).json({ error: 'Error deleting file', }); } -}); +}; + +router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); + +/** + * Alias of the route above, on the path LibreChat's `deleteCodeEnvFile` + * targets — the file-server's own DELETE path, which is not itself + * exposed on `/v1`. + * + * Until LibreChat v0.8.6 this was the only path the client tried, and the + * 404 from an unmounted method was indistinguishable from "the object is + * already gone": every deletion silently failed and objects accumulated + * with nothing to alert on. Newer clients fall back to `/files/...`, but + * mounting the alias costs nothing and makes deletion work for + * deployments still running an older client. + * + * GET on this same path is the metadata proxy above. + */ +router.delete('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); export default router; diff --git a/service/src/session-ownership.test.ts b/service/src/session-ownership.test.ts new file mode 100644 index 00000000..bc9e2b74 --- /dev/null +++ b/service/src/session-ownership.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, test } from 'bun:test'; + +import { + authorizeSessionOwnership, + clearSessionOwnership, + recordSessionOwnership, + sessionCacheKey, + sessionOwnerKey, + type SessionOwnershipStore, +} from './session-ownership'; + +interface Written { + value: string; + ttl: number; +} + +function createStore(seed: Record = {}) { + const values = new Map(Object.entries(seed)); + const writes: Record = {}; + const store: SessionOwnershipStore = { + get: async (key) => values.get(key) ?? null, + set: async (key, value, _expiryMode, ttlSeconds) => { + values.set(key, value); + writes[key] = { value, ttl: ttlSeconds }; + return 'OK'; + }, + del: async (...keys) => { + let removed = 0; + for (const key of keys) { + if (values.delete(key)) removed += 1; + } + return removed; + }, + }; + return { store, writes, values }; +} + +const SESSION_ID = 'session-1'; +const OWNER = 'tenant-1:user:user-1'; + +describe('recordSessionOwnership', () => { + test('writes the cache key and the durable owner record together', async () => { + const { store, writes } = createStore(); + + await recordSessionOwnership(store, SESSION_ID, OWNER, { cacheTtl: 100, ownerTtl: 9000 }); + + expect(writes[sessionCacheKey(SESSION_ID)]).toEqual({ value: OWNER, ttl: 100 }); + expect(writes[sessionOwnerKey(SESSION_ID)]).toEqual({ value: OWNER, ttl: 9000 }); + }); +}); + +describe('clearSessionOwnership', () => { + test('rolls back both records', async () => { + const { store, values } = createStore(); + await recordSessionOwnership(store, SESSION_ID, OWNER, { cacheTtl: 100, ownerTtl: 9000 }); + + await clearSessionOwnership(store, SESSION_ID); + + expect(values.has(sessionCacheKey(SESSION_ID))).toBe(false); + expect(values.has(sessionOwnerKey(SESSION_ID))).toBe(false); + }); +}); + +describe('authorizeSessionOwnership', () => { + test('authorizes from the cache key while it is live', async () => { + const { store } = createStore({ [sessionCacheKey(SESSION_ID)]: OWNER }); + + const result = await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: false, + }); + + expect(result).toEqual({ authorized: true, source: 'session' }); + }); + + test('denies a read once the cache key has expired, even with an owner record', async () => { + const { store } = createStore({ [sessionOwnerKey(SESSION_ID)]: OWNER }); + + const result = await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: false, + }); + + expect(result).toEqual({ authorized: false, reason: 'expired', cachedSessionKey: null }); + }); + + test('authorizes a delete from the owner record once the cache key has expired', async () => { + const { store } = createStore({ [sessionOwnerKey(SESSION_ID)]: OWNER }); + + const result = await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: true, + }); + + expect(result).toEqual({ authorized: true, source: 'owner' }); + }); + + test('a live cache key naming another owner is authoritative and blocks the fallback', async () => { + const { store } = createStore({ + [sessionCacheKey(SESSION_ID)]: 'tenant-1:user:someone-else', + /* Stale owner record that would otherwise match — a re-registered + * session must not be deletable by its previous owner. */ + [sessionOwnerKey(SESSION_ID)]: OWNER, + }); + + const result = await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: true, + }); + + expect(result).toEqual({ + authorized: false, + reason: 'mismatch', + cachedSessionKey: 'tenant-1:user:someone-else', + }); + }); + + test('denies a delete when the owner record names someone else', async () => { + const { store } = createStore({ [sessionOwnerKey(SESSION_ID)]: 'tenant-2:user:user-2' }); + + const result = await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: true, + }); + + expect(result).toEqual({ + authorized: false, + reason: 'mismatch', + cachedSessionKey: 'tenant-2:user:user-2', + }); + }); + + test('reports sessions that predate the owner record as unknown', async () => { + const { store } = createStore(); + + const result = await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: true, + }); + + expect(result).toEqual({ authorized: false, reason: 'unknown', cachedSessionKey: null }); + }); + + test('a session registered through recordSessionOwnership stays deletable past the cache TTL', async () => { + const { store, values } = createStore(); + await recordSessionOwnership(store, SESSION_ID, OWNER, { cacheTtl: 1, ownerTtl: 9000 }); + + /* Simulate the cache key aging out while the owner record lives on. */ + values.delete(sessionCacheKey(SESSION_ID)); + + expect( + await authorizeSessionOwnership(store, { + session_id: SESSION_ID, + expectedSessionKey: OWNER, + allowExpiredCache: true, + }), + ).toEqual({ authorized: true, source: 'owner' }); + }); +}); diff --git a/service/src/session-ownership.ts b/service/src/session-ownership.ts new file mode 100644 index 00000000..bf4ba6dd --- /dev/null +++ b/service/src/session-ownership.ts @@ -0,0 +1,126 @@ +import { env } from './config'; + +/** + * Ownership of a session's stored objects is recorded twice. + * + * `session:` is the hot path: `sessionAuth` compares it on + * every download, metadata fetch and execution input, and its + * `SESSION_CACHE_TTL` (24h by default) is deliberately short — it bounds + * how long a sandbox invocation's outputs stay reachable. + * + * That bound is wrong for deletion. A file is deletable only while + * someone can still prove they own it, so with one key the window in + * which an object can be removed closes a day after upload and the + * object is stranded for the life of the deployment: unreadable, + * unusable as an execution input, and undeletable through every route. + * Clients sweeping their own retention window are typically far outside + * 24h when they get there (LibreChat's default retention is 30 days), so + * in practice every swept object failed to delete and the bucket only + * ever grew. + * + * `session-owner:` is the durable half: same value, TTL + * `SESSION_OWNER_TTL`, consulted only when the cache key has expired and + * only for deletes. Read access keeps the original 24h bound. + */ + +/** The subset of the Redis client these helpers need — narrow enough to + * fake in tests without standing up a connection. */ +export interface SessionOwnershipStore { + get(key: string): Promise; + set(key: string, value: string, expiryMode: 'EX', ttlSeconds: number): Promise; + del(...keys: string[]): Promise; +} + +export const sessionCacheKey = (session_id: string): string => `session:${session_id}`; +export const sessionOwnerKey = (session_id: string): string => `session-owner:${session_id}`; + +export interface SessionOwnershipTtls { + cacheTtl?: number; + ownerTtl?: number; +} + +/** + * Register `sessionKey` as the owner of `session_id`, writing both the + * cache key and the durable owner record. Replaces the bare + * `connection.set('session:…')` at every site that opens a session, so + * the two can never drift apart. + */ +export function recordSessionOwnership( + store: SessionOwnershipStore, + session_id: string, + sessionKey: string, + ttls: SessionOwnershipTtls = {}, +): Promise { + const cacheTtl = ttls.cacheTtl ?? env.SESSION_CACHE_TTL; + const ownerTtl = ttls.ownerTtl ?? env.SESSION_OWNER_TTL; + return Promise.all([ + store.set(sessionCacheKey(session_id), sessionKey, 'EX', cacheTtl), + store.set(sessionOwnerKey(session_id), sessionKey, 'EX', ownerTtl), + ]); +} + +/** + * Roll back a registration, dropping both records. Used where a request + * is rejected after opening a session — leaving the durable half behind + * would keep an owner record alive for `SESSION_OWNER_TTL` on a session + * that never stored anything. + */ +export function clearSessionOwnership( + store: SessionOwnershipStore, + session_id: string, +): Promise { + return store.del(sessionCacheKey(session_id), sessionOwnerKey(session_id)); +} + +export type SessionOwnershipSource = 'session' | 'owner'; + +/** `expired`: nothing live, and the durable record was not consulted or + * had also lapsed. `unknown`: the session predates the owner record, so + * ownership can no longer be established (recoverable with + * `scripts/rehydrate-session-cache.ts`). `mismatch`: a recorded owner + * exists and it is somebody else. */ +export type SessionOwnershipDenial = 'expired' | 'unknown' | 'mismatch'; + +export type SessionOwnershipResult = + | { authorized: true; source: SessionOwnershipSource } + | { authorized: false; reason: SessionOwnershipDenial; cachedSessionKey: string | null }; + +/** + * Decide whether `expectedSessionKey` owns `session_id`. + * + * A live cache key is always authoritative — when one exists and names a + * different owner the answer is no, and the durable record is not + * consulted. The fallback only covers the case where the cache key is + * simply gone. + */ +export async function authorizeSessionOwnership( + store: SessionOwnershipStore, + args: { + session_id: string; + expectedSessionKey: string; + /** Enable the durable fallback. Deletes pass true; reads keep the + * `SESSION_CACHE_TTL` window they have always had. */ + allowExpiredCache: boolean; + }, +): Promise { + const { session_id, expectedSessionKey, allowExpiredCache } = args; + const cachedSessionKey = await store.get(sessionCacheKey(session_id)); + if (cachedSessionKey === expectedSessionKey) { + return { authorized: true, source: 'session' }; + } + if (cachedSessionKey !== null) { + return { authorized: false, reason: 'mismatch', cachedSessionKey }; + } + if (!allowExpiredCache) { + return { authorized: false, reason: 'expired', cachedSessionKey }; + } + + const recordedOwner = await store.get(sessionOwnerKey(session_id)); + if (recordedOwner === expectedSessionKey) { + return { authorized: true, source: 'owner' }; + } + if (recordedOwner === null) { + return { authorized: false, reason: 'unknown', cachedSessionKey }; + } + return { authorized: false, reason: 'mismatch', cachedSessionKey: recordedOwner }; +} From 48526ef9d98108b11b70ad6d49abe0c8aed2738b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 17:34:15 -0400 Subject: [PATCH 022/116] =?UTF-8?q?=F0=9F=8C=B3=20feat:=20Add=20Confined?= =?UTF-8?q?=20Local=20Workspace=20Tools=20(#88)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): add confined local workspace tools * fix(code): confine workspace text search * fix(code): center bounded search previews * fix(code): tighten workspace result semantics * fix(code): harden workspace text search * fix(code): normalize workspace text boundaries --- packages/code/Dockerfile | 4 +- packages/code/README.md | 23 + packages/code/package.json | 4 + packages/code/src/index.ts | 1 + packages/code/src/protocol.test.ts | 35 ++ packages/code/src/protocol.ts | 62 ++- packages/code/src/workspace.test.ts | 577 ++++++++++++++++++++++++ packages/code/src/workspace.ts | 651 ++++++++++++++++++++++++++++ 8 files changed, 1355 insertions(+), 2 deletions(-) create mode 100644 packages/code/src/workspace.test.ts create mode 100644 packages/code/src/workspace.ts diff --git a/packages/code/Dockerfile b/packages/code/Dockerfile index 21fb09bd..015ca2d0 100644 --- a/packages/code/Dockerfile +++ b/packages/code/Dockerfile @@ -7,7 +7,9 @@ RUN npm run build FROM node:24-alpine ENV NODE_ENV=production -RUN addgroup -S librechat-code && adduser -S librechat-code -G librechat-code +RUN apk add --no-cache ripgrep \ + && addgroup -S librechat-code \ + && adduser -S librechat-code -G librechat-code WORKDIR /app COPY --from=build /app/package.json ./package.json COPY --from=build /app/dist ./dist diff --git a/packages/code/README.md b/packages/code/README.md index caf129b5..cd48adb9 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -188,6 +188,29 @@ accepting another assignment. Reset or discard that session's local runner before restarting the worker; its workspace may contain mutations that Code API did not commit. +## Local workspace tools (library preview) + +`@librechat/code/workspace` provides the provider-neutral foundation for +coding-agent access to repositories that already live on the worker machine. +`LocalWorkspaceTools` registers opaque workspace IDs with optional display +names and exposes bounded `read_file` and literal `search_text` operations. +Only IDs, names, protocol version, and supported operations appear in worker +capabilities; absolute host paths remain local to the worker process. + +Reads reject absolute paths, traversal, escaping symlinks, non-regular files, +and files larger than 1 MiB. The opened file is checked against its canonical +in-workspace inode before it is read. Text search uses `rg` only to enumerate a +bounded set of ignored-aware candidates with configuration and symlink following +disabled. It then opens and verifies each candidate through the same confined +1 MiB read boundary before matching locally. Search limits returned line length +and stops after a bounded global result count. The worker process still belongs +inside the trusted BYOM boundary and should receive filesystem access only to +roots the operator intentionally registers. + +This release exposes the library protocol only. The subsequent bridge-dispatch +layer will route signed, deadline-bound workspace tool assignments to it; until +that layer is configured, the CLI does not advertise or execute these tools. + After discarding or resetting that session's local runner, acknowledge recovery with `librechat-code reset-workspace `. The command uses the configured worker credentials, registers a fresh incarnation, and only clears diff --git a/packages/code/package.json b/packages/code/package.json index 0a4257d4..2bce9bc6 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -22,6 +22,10 @@ "./runtime": { "types": "./dist/runtime.d.ts", "import": "./dist/runtime.js" + }, + "./workspace": { + "types": "./dist/workspace.d.ts", + "import": "./dist/workspace.js" } }, "bin": { diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 0f142908..a7c0a830 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -3,4 +3,5 @@ export * from './identity.js'; export * from './pairing.js'; export * from './storage.js'; export * from './runtime.js'; +export * from './workspace.js'; export * from './worker.js'; diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 61c50922..793c7d0f 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -54,3 +54,38 @@ test('bridge worker capabilities enforce registration limits', () => { false, ); }); + +test('bridge worker capabilities accept only bounded public workspace descriptors', () => { + const valid = { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + }; + + assert.equal(isValidBridgeWorkerCapabilities(valid), true); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + workspaces: [{ id: 'primary', root: '/Users/operator/private' }], + }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + workspaces: [{ id: '../escape' }], + }, + }), + false, + ); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index a9ff5649..8a069c36 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -3,15 +3,31 @@ export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; export const BRIDGE_SANDBOX_PROFILE_MAX_LENGTH = 128; export const BRIDGE_RUNTIME_MAX_COUNT = 32; export const BRIDGE_RUNTIME_MAX_LENGTH = 64; +export const BRIDGE_WORKSPACE_MAX_COUNT = 32; +export const BRIDGE_WORKSPACE_NAME_MAX_LENGTH = 128; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; +export type BridgeWorkspaceToolOperation = 'read_file' | 'search_text'; + +export interface BridgeWorkspaceDescriptor { + id: string; + name?: string; +} + +export interface BridgeWorkspaceToolCapabilities { + protocolVersion: BridgeProtocolVersion; + operations: BridgeWorkspaceToolOperation[]; + workspaces: BridgeWorkspaceDescriptor[]; +} + export interface BridgeWorkerCapabilities { statefulWorkspace: boolean; sandboxProfile: string; runtimes: string[]; policyDigest?: string; requiresReadyConfirmation?: boolean; + workspaceTools?: BridgeWorkspaceToolCapabilities; } export interface BridgeWorkerRegistration { @@ -121,6 +137,48 @@ export function isValidBridgeWorkerId(workerId: string): boolean { return BRIDGE_WORKER_ID_PATTERN.test(workerId); } +export function isValidBridgeWorkspaceToolCapabilities( + value: unknown, +): value is BridgeWorkspaceToolCapabilities { + if (typeof value !== 'object' || value === null) return false; + const capabilities = value as Record; + if ( + capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !Array.isArray(capabilities.operations) || + capabilities.operations.length < 1 || + capabilities.operations.length > 2 || + !capabilities.operations.every( + (operation) => operation === 'read_file' || operation === 'search_text', + ) || + new Set(capabilities.operations).size !== capabilities.operations.length || + !Array.isArray(capabilities.workspaces) || + capabilities.workspaces.length < 1 || + capabilities.workspaces.length > BRIDGE_WORKSPACE_MAX_COUNT + ) { + return false; + } + + const workspaceIds = new Set(); + return capabilities.workspaces.every((workspace) => { + if (typeof workspace !== 'object' || workspace === null) return false; + const descriptor = workspace as Record; + if ( + Object.keys(descriptor).some((key) => key !== 'id' && key !== 'name') || + typeof descriptor.id !== 'string' || + !isValidBridgeWorkerId(descriptor.id) || + workspaceIds.has(descriptor.id) || + (descriptor.name !== undefined && + (typeof descriptor.name !== 'string' || + descriptor.name.trim().length === 0 || + descriptor.name.length > BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) + ) { + return false; + } + workspaceIds.add(descriptor.id); + return true; + }); +} + export function isValidBridgeWorkerCapabilities( value: unknown, ): value is BridgeWorkerCapabilities { @@ -143,6 +201,8 @@ export function isValidBridgeWorkerCapabilities( (typeof capabilities.policyDigest === 'string' && /^[a-f0-9]{64}$/.test(capabilities.policyDigest))) && (capabilities.requiresReadyConfirmation === undefined || - typeof capabilities.requiresReadyConfirmation === 'boolean') + typeof capabilities.requiresReadyConfirmation === 'boolean') && + (capabilities.workspaceTools === undefined || + isValidBridgeWorkspaceToolCapabilities(capabilities.workspaceTools)) ); } diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts new file mode 100644 index 00000000..0fd9ad23 --- /dev/null +++ b/packages/code/src/workspace.test.ts @@ -0,0 +1,577 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +const execFileAsync = promisify(execFile); + +test('reads a bounded range from a registered local workspace', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + await writeFile( + join(root, 'src', 'app.ts'), + 'first\nsecond\nthird\nfourth\n', + ); + + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'src/app.ts', + startLine: 2, + maxLines: 2, + }), + { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'src/app.ts', + content: 'second\nthird', + startLine: 2, + endLine: 3, + truncated: true, + nextStartLine: 4, + }, + ); +}); + +test('rejects traversal outside a registered workspace without leaking its host path', async (t) => { + const parent = await mkdtemp( + join(tmpdir(), 'librechat-code-workspace-parent-'), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'repo'); + await mkdir(root); + await writeFile(join(parent, 'secret.txt'), 'host secret'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: '../secret.txt', + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /invalid workspace path/i); + assert.equal(error.message.includes(parent), false); + return true; + }, + ); +}); + +test('rejects non-scalar Unicode workspace paths', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, '\ufffd.txt'), 'needle'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: '\ud800.txt', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + path: '\ud800.txt', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('rejects a symlink that escapes a registered workspace', async (t) => { + const parent = await mkdtemp( + join(tmpdir(), 'librechat-code-workspace-parent-'), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'repo'); + await mkdir(root); + await writeFile(join(parent, 'secret.txt'), 'host secret'); + await symlink(join(parent, 'secret.txt'), join(root, 'linked-secret.txt')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'linked-secret.txt', + }), + /invalid workspace path/i, + ); +}); + +test('searches workspace text with a hard global result bound', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'notes.txt'), 'needle one\nignore\nneedle two\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + maxResults: 1, + }), + { + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + matches: [{ path: 'notes.txt', line: 1, column: 1, text: 'needle one' }], + truncated: true, + }, + ); +}); + +test('search ignores ripgrep config that follows escaping symlinks', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-search-parent-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'repo'); + const outside = join(parent, 'outside'); + await mkdir(root); + await mkdir(outside); + await writeFile(join(outside, 'secret.txt'), 'needle secret'); + await symlink(outside, join(root, 'linked-outside')); + const config = join(parent, 'ripgrep.conf'); + await writeFile(config, '--follow\n'); + const previousConfig = process.env.RIPGREP_CONFIG_PATH; + process.env.RIPGREP_CONFIG_PATH = config; + t.after(() => { + if (previousConfig === undefined) delete process.env.RIPGREP_CONFIG_PATH; + else process.env.RIPGREP_CONFIG_PATH = previousConfig; + }); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.deepEqual(result.matches, []); +}); + +test('search does not read an explicitly targeted escaping symlink', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-search-parent-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'repo'); + await mkdir(root); + await writeFile(join(parent, 'secret.txt'), 'needle secret'); + await symlink(join(parent, 'secret.txt'), join(root, 'linked-secret.txt')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + path: 'linked-secret.txt', + }), + /invalid workspace path/i, + ); +}); + +test('search returns a bounded match for a very long line', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'long.txt'), `${'a'.repeat(512 * 1024)} needle`); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.equal(result.matches.length, 1); + assert.equal(result.matches[0]?.text.length, 2000); + assert.match(result.matches[0]?.text ?? '', /needle/); +}); + +test('search rejects multiline literal queries', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'first\nsecond', + }), + /invalid workspace search/i, + ); +}); + +test('search rejects queries larger than its bounded preview', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'a'.repeat(2001), + }), + /invalid workspace search/i, + ); +}); + +test('search rejects non-scalar Unicode queries', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'notes.txt'), '\ufffd'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: '\ud800', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('search keeps valid UTF-8 intact in a centered preview', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile( + join(root, 'unicode.txt'), + `${'é'.repeat(2500)} needle ${'é'.repeat(2500)}`, + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.equal(result.matches[0]?.text.includes('\ufffd'), false); + assert.match(result.matches[0]?.text ?? '', /needle/); +}); + +test('search does not split surrogate pairs in a centered preview', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile( + join(root, 'emoji.txt'), + `${'\ud83d\ude00'.repeat(1500)}needle${'\ud83d\ude00'.repeat(1500)}`, + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + const preview = result.matches[0]?.text ?? ''; + assert.equal(Buffer.from(preview).toString('utf8'), preview); + assert.match(preview, /needle/); +}); + +test('search strips a UTF-8 BOM before reporting columns', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile( + join(root, 'utf8-bom.txt'), + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('needle')]), + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.deepEqual(result.matches, [ + { path: 'utf8-bom.txt', line: 1, column: 1, text: 'needle' }, + ]); +}); + +test('search handles invalid UTF-8 without silently dropping an ASCII match', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile( + join(root, 'legacy.txt'), + Buffer.concat([Buffer.from([0xff]), Buffer.from(' needle\n')]), + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.equal(result.matches.length, 1); + assert.equal(result.matches[0]?.path, 'legacy.txt'); +}); + +test('search decodes BOM-marked UTF-16 text', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const utf16le = Buffer.from('before needle after', 'utf16le'); + const utf16be = Buffer.from(utf16le); + utf16be.swap16(); + await writeFile( + join(root, 'little-endian.txt'), + Buffer.concat([Buffer.from([0xff, 0xfe]), utf16le]), + ); + await writeFile( + join(root, 'big-endian.txt'), + Buffer.concat([Buffer.from([0xfe, 0xff]), utf16be]), + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.deepEqual( + result.matches.map(({ path, column, text }) => ({ path, column, text })), + [ + { path: 'big-endian.txt', column: 8, text: 'before needle after' }, + { path: 'little-endian.txt', column: 8, text: 'before needle after' }, + ], + ); + + for (const path of ['big-endian.txt', 'little-endian.txt']) { + const read = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path, + }); + if (read.operation !== 'read_file') assert.fail('expected read result'); + assert.equal(read.content, 'before needle after'); + } +}); + +test('read rejects a FIFO without waiting for a writer', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fifo = join(root, 'pipe'); + await execFileAsync('mkfifo', [fifo]); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'pipe', + }), + /invalid workspace path/i, + ); +}); + +test('advertises workspace IDs and names without exposing host roots', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', name: 'LibreChat', root }], + }); + + assert.deepEqual(tools.capabilities, { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }); + assert.equal(JSON.stringify(tools.capabilities).includes(root), false); +}); + +test('rejects unbounded file read parameters before reading the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'notes.txt'), 'safe'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'notes.txt', + startLine: 0, + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'notes.txt', + maxLines: 501, + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('bounds bytes read from a workspace file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'large.txt'), Buffer.alloc(1024 * 1024 + 1, 'a')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'large.txt', + }), + /workspace file exceeds read limit/i, + ); +}); + +test('rejects ambiguous workspace registrations', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + + await assert.rejects( + LocalWorkspaceTools.create({ + workspaces: [ + { id: 'primary', root }, + { id: 'primary', root }, + ], + }), + /invalid workspace registration/i, + ); + await assert.rejects( + LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', name: '', root }], + }), + /invalid workspace registration/i, + ); +}); + +test('rejects unsupported workspace tool protocol versions', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 2, + operation: 'read_file', + workspaceId: 'primary', + path: 'notes.txt', + } as never), + /invalid workspace tool request/i, + ); +}); + +test('does not start workspace I/O after its execution is aborted', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'notes.txt'), 'needle'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + const controller = new AbortController(); + controller.abort(); + + await assert.rejects( + tools.execute( + { + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }, + controller.signal, + ), + /workspace tool execution aborted/i, + ); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts new file mode 100644 index 00000000..b56179c8 --- /dev/null +++ b/packages/code/src/workspace.ts @@ -0,0 +1,651 @@ +import { spawn } from 'node:child_process'; +import { constants } from 'node:fs'; +import { open, realpath, stat } from 'node:fs/promises'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; + +import type { FileHandle } from 'node:fs/promises'; + +import { + BRIDGE_PROTOCOL_VERSION, + isValidBridgeWorkerId, + isValidBridgeWorkspaceToolCapabilities, +} from './protocol.js'; + +import type { + BridgeProtocolVersion, + BridgeWorkspaceDescriptor, + BridgeWorkspaceToolCapabilities, +} from './protocol.js'; + +export interface LocalWorkspaceConfig { + id: string; + name?: string; + root: string; +} + +export interface LocalWorkspaceToolsOptions { + workspaces: LocalWorkspaceConfig[]; +} + +export interface WorkspaceReadFileRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'read_file'; + workspaceId: string; + path: string; + startLine?: number; + maxLines?: number; +} + +export interface WorkspaceReadFileResult { + protocolVersion: BridgeProtocolVersion; + operation: 'read_file'; + workspaceId: string; + path: string; + content: string; + startLine: number; + endLine: number; + truncated: boolean; + nextStartLine?: number; +} + +export interface WorkspaceSearchTextRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'search_text'; + workspaceId: string; + query: string; + path?: string; + maxResults?: number; +} + +export interface WorkspaceSearchMatch { + path: string; + line: number; + column: number; + text: string; +} + +export interface WorkspaceSearchTextResult { + protocolVersion: BridgeProtocolVersion; + operation: 'search_text'; + workspaceId: string; + matches: WorkspaceSearchMatch[]; + truncated: boolean; +} + +export type WorkspaceToolRequest = + WorkspaceReadFileRequest | WorkspaceSearchTextRequest; +export type WorkspaceToolResult = + WorkspaceReadFileResult | WorkspaceSearchTextResult; + +const MAX_READ_LINES = 500; +const MAX_READ_BYTES = 1024 * 1024; +const MAX_SEARCH_PREVIEW_LENGTH = 2000; +const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; +const MAX_SEARCH_CANDIDATES = 20_000; +const SEARCH_TIMEOUT_MS = 10_000; + +function isUtf8ScalarString(value: string): boolean { + return Buffer.from(value).toString('utf8') === value; +} + +function decodeWorkspaceText(content: Buffer): string { + if (content[0] === 0xff && content[1] === 0xfe) { + return new TextDecoder('utf-16le').decode(content.subarray(2)); + } + if (content[0] === 0xfe && content[1] === 0xff) { + return new TextDecoder('utf-16be').decode(content.subarray(2)); + } + const utf8Start = + content[0] === 0xef && content[1] === 0xbb && content[2] === 0xbf ? 3 : 0; + return content.subarray(utf8Start).toString('utf8'); +} + +function sliceWithoutSplittingSurrogates( + value: string, + start: number, + maxLength: number, +): string { + let safeStart = start; + if ( + safeStart > 0 && + safeStart < value.length && + value.charCodeAt(safeStart) >= 0xdc00 && + value.charCodeAt(safeStart) <= 0xdfff && + value.charCodeAt(safeStart - 1) >= 0xd800 && + value.charCodeAt(safeStart - 1) <= 0xdbff + ) { + safeStart += 1; + } + let safeEnd = Math.min(value.length, safeStart + maxLength); + if ( + safeEnd < value.length && + value.charCodeAt(safeEnd - 1) >= 0xd800 && + value.charCodeAt(safeEnd - 1) <= 0xdbff && + value.charCodeAt(safeEnd) >= 0xdc00 && + value.charCodeAt(safeEnd) <= 0xdfff + ) { + safeEnd -= 1; + } + return value.slice(safeStart, safeEnd); +} + +export class WorkspaceToolError extends Error { + constructor( + message: string, + public readonly code: + | 'INVALID_PATH' + | 'INVALID_REQUEST' + | 'READ_LIMIT_EXCEEDED' + | 'REGISTRATION_INVALID' + | 'EXECUTION_ABORTED' + | 'SEARCH_TIMEOUT' + | 'SEARCH_UNAVAILABLE', + ) { + super(message); + this.name = 'WorkspaceToolError'; + } +} + +export function isWorkspaceToolRequest( + value: unknown, +): value is WorkspaceToolRequest { + if (typeof value !== 'object' || value === null) return false; + const request = value as Record; + if ( + request.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + typeof request.workspaceId !== 'string' || + !isValidBridgeWorkerId(request.workspaceId) + ) { + return false; + } + if (request.operation === 'read_file') { + return ( + typeof request.path === 'string' && + request.path.length > 0 && + request.path.length <= 4096 && + isUtf8ScalarString(request.path) && + (request.startLine === undefined || + Number.isSafeInteger(request.startLine)) && + (request.maxLines === undefined || Number.isSafeInteger(request.maxLines)) + ); + } + if (request.operation === 'search_text') { + return ( + typeof request.query === 'string' && + request.query.length > 0 && + request.query.length <= 4096 && + (request.path === undefined || + (typeof request.path === 'string' && + request.path.length > 0 && + request.path.length <= 4096 && + isUtf8ScalarString(request.path))) && + (request.maxResults === undefined || + Number.isSafeInteger(request.maxResults)) + ); + } + return false; +} + +function isWithinRoot(root: string, candidate: string): boolean { + const relativePath = relative(root, candidate); + return !( + relativePath === '..' || + relativePath.startsWith(`..${sep}`) || + isAbsolute(relativePath) + ); +} + +function resolveWorkspacePath(root: string, requestedPath: string): string { + if ( + !requestedPath || + requestedPath.includes('\0') || + !isUtf8ScalarString(requestedPath) || + isAbsolute(requestedPath) + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + const candidate = resolve(root, requestedPath); + if (!isWithinRoot(root, candidate)) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + return candidate; +} + +async function readConfinedFileBuffer( + root: string, + requestedPath: string, +): Promise { + const candidate = resolveWorkspacePath(root, requestedPath); + let handle: FileHandle | undefined; + try { + handle = await open( + candidate, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const [openedFile, canonicalPath] = await Promise.all([ + handle.stat(), + realpath(candidate), + ]); + const canonicalFile = await stat(canonicalPath); + if ( + !openedFile.isFile() || + !isWithinRoot(root, canonicalPath) || + openedFile.dev !== canonicalFile.dev || + openedFile.ino !== canonicalFile.ino + ) { + throw new Error('Invalid workspace path'); + } + if (openedFile.size > MAX_READ_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds read limit', + 'READ_LIMIT_EXCEEDED', + ); + } + const buffer = Buffer.allocUnsafe(MAX_READ_BYTES + 1); + let bytesRead = 0; + while (bytesRead <= MAX_READ_BYTES) { + const result = await handle.read( + buffer, + bytesRead, + buffer.length - bytesRead, + bytesRead, + ); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + if (bytesRead > MAX_READ_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds read limit', + 'READ_LIMIT_EXCEEDED', + ); + } + return buffer.subarray(0, bytesRead); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } finally { + await handle?.close(); + } +} + +async function readConfinedFile( + root: string, + requestedPath: string, +): Promise { + return decodeWorkspaceText(await readConfinedFileBuffer(root, requestedPath)); +} + +interface SearchCandidates { + paths: string[]; + truncated: boolean; +} + +async function listSearchCandidates( + root: string, + searchPath: string, + signal: AbortSignal | undefined, + deadline: number, +): Promise { + return new Promise((resolvePromise, reject) => { + const chunks: Buffer[] = []; + let outputBytes = 0; + let stoppedForLimit = false; + const child = spawn( + 'rg', + [ + '--files', + '--no-config', + '--no-follow', + '--null', + '--max-filesize', + '1M', + '--', + searchPath, + ], + { cwd: root, stdio: ['ignore', 'pipe', 'ignore'] }, + ); + let aborted = false; + let timedOut = false; + const abort = () => { + aborted = true; + child.kill(); + }; + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + const timeout = setTimeout( + () => { + timedOut = true; + child.kill(); + }, + Math.max(0, deadline - Date.now()), + ); + const cleanup = () => { + clearTimeout(timeout); + signal?.removeEventListener('abort', abort); + }; + + child.stdout.on('data', (chunk: Buffer) => { + if (stoppedForLimit) return; + const remaining = MAX_SEARCH_CANDIDATE_BYTES - outputBytes; + if (chunk.length > remaining) { + if (remaining > 0) chunks.push(chunk.subarray(0, remaining)); + outputBytes = MAX_SEARCH_CANDIDATE_BYTES; + stoppedForLimit = true; + child.kill(); + return; + } + chunks.push(chunk); + outputBytes += chunk.length; + }); + child.once('error', () => { + cleanup(); + reject( + new WorkspaceToolError( + 'Workspace search unavailable', + 'SEARCH_UNAVAILABLE', + ), + ); + }); + child.once('close', (code) => { + cleanup(); + if (aborted) { + reject( + new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ); + return; + } + if (timedOut) { + reject( + new WorkspaceToolError( + 'Workspace search timed out', + 'SEARCH_TIMEOUT', + ), + ); + return; + } + if (!stoppedForLimit && code !== 0 && code !== 1) { + reject( + new WorkspaceToolError( + 'Workspace search unavailable', + 'SEARCH_UNAVAILABLE', + ), + ); + return; + } + + const output = Buffer.concat(chunks); + const paths: string[] = []; + let start = 0; + let end = output.indexOf(0, start); + while (end >= 0 && paths.length <= MAX_SEARCH_CANDIDATES) { + if (end > start) + paths.push(output.subarray(start, end).toString('utf8')); + start = end + 1; + end = output.indexOf(0, start); + } + const exceededCandidateLimit = paths.length > MAX_SEARCH_CANDIDATES; + if (exceededCandidateLimit) paths.pop(); + resolvePromise({ + paths, + truncated: + stoppedForLimit || exceededCandidateLimit || start < output.length, + }); + }); + }); +} + +async function searchWorkspace( + root: string, + request: WorkspaceSearchTextRequest, + signal?: AbortSignal, +): Promise { + const encodedQuery = Buffer.from(request.query); + if ( + !request.query || + request.query.length > 4096 || + encodedQuery.length > MAX_SEARCH_PREVIEW_LENGTH || + encodedQuery.toString('utf8') !== request.query || + request.query.includes('\0') || + request.query.includes('\n') || + request.query.includes('\r') + ) { + throw new WorkspaceToolError('Invalid workspace search', 'INVALID_REQUEST'); + } + const maxResults = request.maxResults ?? 50; + if (!Number.isSafeInteger(maxResults) || maxResults < 1 || maxResults > 200) { + throw new WorkspaceToolError('Invalid workspace search', 'INVALID_REQUEST'); + } + + const searchPath = request.path ?? '.'; + const target = resolveWorkspacePath(root, searchPath); + let canonicalTarget: string; + try { + canonicalTarget = await realpath(target); + } catch { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + if (!isWithinRoot(root, canonicalTarget)) + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + const canonicalSearchPath = relative(root, canonicalTarget) || '.'; + + const deadline = Date.now() + SEARCH_TIMEOUT_MS; + const candidates = await listSearchCandidates( + root, + canonicalSearchPath, + signal, + deadline, + ); + const matches: WorkspaceSearchMatch[] = []; + let truncated = candidates.truncated; + for (const candidate of candidates.paths) { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (Date.now() >= deadline) { + throw new WorkspaceToolError( + 'Workspace search timed out', + 'SEARCH_TIMEOUT', + ); + } + const path = candidate.startsWith(`.${sep}`) + ? candidate.slice(2) + : candidate; + let content: Buffer; + try { + content = await readConfinedFileBuffer(root, path); + } catch (error) { + if ( + error instanceof WorkspaceToolError && + (error.code === 'INVALID_PATH' || error.code === 'READ_LIMIT_EXCEEDED') + ) { + continue; + } + throw error; + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (Date.now() >= deadline) { + throw new WorkspaceToolError( + 'Workspace search timed out', + 'SEARCH_TIMEOUT', + ); + } + const decodedContent = decodeWorkspaceText(content); + let lineStart = 0; + let lineNumber = 1; + while (lineStart <= decodedContent.length) { + const newline = decodedContent.indexOf('\n', lineStart); + const lineEnd = newline < 0 ? decodedContent.length : newline; + const line = decodedContent.slice( + lineStart, + lineEnd > lineStart && decodedContent[lineEnd - 1] === '\r' + ? lineEnd - 1 + : lineEnd, + ); + const column = line.indexOf(request.query); + if (column >= 0) { + if (matches.length === maxResults) { + truncated = true; + break; + } + const previewStart = Math.min( + Math.max( + 0, + column - + Math.floor( + (MAX_SEARCH_PREVIEW_LENGTH - request.query.length) / 2, + ), + ), + Math.max(0, line.length - MAX_SEARCH_PREVIEW_LENGTH), + ); + matches.push({ + path, + line: lineNumber, + column: column + 1, + text: sliceWithoutSplittingSurrogates( + line, + previewStart, + MAX_SEARCH_PREVIEW_LENGTH, + ), + }); + } + if (newline < 0) break; + lineStart = newline + 1; + lineNumber += 1; + } + if (matches.length === maxResults && truncated) break; + } + + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'search_text', + workspaceId: request.workspaceId, + matches, + truncated, + }; +} + +export class LocalWorkspaceTools { + readonly capabilities: BridgeWorkspaceToolCapabilities; + + private constructor( + private readonly roots: ReadonlyMap, + workspaces: BridgeWorkspaceDescriptor[], + ) { + this.capabilities = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'search_text'], + workspaces, + }; + } + + static async create( + options: LocalWorkspaceToolsOptions, + ): Promise { + const roots = new Map(); + const workspaces: BridgeWorkspaceDescriptor[] = options.workspaces.map( + (workspace) => ({ + id: workspace.id, + ...(workspace.name !== undefined ? { name: workspace.name } : {}), + }), + ); + const capabilities: BridgeWorkspaceToolCapabilities = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'search_text'], + workspaces, + }; + if (!isValidBridgeWorkspaceToolCapabilities(capabilities)) { + throw new WorkspaceToolError( + 'Invalid workspace registration', + 'REGISTRATION_INVALID', + ); + } + for (const workspace of options.workspaces) { + let canonicalRoot: string; + try { + canonicalRoot = await realpath(workspace.root); + if (!(await stat(canonicalRoot)).isDirectory()) throw new Error(); + } catch { + throw new WorkspaceToolError( + 'Invalid workspace registration', + 'REGISTRATION_INVALID', + ); + } + roots.set(workspace.id, canonicalRoot); + } + return new LocalWorkspaceTools(roots, workspaces); + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (!isWorkspaceToolRequest(request)) { + throw new WorkspaceToolError( + 'Invalid workspace tool request', + 'INVALID_REQUEST', + ); + } + const root = this.roots.get(request.workspaceId); + if (!root) { + throw new WorkspaceToolError('Unknown workspace', 'INVALID_REQUEST'); + } + + if (request.operation === 'search_text') { + return searchWorkspace(root, request, signal); + } + + const startLine = request.startLine ?? 1; + const maxLines = request.maxLines ?? 200; + if ( + !Number.isSafeInteger(startLine) || + startLine < 1 || + !Number.isSafeInteger(maxLines) || + maxLines < 1 || + maxLines > MAX_READ_LINES + ) { + throw new WorkspaceToolError('Invalid workspace read', 'INVALID_REQUEST'); + } + const content = await readConfinedFile(root, request.path); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + const lines = content.endsWith('\n') + ? content.slice(0, -1).split('\n') + : content.split('\n'); + const selected = lines.slice(startLine - 1, startLine - 1 + maxLines); + const endLine = startLine + selected.length - 1; + const truncated = endLine < lines.length; + + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: request.workspaceId, + path: request.path, + content: selected.join('\n'), + startLine, + endLine, + truncated, + ...(truncated ? { nextStartLine: endLine + 1 } : {}), + }; + } +} From 65084820c0e3ecd29dd6aafc57275563017742b6 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 19:01:39 -0400 Subject: [PATCH 023/116] =?UTF-8?q?=F0=9F=8F=93=20feat:=20Dispatch=20Worke?= =?UTF-8?q?r-Local=20Workspace=20Tools=20(#89)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): dispatch local workspace tools * fix(code): fence workspace tool completion * fix(code): drain workspace cancellation before settlement * fix(code): drain active cancellation responses * fix(code): preserve drained cancellation fences * fix(code): Fall back from invalid workspace names --- packages/code/README.md | 27 +- packages/code/src/cli.ts | 41 +- packages/code/src/protocol.ts | 5 +- packages/code/src/worker.ts | 186 ++++-- packages/code/src/workspace-cli.test.ts | 118 ++++ packages/code/src/workspace-worker.test.ts | 684 +++++++++++++++++++++ packages/code/src/workspace.ts | 10 +- 7 files changed, 1017 insertions(+), 54 deletions(-) create mode 100644 packages/code/src/workspace-cli.test.ts create mode 100644 packages/code/src/workspace-worker.test.ts diff --git a/packages/code/README.md b/packages/code/README.md index cd48adb9..499f85cb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -188,7 +188,7 @@ accepting another assignment. Reset or discard that session's local runner before restarting the worker; its workspace may contain mutations that Code API did not commit. -## Local workspace tools (library preview) +## Local workspace tools (bridge preview) `@librechat/code/workspace` provides the provider-neutral foundation for coding-agent access to repositories that already live on the worker machine. @@ -207,9 +207,28 @@ and stops after a bounded global result count. The worker process still belongs inside the trusted BYOM boundary and should receive filesystem access only to roots the operator intentionally registers. -This release exposes the library protocol only. The subsequent bridge-dispatch -layer will route signed, deadline-bound workspace tool assignments to it; until -that layer is configured, the CLI does not advertise or execute these tools. +Register one repository already present on the worker machine with the +Cursor-style worker-directory option: + +```bash +librechat-code run --worker-dir /path/to/repository +``` + +The default public workspace ID is `primary` and the default display name is +the directory basename. Operators can use `--workspace-id` and +`--workspace-name`, or `LIBRECHAT_CODE_WORKER_DIR`, +`LIBRECHAT_CODE_WORKSPACE_ID`, and `LIBRECHAT_CODE_WORKSPACE_NAME`, to set them +explicitly. `rg` must be installed on the worker for `search_text`. + +The worker advertises these capabilities only when a directory is configured +and executes matching assignments under the bridge's existing lease, +deadline, cancellation, credential-refresh, and settlement fencing. The +repository itself remains on the worker. As with Cursor's self-hosted agents, +text deliberately selected by `read_file` or `search_text` crosses the outbound +bridge so the remote agent/model can reason over it. Host paths are never part +of that payload. The Code API workspace-tool endpoint is delivered as a +dependent layer; deployments without it continue to use sandbox assignments +unchanged. After discarding or resetting that session's local runner, acknowledge recovery with `librechat-code reset-workspace `. The command uses the diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 9aa6f589..a8cb1f93 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +import { basename, resolve } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { startFileRelay } from './relay.js'; @@ -13,7 +13,9 @@ import { } from './storage.js'; import { BridgeWorker } from './worker.js'; import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; +import { LocalWorkspaceTools } from './workspace.js'; import { + BRIDGE_WORKSPACE_NAME_MAX_LENGTH, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, } from './protocol.js'; @@ -65,6 +67,14 @@ function option(args: string[], name: string): string | undefined { return args.find((value) => value.startsWith(`${name}=`))?.slice(name.length + 1); } +function defaultWorkspaceName(workerDirectory: string, workspaceId: string): string { + const directoryName = basename(resolve(workerDirectory)); + return directoryName.trim().length > 0 && + directoryName.length <= BRIDGE_WORKSPACE_NAME_MAX_LENGTH + ? directoryName + : workspaceId; +} + async function pair(args: string[]): Promise { const codeApiUrl = required('instance URL', args[1]); const code = required('one-time pairing code', args[2]); @@ -117,7 +127,7 @@ async function relay(): Promise { await handle.close(); } -async function run(runtimeSessionId?: string): Promise { +async function run(runtimeSessionId?: string, args: string[] = []): Promise { const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); @@ -187,6 +197,29 @@ async function run(runtimeSessionId?: string): Promise { runtimeMode === 'docker-macos-nsjail' && runtimeSessionId == null && (fileRelayUpstream?.length ?? 0) > 0; + const workerDirectory = + runtimeSessionId == null + ? option(args, '--worker-dir') ?? + process.env.LIBRECHAT_CODE_WORKER_DIR?.trim() + : undefined; + const workspaceId = + option(args, '--workspace-id') ?? + process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? + 'primary'; + const workspaceTools = workerDirectory + ? await LocalWorkspaceTools.create({ + workspaces: [ + { + id: workspaceId, + name: + option(args, '--workspace-name') ?? + process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? + defaultWorkspaceName(workerDirectory, workspaceId), + root: workerDirectory, + }, + ], + }) + : undefined; const capabilities = { statefulWorkspace, sandboxProfile: @@ -195,6 +228,7 @@ async function run(runtimeSessionId?: string): Promise { runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), policyDigest: createHash('sha256').update(policy).digest('hex'), ...(fileRelayEnabled ? { requiresReadyConfirmation: true } : {}), + ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { throw new Error( @@ -330,6 +364,7 @@ async function run(runtimeSessionId?: string): Promise { statefulWorkspace, }), capabilities, + workspaceTools, onIdentityChange: pairedIdentity && identityPath ? async (identity) => { @@ -401,7 +436,7 @@ async function main(): Promise { if (args[0] && args[0] !== 'run') { throw new Error(`Unknown command: ${args[0]}`); } - await run(); + await run(undefined, args.slice(1)); } main().catch((error: Error) => { process.stderr.write(`librechat-code: ${error.message}\n`); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 8a069c36..94d5d0e1 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -1,3 +1,5 @@ +import type { WorkspaceToolRequest } from './workspace.js'; + export const BRIDGE_PROTOCOL_VERSION = 1 as const; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; export const BRIDGE_SANDBOX_PROFILE_MAX_LENGTH = 128; @@ -77,7 +79,8 @@ export interface BridgeAssignment { /** Server-calculated execution budget at lease time; avoids VM clock skew. */ remainingMs?: number; runtimeSessionId?: string; - request: BridgeSandboxRequest; + executionKind?: 'sandbox' | 'workspace_tool'; + request: BridgeSandboxRequest | WorkspaceToolRequest; } export interface BridgeLeaseResponse { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 81a2d91e..302b290f 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,10 +7,12 @@ import { } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; +import { isWorkspaceToolRequest } from './workspace.js'; import type { BridgeAssignment, BridgeLeaseResponse, + BridgeSandboxRequest, BridgeSettlement, BridgeSettlementResponse, BridgeWorkerCapabilities, @@ -18,6 +20,7 @@ import type { BridgeWorkerRegistrationResponse, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; export interface BridgeWorkerOptions { codeApiUrl: string; @@ -28,6 +31,7 @@ export interface BridgeWorkerOptions { sandboxEndpoint?: string; runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; + workspaceTools?: WorkspaceToolExecutor; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -114,6 +118,25 @@ function errorCode(value: object): string | undefined { return undefined; } +function workspaceCapabilitiesMatch( + advertised: NonNullable, + executor: NonNullable, +): boolean { + return ( + advertised.protocolVersion === executor.protocolVersion && + advertised.operations.length === executor.operations.length && + advertised.operations.every( + (operation, index) => operation === executor.operations[index], + ) && + advertised.workspaces.length === executor.workspaces.length && + advertised.workspaces.every( + (workspace, index) => + workspace.id === executor.workspaces[index]?.id && + workspace.name === executor.workspaces[index]?.name, + ) + ); +} + export class BridgeWorkspaceQuarantinedError extends Error { constructor( message: string, @@ -147,6 +170,20 @@ export class BridgeWorker { if (options.runtimeSupervisor == null && !options.sandboxEndpoint?.trim()) { throw new BridgeProtocolError('Bridge worker requires a runtime supervisor'); } + if ( + (options.workspaceTools == null) !== + (options.capabilities.workspaceTools == null) || + (options.workspaceTools != null && + options.capabilities.workspaceTools != null && + !workspaceCapabilitiesMatch( + options.capabilities.workspaceTools, + options.workspaceTools.capabilities, + )) + ) { + throw new BridgeProtocolError( + 'Workspace tool capabilities require a matching executor', + ); + } this.fetchImpl = options.fetchImpl ?? fetch; this.codeApiUrl = normalizedBaseUrl(options.codeApiUrl); this.runtimeSupervisor = @@ -622,57 +659,114 @@ export class BridgeWorker { credentialMaintenanceError = error; executionController.abort(); }); - runtimeLease = await this.runtimeSupervisor.acquire( - assignment, - executionController.signal, - ); + let payload: object = {}; + if (assignment.executionKind === 'workspace_tool') { + if (this.options.workspaceTools == null) { + throw new BridgeProtocolError( + 'Worker does not provide local workspace tools', + ); + } + if (!isWorkspaceToolRequest(assignment.request)) { + throw new BridgeProtocolError('Invalid workspace tool request'); + } + const workspaceRequest = assignment.request; + const advertised = this.options.workspaceTools.capabilities; + if (!advertised.operations.includes(workspaceRequest.operation)) { + throw new BridgeProtocolError( + 'Workspace tool operation is not advertised', + ); + } + if ( + !advertised.workspaces.some( + (workspace) => workspace.id === workspaceRequest.workspaceId, + ) + ) { + throw new BridgeProtocolError('Workspace is not advertised'); + } + payload = await this.options.workspaceTools.execute( + workspaceRequest, + executionController.signal, + ); + if (executionController.signal.aborted) { + throw ( + executionController.signal.reason ?? + new DOMException('aborted', 'AbortError') + ); + } + if (Date.now() >= localDeadlineAtMs) { + throw new BridgeProtocolError( + 'Bridge assignment expired during workspace execution', + ); + } + } else { + runtimeLease = await this.runtimeSupervisor.acquire( + assignment, + executionController.signal, + ); + if (executionController.signal.aborted) { + throw ( + executionController.signal.reason ?? + new DOMException('aborted', 'AbortError') + ); + } + const sandboxRequest = assignment.request as BridgeSandboxRequest; + const headers = { + ...sandboxRequest.headers, + ...(runtimeLease.sessionId + ? { 'X-Runtime-Session-Id': runtimeLease.sessionId } + : {}), + }; + const sandboxRequestBody = JSON.stringify(sandboxRequest.body); + if (Date.now() >= localDeadlineAtMs) { + throw new BridgeProtocolError( + 'Bridge assignment expired before sandbox execution', + ); + } + sandboxStarted = true; + const response = await this.executeRuntime( + runtimeLease, + sandboxRequestBody, + { + ...headers, + 'Content-Type': 'application/json', + }, + executionController.signal, + ); + try { + payload = JSON.parse(response.body) as object; + } catch (error) { + if (response.status >= 200 && response.status < 300) throw error; + } + if (response.status < 200 || response.status >= 300) { + sandboxRejectedExecution = + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 && + errorMessage(payload) !== 'session_workspace_dirty'; + throw new BridgeProtocolError( + errorMessage(payload) ?? + `Sandbox rejected execution with HTTP ${response.status}`, + response.status, + ); + } + } + cancellationController.abort(); + await cancellationWatcher; if (executionController.signal.aborted) { - throw executionController.signal.reason ?? new DOMException('aborted', 'AbortError'); + throw ( + executionController.signal.reason ?? + new DOMException('aborted', 'AbortError') + ); } - const headers = { - ...assignment.request.headers, - ...(runtimeLease.sessionId - ? { 'X-Runtime-Session-Id': runtimeLease.sessionId } - : {}), - }; - const sandboxRequestBody = JSON.stringify(assignment.request.body); if (Date.now() >= localDeadlineAtMs) { throw new BridgeProtocolError( - 'Bridge assignment expired before sandbox execution', + 'Bridge assignment expired while draining cancellation', ); } - sandboxStarted = true; - const response = await this.executeRuntime( - runtimeLease, - sandboxRequestBody, - { - ...headers, - 'Content-Type': 'application/json', - }, - executionController.signal, - ); - let payload: object = {}; - try { - payload = JSON.parse(response.body) as object; - } catch (error) { - if (response.status >= 200 && response.status < 300) throw error; - } if (credentialMaintenanceError != null) { throw credentialMaintenanceError; } - if (response.status < 200 || response.status >= 300) { - sandboxRejectedExecution = - response.status >= 400 && - response.status < 500 && - response.status !== 408 && - response.status !== 429 && - errorMessage(payload) !== 'session_workspace_dirty'; - throw new BridgeProtocolError( - errorMessage(payload) ?? - `Sandbox rejected execution with HTTP ${response.status}`, - response.status, - ); - } if (heartbeatError != null) throw heartbeatError; settlement = { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -1005,7 +1099,9 @@ export class BridgeWorker { if (signal.aborted || executionController.signal.aborted) return; const pollController = new AbortController(); const abortPoll = (): void => pollController.abort(); - signal.addEventListener('abort', abortPoll, { once: true }); + executionController.signal.addEventListener('abort', abortPoll, { + once: true, + }); const timeout = setTimeout( abortPoll, Math.max( @@ -1028,14 +1124,14 @@ export class BridgeWorker { return; } } catch (error) { - if (signal.aborted) return; if (error instanceof BridgeProtocolError && error.status === 404) { executionController.abort(); return; } + if (signal.aborted) return; } finally { clearTimeout(timeout); - signal.removeEventListener('abort', abortPoll); + executionController.signal.removeEventListener('abort', abortPoll); } } } diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts new file mode 100644 index 00000000..4ac486ee --- /dev/null +++ b/packages/code/src/workspace-cli.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, mkdir, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +test('CLI validates a configured worker directory before registration', () => { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + '/definitely/missing/librechat-code-workspace', + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /invalid workspace registration/i); +}); + +test('CLI falls back to the workspace ID when the directory basename is invalid', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); + const workspaceRoot = join(root, ' '); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + let resolveRegistration: ((value: Record) => void) | undefined; + const registration = new Promise>((resolve) => { + resolveRegistration = resolve; + }); + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer) => chunks.push(chunk)); + request.on('end', () => { + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; + if (request.url?.endsWith('/bridge/workers/register')) { + resolveRegistration?.(body); + response.setHeader('Content-Type', 'application/json'); + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + incarnationId: body.incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + ); + return; + } + response.setHeader('Content-Type', 'application/json'); + response.end(JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0 })); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => server.close()); + const address = server.address(); + if (address == null || typeof address === 'string') { + assert.fail('expected TCP listener'); + } + + const child = spawn( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--workspace-id', + 'root-workspace', + ], + { + env: { + ...process.env, + LIBRECHAT_CODE_URL: `http://127.0.0.1:${address.port}/v1`, + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_SANDBOX_ENDPOINT: + 'http://127.0.0.1:2000/api/v2', + }, + stdio: 'ignore', + }, + ); + t.after(() => child.kill()); + + const body = await Promise.race([ + registration, + new Promise((_, reject) => + setTimeout(() => reject(new Error('registration timed out')), 2_000), + ), + ]); + child.kill(); + await once(child, 'exit'); + + assert.deepEqual( + (body.capabilities as Record).workspaceTools, + { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'root-workspace', name: 'root-workspace' }], + }, + ); +}); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts new file mode 100644 index 00000000..e206311e --- /dev/null +++ b/packages/code/src/workspace-worker.test.ts @@ -0,0 +1,684 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { BridgeWorker } from './worker.js'; + +const incarnationId = 'incarnation-00000001'; + +test('worker executes a workspace tool assignment locally without acquiring a sandbox', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const workspaceRequests: object[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + runtimeSupervisor: { + async acquire() { + throw new Error('workspace tools must not acquire a sandbox'); + }, + async reset() {}, + async quarantine() {}, + }, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + }, + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + async execute(request) { + workspaceRequests.push(request); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + fetchImpl: async (input, init) => { + requests.push({ url: String(input), init }); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-1', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + assert.deepEqual(workspaceRequests, [ + { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + ]); + assert.equal(requests.length, 1); + assert.deepEqual(JSON.parse(String(requests[0].init?.body)), { + protocolVersion: 1, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + incarnationId, + status: 'fulfilled', + result: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }, + }); +}); + +test('worker refuses to advertise workspace tools without a matching executor', () => { + assert.throws( + () => + new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }), + /workspace tool capabilities require a matching executor/i, + ); +}); + +test('worker compares workspace capabilities structurally', () => { + assert.doesNotThrow( + () => + new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + }, + workspaceTools: { + capabilities: { + operations: ['read_file'], + workspaces: [{ name: 'LibreChat', id: 'primary' }], + protocolVersion: 1, + }, + async execute() { + throw new Error('not executed'); + }, + }, + }), + ); +}); + +test('worker rejects a workspace result returned after its deadline', async () => { + const settlements: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request, signal) { + await new Promise((resolve) => { + if (signal?.aborted) return resolve(); + signal?.addEventListener('abort', () => resolve(), { once: true }); + }); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: request.workspaceId, + path: 'README.md', + content: '# late', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + fetchImpl: async (_input, init) => { + if (init?.body != null) { + settlements.push( + JSON.parse(String(init.body)) as Record, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-deadline', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 20).toISOString(), + remainingMs: 20, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.status, 'rejected'); + assert.match(String(settlements[0]?.error), /aborted|expired/i); +}); + +test('worker drains a completed cancellation poll before fulfilling workspace work', async () => { + const settlements: Array> = []; + let finishExecution: (() => void) | undefined; + let finishCancellation: (() => void) | undefined; + let markPollStarted: (() => void) | undefined; + const pollStarted = new Promise((resolve) => { + markPollStarted = resolve; + }); + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + await new Promise((resolve) => { + finishExecution = resolve; + }); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: request.workspaceId, + path: 'README.md', + content: '# cancelled', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + cancellationPollIntervalMs: 1, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + markPollStarted?.(); + return await new Promise((resolve) => { + finishCancellation = () => + resolve(Response.json({ protocolVersion: 1, cancelled: true })); + }); + } + if (String(input).endsWith('/settle') && init?.body != null) { + settlements.push( + JSON.parse(String(init.body)) as Record, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + const completion = worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-cancelled', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + await pollStarted; + finishExecution?.(); + finishCancellation?.(); + await completion; + + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.status, 'rejected'); + assert.match(String(settlements[0]?.error), /aborted/i); +}); + +test('worker drains a cancellation response body before fulfilling workspace work', async () => { + const settlements: Array> = []; + let finishExecution: (() => void) | undefined; + let markHeadersReceived: (() => void) | undefined; + const headersReceived = new Promise((resolve) => { + markHeadersReceived = resolve; + }); + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + await new Promise((resolve) => { + finishExecution = resolve; + }); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: request.workspaceId, + path: 'README.md', + content: '# cancelled', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + cancellationPollIntervalMs: 1, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + const response = new Response( + new ReadableStream({ + start(controller) { + let finished = false; + init?.signal?.addEventListener( + 'abort', + () => { + if (finished) return; + finished = true; + controller.error(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + setTimeout(() => { + if (!init?.signal?.aborted && !finished) { + finished = true; + controller.enqueue( + new TextEncoder().encode( + JSON.stringify({ protocolVersion: 1, cancelled: true }), + ), + ); + controller.close(); + } + }, 0); + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ); + markHeadersReceived?.(); + return response; + } + if (String(input).endsWith('/settle') && init?.body != null) { + settlements.push( + JSON.parse(String(init.body)) as Record, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + const completion = worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-cancelled-body', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + await headersReceived; + finishExecution?.(); + await completion; + + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.status, 'rejected'); + assert.match(String(settlements[0]?.error), /aborted/i); +}); + +test('worker rechecks its deadline after draining cancellation', async () => { + const settlements: Array> = []; + let finishExecution: (() => void) | undefined; + let releaseBody: (() => void) | undefined; + let markHeadersReceived: (() => void) | undefined; + const headersReceived = new Promise((resolve) => { + markHeadersReceived = resolve; + }); + const bodyReleased = new Promise((resolve) => { + releaseBody = resolve; + }); + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + await new Promise((resolve) => { + finishExecution = resolve; + }); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: request.workspaceId, + path: 'README.md', + content: '# late', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + cancellationPollIntervalMs: 1, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + markHeadersReceived?.(); + return { + ok: true, + status: 200, + async json() { + await bodyReleased; + const blockedUntil = Date.now() + 60; + while (Date.now() < blockedUntil) { + // Model synchronous body parsing that crosses the deadline. + } + return { protocolVersion: 1, cancelled: false }; + }, + } as Response; + } + if (String(input).endsWith('/settle') && init?.body != null) { + settlements.push( + JSON.parse(String(init.body)) as Record, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + const completion = worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-drain-deadline', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 50).toISOString(), + remainingMs: 50, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + await headersReceived; + finishExecution?.(); + await new Promise((resolve) => setImmediate(resolve)); + releaseBody?.(); + await completion; + + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.status, 'rejected'); + assert.match(String(settlements[0]?.error), /expired/i); +}); + +test('worker preserves a drained 404 cancellation response', async () => { + const settlements: Array> = []; + let finishExecution: (() => void) | undefined; + let releaseBody: (() => void) | undefined; + let markHeadersReceived: (() => void) | undefined; + const headersReceived = new Promise((resolve) => { + markHeadersReceived = resolve; + }); + const bodyReleased = new Promise((resolve) => { + releaseBody = resolve; + }); + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + await new Promise((resolve) => { + finishExecution = resolve; + }); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: request.workspaceId, + path: 'README.md', + content: '# cancelled', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + cancellationPollIntervalMs: 1, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + markHeadersReceived?.(); + return { + ok: false, + status: 404, + async json() { + await bodyReleased; + return {}; + }, + } as Response; + } + if (String(input).endsWith('/settle') && init?.body != null) { + settlements.push( + JSON.parse(String(init.body)) as Record, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + const completion = worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-cancelled-404', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 5_000, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + await headersReceived; + finishExecution?.(); + await new Promise((resolve) => setImmediate(resolve)); + releaseBody?.(); + await completion; + + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.status, 'rejected'); + assert.match(String(settlements[0]?.error), /aborted/i); +}); + +test('worker rejects workspace operations outside its advertised capability', async () => { + let executions = 0; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + executions += 1; + throw new Error('must not execute'); + }, + }, + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-1', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }, + }); + + assert.equal(executions, 0); + assert.equal(settlement?.status, 'rejected'); + assert.match(String(settlement?.error), /operation is not advertised/i); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index b56179c8..bde330aa 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -27,6 +27,14 @@ export interface LocalWorkspaceToolsOptions { workspaces: LocalWorkspaceConfig[]; } +export interface WorkspaceToolExecutor { + capabilities: BridgeWorkspaceToolCapabilities; + execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise; +} + export interface WorkspaceReadFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'read_file'; @@ -535,7 +543,7 @@ async function searchWorkspace( }; } -export class LocalWorkspaceTools { +export class LocalWorkspaceTools implements WorkspaceToolExecutor { readonly capabilities: BridgeWorkspaceToolCapabilities; private constructor( From 0498f7d67cc6c2e6fc5e86b2ec0666036d850afb Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 20:06:04 -0400 Subject: [PATCH 024/116] =?UTF-8?q?=F0=9F=8E=96=EF=B8=8F=20feat:=20Expose?= =?UTF-8?q?=20Authenticated=20Workspace=20Tool=20API=20(#90)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): add workspace tool API * docs(code): clarify repository-optional workspaces * fix(code): validate workspace API boundaries * fix(code): Harden workspace API validation * fix(code): Preserve workspace search scope * fix(code): close workspace API lifecycle gaps --- docs/remote-bridge/README.md | 46 ++++ packages/code/README.md | 12 +- packages/code/src/protocol.ts | 293 ++++++++++++++++++++- packages/code/src/worker.ts | 6 +- packages/code/src/workspace-worker.test.ts | 56 ++++ packages/code/src/workspace.test.ts | 179 ++++++++++++- packages/code/src/workspace.ts | 192 ++++++-------- service/src/api-server.ts | 2 + service/src/bridge/router.ts | 8 +- service/src/bridge/store.ts | 100 ++++++- service/src/bridge/workspace-store.test.ts | 159 +++++++++++ service/src/local-api.ts | 2 + service/src/service-api.ts | 2 + service/src/workspace-tools/index.ts | 20 ++ service/src/workspace-tools/router.test.ts | 203 ++++++++++++++ service/src/workspace-tools/router.ts | 130 +++++++++ 16 files changed, 1271 insertions(+), 139 deletions(-) create mode 100644 service/src/bridge/workspace-store.test.ts create mode 100644 service/src/workspace-tools/index.ts create mode 100644 service/src/workspace-tools/router.test.ts create mode 100644 service/src/workspace-tools/router.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 8d10f024..5fbb5b37 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -86,6 +86,52 @@ locally, proves possession on every request, and rotates its short-lived credential before expiry. `CODEAPI_BRIDGE_AUTH_MODE=static` remains available for non-hardened development compatibility only. +To expose an existing checkout as a worker-local workspace, start the CLI with +an explicit directory and logical ID: + +```bash +librechat-code run \ + --worker-dir /srv/checkouts/librechat \ + --workspace-id primary \ + --workspace-name LibreChat +``` + +The worker advertises only the workspace ID, optional display name, and +supported operations. Its host path is never registered with Code API. An +authenticated caller can execute the initial read-only operations through: + +```bash +curl -fsS https://code.example.com/v1/workspace-tools/execute \ + -H "Authorization: Bearer $LIBRECHAT_JWT" \ + -H 'Content-Type: application/json' \ + --data '{ + "protocolVersion":1, + "operation":"read_file", + "workspaceId":"primary", + "path":"README.md", + "startLine":1, + "maxLines":200 + }' +``` + +The endpoint uses the same authenticated principal-bound worker selection, +tenant fence, lease deadline, cancellation, and settlement lifecycle as remote +sandbox execution. Requests must name a workspace and operation advertised by +that worker. Results are validated against the originating request before they +leave Code API, and are bounded to 1 MiB/500 lines for reads or 200 matches for +searches. Absolute paths, traversal, backslashes, symlink escapes, unexpected +fields, and host roots are rejected. + +The workspace root can be an existing project, a Git repository, or an empty +directory; Git is not required. This boundary keeps that directory local to the +operator's machine, but the selected file contents, search matches, and later +tool results necessarily cross the outbound bridge to Code API and the model. +Treat them as explicit tool outputs, apply the same retention and audit policy +as chat content, and do not register a directory containing secrets. The +default operations are read-only; future mutation and shell operations must be +gated by LibreChat's tool-approval hooks in addition to worker capability +checks. + Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, persistent local runner per session. A single sandbox endpoint is stateless and diff --git a/packages/code/README.md b/packages/code/README.md index 499f85cb..076f831d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -191,7 +191,9 @@ API did not commit. ## Local workspace tools (bridge preview) `@librechat/code/workspace` provides the provider-neutral foundation for -coding-agent access to repositories that already live on the worker machine. +coding-agent access to workspace directories on the worker machine. A workspace +may be an existing project, a Git repository, or a newly created empty +directory; Git is optional. `LocalWorkspaceTools` registers opaque workspace IDs with optional display names and exposes bounded `read_file` and literal `search_text` operations. Only IDs, names, protocol version, and supported operations appear in worker @@ -207,11 +209,11 @@ and stops after a bounded global result count. The worker process still belongs inside the trusted BYOM boundary and should receive filesystem access only to roots the operator intentionally registers. -Register one repository already present on the worker machine with the -Cursor-style worker-directory option: +Register one directory already present on the worker machine with the +worker-directory option: ```bash -librechat-code run --worker-dir /path/to/repository +librechat-code run --worker-dir /path/to/workspace ``` The default public workspace ID is `primary` and the default display name is @@ -223,7 +225,7 @@ explicitly. `rg` must be installed on the worker for `search_text`. The worker advertises these capabilities only when a directory is configured and executes matching assignments under the bridge's existing lease, deadline, cancellation, credential-refresh, and settlement fencing. The -repository itself remains on the worker. As with Cursor's self-hosted agents, +workspace itself remains on the worker. As with Cursor's self-hosted agents, text deliberately selected by `read_file` or `search_text` crosses the outbound bridge so the remote agent/model can reason over it. Host paths are never part of that payload. The Code API workspace-tool endpoint is delivered as a diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 94d5d0e1..286b8125 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -1,5 +1,3 @@ -import type { WorkspaceToolRequest } from './workspace.js'; - export const BRIDGE_PROTOCOL_VERSION = 1 as const; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; export const BRIDGE_SANDBOX_PROFILE_MAX_LENGTH = 128; @@ -7,6 +5,12 @@ export const BRIDGE_RUNTIME_MAX_COUNT = 32; export const BRIDGE_RUNTIME_MAX_LENGTH = 64; export const BRIDGE_WORKSPACE_MAX_COUNT = 32; export const BRIDGE_WORKSPACE_NAME_MAX_LENGTH = 128; +export const BRIDGE_WORKSPACE_PATH_MAX_LENGTH = 4096; +export const BRIDGE_WORKSPACE_QUERY_MAX_LENGTH = 4096; +export const BRIDGE_WORKSPACE_READ_MAX_BYTES = 1024 * 1024; +export const BRIDGE_WORKSPACE_READ_MAX_LINES = 500; +export const BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS = 200; +export const BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH = 2000; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; @@ -23,6 +27,99 @@ export interface BridgeWorkspaceToolCapabilities { workspaces: BridgeWorkspaceDescriptor[]; } +export interface WorkspaceReadFileRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'read_file'; + workspaceId: string; + path: string; + startLine?: number; + maxLines?: number; +} + +export interface WorkspaceReadFileResult { + protocolVersion: BridgeProtocolVersion; + operation: 'read_file'; + workspaceId: string; + path: string; + content: string; + startLine: number; + endLine: number; + truncated: boolean; + nextStartLine?: number; +} + +export interface WorkspaceSearchTextRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'search_text'; + workspaceId: string; + query: string; + path?: string; + maxResults?: number; +} + +export interface WorkspaceSearchMatch { + path: string; + line: number; + column: number; + text: string; +} + +export interface WorkspaceSearchTextResult { + protocolVersion: BridgeProtocolVersion; + operation: 'search_text'; + workspaceId: string; + matches: WorkspaceSearchMatch[]; + truncated: boolean; +} + +export type WorkspaceToolRequest = + | WorkspaceReadFileRequest + | WorkspaceSearchTextRequest; +export type WorkspaceToolResult = + | WorkspaceReadFileResult + | WorkspaceSearchTextResult; + +const WORKSPACE_READ_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'startLine', + 'maxLines', +]); +const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'query', + 'path', + 'maxResults', +]); +const WORKSPACE_READ_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'content', + 'startLine', + 'endLine', + 'truncated', + 'nextStartLine', +]); +const WORKSPACE_SEARCH_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'matches', + 'truncated', +]); +const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ + 'path', + 'line', + 'column', + 'text', +]); + export interface BridgeWorkerCapabilities { statefulWorkspace: boolean; sandboxProfile: string; @@ -106,6 +203,35 @@ export interface BridgeRejectedSettlement { incarnationId: string; status: 'rejected'; error: string; + errorCode?: WorkspaceToolErrorCode; +} + +export type WorkspaceToolErrorCode = + | 'INVALID_PATH' + | 'INVALID_REQUEST' + | 'READ_LIMIT_EXCEEDED' + | 'REGISTRATION_INVALID' + | 'EXECUTION_ABORTED' + | 'SEARCH_TIMEOUT' + | 'SEARCH_UNAVAILABLE'; + +const WORKSPACE_TOOL_ERROR_CODES = new Set([ + 'INVALID_PATH', + 'INVALID_REQUEST', + 'READ_LIMIT_EXCEEDED', + 'REGISTRATION_INVALID', + 'EXECUTION_ABORTED', + 'SEARCH_TIMEOUT', + 'SEARCH_UNAVAILABLE', +]); + +export function isWorkspaceToolErrorCode( + value: unknown, +): value is WorkspaceToolErrorCode { + return ( + typeof value === 'string' && + WORKSPACE_TOOL_ERROR_CODES.has(value as WorkspaceToolErrorCode) + ); } export type BridgeSettlement = @@ -140,6 +266,169 @@ export function isValidBridgeWorkerId(workerId: string): boolean { return BRIDGE_WORKER_ID_PATTERN.test(workerId); } +function isSafePortableRelativePath(value: unknown): value is string { + if ( + typeof value !== 'string' || + value.length === 0 || + value.length > BRIDGE_WORKSPACE_PATH_MAX_LENGTH || + Buffer.from(value).toString('utf8') !== value || + value.includes('\0') || + value.includes('\\') || + value.startsWith('/') || + /^[A-Za-z]:/.test(value) + ) { + return false; + } + return value.split('/').every((segment) => segment !== '..'); +} + +function normalizePortableRelativePath(value: string): string { + return ( + value + .split('/') + .filter((segment) => segment.length > 0 && segment !== '.') + .join('/') || '.' + ); +} + +function isWithinRequestedPath(candidate: string, requested?: string): boolean { + if (requested == null) return true; + const normalizedCandidate = normalizePortableRelativePath(candidate); + const normalizedRequested = normalizePortableRelativePath(requested); + return ( + normalizedRequested === '.' || + normalizedCandidate === normalizedRequested || + normalizedCandidate.startsWith(`${normalizedRequested}/`) + ); +} + +function hasOnlyKeys( + value: Record, + allowed: ReadonlySet, +): boolean { + return Object.keys(value).every((key) => allowed.has(key)); +} + +export function isWorkspaceToolRequest( + value: unknown, +): value is WorkspaceToolRequest { + if (typeof value !== 'object' || value === null) return false; + const request = value as Record; + if ( + request.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + typeof request.workspaceId !== 'string' || + !isValidBridgeWorkerId(request.workspaceId) + ) { + return false; + } + if (request.operation === 'read_file') { + return ( + hasOnlyKeys(request, WORKSPACE_READ_REQUEST_KEYS) && + isSafePortableRelativePath(request.path) && + (request.startLine === undefined || + (Number.isSafeInteger(request.startLine) && + Number(request.startLine) >= 1)) && + (request.maxLines === undefined || + (Number.isSafeInteger(request.maxLines) && + Number(request.maxLines) >= 1 && + Number(request.maxLines) <= BRIDGE_WORKSPACE_READ_MAX_LINES)) + ); + } + if (request.operation === 'search_text') { + return ( + hasOnlyKeys(request, WORKSPACE_SEARCH_REQUEST_KEYS) && + typeof request.query === 'string' && + request.query.length > 0 && + request.query.length <= BRIDGE_WORKSPACE_QUERY_MAX_LENGTH && + Buffer.from(request.query).toString('utf8') === request.query && + new TextEncoder().encode(request.query).byteLength <= + BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && + !request.query.includes('\0') && + !request.query.includes('\n') && + !request.query.includes('\r') && + (request.path === undefined || + isSafePortableRelativePath(request.path)) && + (request.maxResults === undefined || + (Number.isSafeInteger(request.maxResults) && + Number(request.maxResults) >= 1 && + Number(request.maxResults) <= BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) + ); + } + return false; +} + +export function isWorkspaceToolResult( + request: WorkspaceToolRequest, + value: unknown, +): value is WorkspaceToolResult { + if (typeof value !== 'object' || value === null) return false; + const result = value as Record; + if ( + result.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + result.operation !== request.operation || + result.workspaceId !== request.workspaceId || + typeof result.truncated !== 'boolean' + ) { + return false; + } + + if (request.operation === 'read_file') { + const startLine = request.startLine ?? 1; + const maxLines = request.maxLines ?? 200; + const content = typeof result.content === 'string' ? result.content : null; + const reportedLineCount = + Number.isSafeInteger(result.endLine) && Number(result.endLine) >= startLine - 1 + ? Number(result.endLine) - startLine + 1 + : -1; + const actualLineCount = + content === null ? -1 : content.length === 0 ? reportedLineCount : content.split('\n').length; + return ( + hasOnlyKeys(result, WORKSPACE_READ_RESULT_KEYS) && + result.path === request.path && + isSafePortableRelativePath(result.path) && + content !== null && + new TextEncoder().encode(content).byteLength <= + BRIDGE_WORKSPACE_READ_MAX_BYTES && + result.startLine === startLine && + Number.isSafeInteger(result.endLine) && + Number(result.endLine) >= startLine - 1 && + Number(result.endLine) < startLine + maxLines && + reportedLineCount >= 0 && + reportedLineCount <= maxLines && + (content.length !== 0 || reportedLineCount <= 1) && + actualLineCount === reportedLineCount && + (result.truncated === true + ? Number.isSafeInteger(result.nextStartLine) && + Number(result.nextStartLine) === Number(result.endLine) + 1 && + Number(result.nextStartLine) > startLine + : result.nextStartLine === undefined) + ); + } + + if (!Array.isArray(result.matches)) return false; + const maxResults = request.maxResults ?? 50; + return ( + hasOnlyKeys(result, WORKSPACE_SEARCH_RESULT_KEYS) && + result.matches.length <= maxResults && + result.matches.every((match) => { + if (typeof match !== 'object' || match === null) return false; + const candidate = match as Record; + return ( + hasOnlyKeys(candidate, WORKSPACE_SEARCH_MATCH_KEYS) && + isSafePortableRelativePath(candidate.path) && + isWithinRequestedPath(candidate.path, request.path) && + Number.isSafeInteger(candidate.line) && + Number(candidate.line) >= 1 && + Number.isSafeInteger(candidate.column) && + Number(candidate.column) >= 1 && + typeof candidate.text === 'string' && + candidate.text.length <= BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && + candidate.text.includes(request.query) + ); + }) + ); +} + export function isValidBridgeWorkspaceToolCapabilities( value: unknown, ): value is BridgeWorkspaceToolCapabilities { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 302b290f..27526747 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,7 +7,7 @@ import { } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; -import { isWorkspaceToolRequest } from './workspace.js'; +import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; import type { BridgeAssignment, @@ -790,6 +790,10 @@ export class BridgeWorker { leaseToken: assignment.leaseToken, incarnationId: this.incarnationId, status: 'rejected', + ...(assignment.executionKind === 'workspace_tool' && + error instanceof WorkspaceToolError + ? { errorCode: error.code } + : {}), error: (error instanceof Error ? error.message diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index e206311e..352cbe33 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -2,9 +2,65 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { BridgeWorker } from './worker.js'; +import { WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('worker preserves bounded workspace rejection codes', async () => { + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['search_text' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new WorkspaceToolError( + 'Workspace search timed out', + 'SEARCH_TIMEOUT', + ); + }, + }, + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-timeout', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }, + }); + + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'SEARCH_TIMEOUT'); +}); + test('worker executes a workspace tool assignment locally without acquiring a sandbox', async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; const workspaceRequests: object[] = []; diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 0fd9ad23..e8ee53bf 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -6,7 +6,11 @@ import { join } from 'node:path'; import test from 'node:test'; import { promisify } from 'node:util'; -import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { + isWorkspaceToolResult, + LocalWorkspaceTools, + WorkspaceToolError, +} from './workspace.js'; const execFileAsync = promisify(execFile); @@ -67,7 +71,7 @@ test('rejects traversal outside a registered workspace without leaking its host }), (error: unknown) => { assert.ok(error instanceof Error); - assert.match(error.message, /invalid workspace path/i); + assert.match(error.message, /invalid workspace/i); assert.equal(error.message.includes(parent), false); return true; }, @@ -210,6 +214,35 @@ test('search does not read an explicitly targeted escaping symlink', async (t) = ); }); +test('search preserves an in-workspace symlink namespace in returned paths', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + await writeFile(join(root, 'src', 'app.ts'), 'const needle = true;'); + await symlink(join(root, 'src'), join(root, 'alias')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + path: 'alias', + }); + + if (result.operation !== 'search_text') assert.fail('expected search result'); + assert.deepEqual(result.matches, [ + { + path: 'alias/app.ts', + line: 1, + column: 7, + text: 'const needle = true;', + }, + ]); +}); + test('search returns a bounded match for a very long line', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -245,7 +278,8 @@ test('search rejects multiline literal queries', async (t) => { workspaceId: 'primary', query: 'first\nsecond', }), - /invalid workspace search/i, + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', ); }); @@ -263,7 +297,8 @@ test('search rejects queries larger than its bounded preview', async (t) => { workspaceId: 'primary', query: 'a'.repeat(2001), }), - /invalid workspace search/i, + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', ); }); @@ -513,6 +548,31 @@ test('bounds bytes read from a workspace file', async (t) => { ); }); +test('bounds workspace reads after UTF-16 decoding', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const utf16 = Buffer.from('\u4e00'.repeat(400_000), 'utf16le'); + await writeFile( + join(root, 'large-utf16.txt'), + Buffer.concat([Buffer.from([0xff, 0xfe]), utf16]), + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'large-utf16.txt', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'READ_LIMIT_EXCEEDED', + ); +}); + test('rejects ambiguous workspace registrations', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -575,3 +635,114 @@ test('does not start workspace I/O after its execution is aborted', async (t) => /workspace tool execution aborted/i, ); }); + +test('validates workspace results against the originating request', () => { + const request = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + path: 'README.md', + }; + const result = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }; + + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal( + isWorkspaceToolResult(request, { ...result, path: '/Users/operator/key' }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + root: '/Users/operator/private', + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + truncated: true, + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + content: '', + endLine: 0, + truncated: true, + nextStartLine: 1, + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + nextStartLine: 2, + }), + false, + ); + assert.equal( + isWorkspaceToolResult( + { ...request, maxLines: 1 }, + { ...result, content: 'first\nsecond' }, + ), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + matches: [ + { path: 'src/index.ts', line: 1, column: 1, text: 'unrelated' }, + ], + }), + false, + ); +}); + +test('validates search result paths against the requested scope', () => { + const request = { + protocolVersion: 1 as const, + operation: 'search_text' as const, + workspaceId: 'primary', + query: 'needle', + path: './src', + }; + const result = { + protocolVersion: 1 as const, + operation: 'search_text' as const, + workspaceId: 'primary', + matches: [ + { path: 'src/index.ts', line: 1, column: 1, text: 'needle' }, + ], + truncated: false, + }; + + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + matches: [ + { path: 'src-old/index.ts', line: 1, column: 1, text: 'needle' }, + ], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + matches: [ + { path: 'secrets.env', line: 1, column: 1, text: 'needle' }, + ], + }), + false, + ); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index bde330aa..3e6c27c0 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -7,16 +7,39 @@ import type { FileHandle } from 'node:fs/promises'; import { BRIDGE_PROTOCOL_VERSION, - isValidBridgeWorkerId, + BRIDGE_WORKSPACE_READ_MAX_BYTES, + BRIDGE_WORKSPACE_READ_MAX_LINES, + BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS, + BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, isValidBridgeWorkspaceToolCapabilities, + isWorkspaceToolRequest, + isWorkspaceToolResult, } from './protocol.js'; import type { - BridgeProtocolVersion, BridgeWorkspaceDescriptor, BridgeWorkspaceToolCapabilities, + WorkspaceReadFileRequest, + WorkspaceReadFileResult, + WorkspaceSearchMatch, + WorkspaceSearchTextRequest, + WorkspaceSearchTextResult, + WorkspaceToolRequest, + WorkspaceToolErrorCode, + WorkspaceToolResult, } from './protocol.js'; +export { isWorkspaceToolRequest, isWorkspaceToolResult }; +export type { + WorkspaceReadFileRequest, + WorkspaceReadFileResult, + WorkspaceSearchMatch, + WorkspaceSearchTextRequest, + WorkspaceSearchTextResult, + WorkspaceToolRequest, + WorkspaceToolResult, +}; + export interface LocalWorkspaceConfig { id: string; name?: string; @@ -35,59 +58,6 @@ export interface WorkspaceToolExecutor { ): Promise; } -export interface WorkspaceReadFileRequest { - protocolVersion: BridgeProtocolVersion; - operation: 'read_file'; - workspaceId: string; - path: string; - startLine?: number; - maxLines?: number; -} - -export interface WorkspaceReadFileResult { - protocolVersion: BridgeProtocolVersion; - operation: 'read_file'; - workspaceId: string; - path: string; - content: string; - startLine: number; - endLine: number; - truncated: boolean; - nextStartLine?: number; -} - -export interface WorkspaceSearchTextRequest { - protocolVersion: BridgeProtocolVersion; - operation: 'search_text'; - workspaceId: string; - query: string; - path?: string; - maxResults?: number; -} - -export interface WorkspaceSearchMatch { - path: string; - line: number; - column: number; - text: string; -} - -export interface WorkspaceSearchTextResult { - protocolVersion: BridgeProtocolVersion; - operation: 'search_text'; - workspaceId: string; - matches: WorkspaceSearchMatch[]; - truncated: boolean; -} - -export type WorkspaceToolRequest = - WorkspaceReadFileRequest | WorkspaceSearchTextRequest; -export type WorkspaceToolResult = - WorkspaceReadFileResult | WorkspaceSearchTextResult; - -const MAX_READ_LINES = 500; -const MAX_READ_BYTES = 1024 * 1024; -const MAX_SEARCH_PREVIEW_LENGTH = 2000; const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; @@ -140,60 +110,13 @@ function sliceWithoutSplittingSurrogates( export class WorkspaceToolError extends Error { constructor( message: string, - public readonly code: - | 'INVALID_PATH' - | 'INVALID_REQUEST' - | 'READ_LIMIT_EXCEEDED' - | 'REGISTRATION_INVALID' - | 'EXECUTION_ABORTED' - | 'SEARCH_TIMEOUT' - | 'SEARCH_UNAVAILABLE', + public readonly code: WorkspaceToolErrorCode, ) { super(message); this.name = 'WorkspaceToolError'; } } -export function isWorkspaceToolRequest( - value: unknown, -): value is WorkspaceToolRequest { - if (typeof value !== 'object' || value === null) return false; - const request = value as Record; - if ( - request.protocolVersion !== BRIDGE_PROTOCOL_VERSION || - typeof request.workspaceId !== 'string' || - !isValidBridgeWorkerId(request.workspaceId) - ) { - return false; - } - if (request.operation === 'read_file') { - return ( - typeof request.path === 'string' && - request.path.length > 0 && - request.path.length <= 4096 && - isUtf8ScalarString(request.path) && - (request.startLine === undefined || - Number.isSafeInteger(request.startLine)) && - (request.maxLines === undefined || Number.isSafeInteger(request.maxLines)) - ); - } - if (request.operation === 'search_text') { - return ( - typeof request.query === 'string' && - request.query.length > 0 && - request.query.length <= 4096 && - (request.path === undefined || - (typeof request.path === 'string' && - request.path.length > 0 && - request.path.length <= 4096 && - isUtf8ScalarString(request.path))) && - (request.maxResults === undefined || - Number.isSafeInteger(request.maxResults)) - ); - } - return false; -} - function isWithinRoot(root: string, candidate: string): boolean { const relativePath = relative(root, candidate); return !( @@ -243,15 +166,15 @@ async function readConfinedFileBuffer( ) { throw new Error('Invalid workspace path'); } - if (openedFile.size > MAX_READ_BYTES) { + if (openedFile.size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { throw new WorkspaceToolError( 'Workspace file exceeds read limit', 'READ_LIMIT_EXCEEDED', ); } - const buffer = Buffer.allocUnsafe(MAX_READ_BYTES + 1); + const buffer = Buffer.allocUnsafe(BRIDGE_WORKSPACE_READ_MAX_BYTES + 1); let bytesRead = 0; - while (bytesRead <= MAX_READ_BYTES) { + while (bytesRead <= BRIDGE_WORKSPACE_READ_MAX_BYTES) { const result = await handle.read( buffer, bytesRead, @@ -261,7 +184,7 @@ async function readConfinedFileBuffer( if (result.bytesRead === 0) break; bytesRead += result.bytesRead; } - if (bytesRead > MAX_READ_BYTES) { + if (bytesRead > BRIDGE_WORKSPACE_READ_MAX_BYTES) { throw new WorkspaceToolError( 'Workspace file exceeds read limit', 'READ_LIMIT_EXCEEDED', @@ -280,7 +203,16 @@ async function readConfinedFile( root: string, requestedPath: string, ): Promise { - return decodeWorkspaceText(await readConfinedFileBuffer(root, requestedPath)); + const decoded = decodeWorkspaceText( + await readConfinedFileBuffer(root, requestedPath), + ); + if (Buffer.byteLength(decoded, 'utf8') > BRIDGE_WORKSPACE_READ_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds read limit', + 'READ_LIMIT_EXCEEDED', + ); + } + return decoded; } interface SearchCandidates { @@ -304,6 +236,8 @@ async function listSearchCandidates( '--files', '--no-config', '--no-follow', + '--path-separator', + '/', '--null', '--max-filesize', '1M', @@ -414,7 +348,7 @@ async function searchWorkspace( if ( !request.query || request.query.length > 4096 || - encodedQuery.length > MAX_SEARCH_PREVIEW_LENGTH || + encodedQuery.length > BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH || encodedQuery.toString('utf8') !== request.query || request.query.includes('\0') || request.query.includes('\n') || @@ -423,7 +357,11 @@ async function searchWorkspace( throw new WorkspaceToolError('Invalid workspace search', 'INVALID_REQUEST'); } const maxResults = request.maxResults ?? 50; - if (!Number.isSafeInteger(maxResults) || maxResults < 1 || maxResults > 200) { + if ( + !Number.isSafeInteger(maxResults) || + maxResults < 1 || + maxResults > BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS + ) { throw new WorkspaceToolError('Invalid workspace search', 'INVALID_REQUEST'); } @@ -438,6 +376,12 @@ async function searchWorkspace( if (!isWithinRoot(root, canonicalTarget)) throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); const canonicalSearchPath = relative(root, canonicalTarget) || '.'; + const portableCanonicalSearchPath = canonicalSearchPath.split(sep).join('/'); + const normalizedRequestedResultPath = request.path + ?.split('/') + .filter((segment) => segment.length > 0 && segment !== '.') + .join('/'); + const requestedResultPath = normalizedRequestedResultPath || undefined; const deadline = Date.now() + SEARCH_TIMEOUT_MS; const candidates = await listSearchCandidates( @@ -461,9 +405,16 @@ async function searchWorkspace( 'SEARCH_TIMEOUT', ); } - const path = candidate.startsWith(`.${sep}`) - ? candidate.slice(2) - : candidate; + const path = candidate.startsWith('./') ? candidate.slice(2) : candidate; + const resultPath = + requestedResultPath == null + ? path + : portableCanonicalSearchPath === '.' + ? `${requestedResultPath}/${path}` + : path === portableCanonicalSearchPath || + path.startsWith(`${portableCanonicalSearchPath}/`) + ? `${requestedResultPath}${path.slice(portableCanonicalSearchPath.length)}` + : path; let content: Buffer; try { content = await readConfinedFileBuffer(root, path); @@ -511,19 +462,24 @@ async function searchWorkspace( 0, column - Math.floor( - (MAX_SEARCH_PREVIEW_LENGTH - request.query.length) / 2, + (BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH - + request.query.length) / + 2, ), ), - Math.max(0, line.length - MAX_SEARCH_PREVIEW_LENGTH), + Math.max( + 0, + line.length - BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, + ), ); matches.push({ - path, + path: resultPath, line: lineNumber, column: column + 1, text: sliceWithoutSplittingSurrogates( line, previewStart, - MAX_SEARCH_PREVIEW_LENGTH, + BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, ), }); } @@ -626,7 +582,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { startLine < 1 || !Number.isSafeInteger(maxLines) || maxLines < 1 || - maxLines > MAX_READ_LINES + maxLines > BRIDGE_WORKSPACE_READ_MAX_LINES ) { throw new WorkspaceToolError('Invalid workspace read', 'INVALID_REQUEST'); } diff --git a/service/src/api-server.ts b/service/src/api-server.ts index 1e4634a4..4a098576 100644 --- a/service/src/api-server.ts +++ b/service/src/api-server.ts @@ -19,6 +19,7 @@ import { localAuth } from './auth/local'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; import bridgeRouter from './bridge'; +import workspaceToolsRouter from './workspace-tools'; import { connection } from './queue'; import { metricsHandler } from './metrics'; import { httpMetricsMiddleware } from './middleware/httpMetrics'; @@ -55,6 +56,7 @@ app.get('/v1/health', async (_, res) => { v1.use('/bridge', bridgeRouter); v1.use(isLocalMode ? localAuth : apiKeyAuth); +v1.use(workspaceToolsRouter); v1.use(serviceRouter); v1.use(programmaticRouter); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 996dd16b..0905993a 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -10,6 +10,7 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceToolErrorCode, } from '../../../packages/code/src/protocol'; import { BridgePairingError, RedisBridgePairingStore } from './pairing'; import { BridgeStoreError, RedisBridgeStore } from './store'; @@ -113,7 +114,12 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement { return false; } if (value.status === 'rejected') { - return typeof value.error === 'string' && value.error.length <= 4096; + return ( + typeof value.error === 'string' && + value.error.length <= 4096 && + (value.errorCode === undefined || + isWorkspaceToolErrorCode(value.errorCode)) + ); } return ( value.status === 'fulfilled' && diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index afc8a72b..649224cf 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -6,9 +6,15 @@ import type { BridgeAssignment, BridgeSettlement, BridgeWorkerRegistration, + WorkspaceToolRequest, + WorkspaceToolResult, } from '../../../packages/code/src/protocol'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + isWorkspaceToolRequest, + isWorkspaceToolResult, +} from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; const PREFIX = 'codeapi:bridge:v1'; @@ -24,6 +30,10 @@ export type CodeBridgeSettlement = BridgeSettlement< run?: t.ExecuteResponse['run']; } >; +export type CodeBridgeWorkspaceSettlement = BridgeSettlement; +type AnyCodeBridgeSettlement = + | CodeBridgeSettlement + | CodeBridgeWorkspaceSettlement; export class BridgeStoreError extends Error { constructor( @@ -37,7 +47,9 @@ export class BridgeStoreError extends Error { | 'WORKER_FENCED' | 'WORKER_QUARANTINED' | 'WORKSPACE_QUARANTINED' - | 'WORKER_MISMATCH', + | 'WORKER_MISMATCH' + | 'ASSIGNMENT_INVALID' + | 'RESULT_INVALID', message: string, ) { super(message); @@ -56,6 +68,20 @@ export interface RegisteredBridgeWorker extends BridgeWorkerRegistration { identityId?: string; } +function supportsWorkspaceTool( + registration: RegisteredBridgeWorker, + request: WorkspaceToolRequest, +): boolean { + const capabilities = registration.capabilities.workspaceTools; + return ( + capabilities != null && + capabilities.operations.includes(request.operation) && + capabilities.workspaces.some( + (workspace) => workspace.id === request.workspaceId, + ) + ); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -442,12 +468,45 @@ export class RedisBridgeStore { } } + async dispatchWorkspaceTool(args: { + workerId: string; + tenantId?: string; + requireTenantBinding?: boolean; + request: WorkspaceToolRequest; + deadlineAtMs: number; + signal: AbortSignal; + }): Promise { + if (!isWorkspaceToolRequest(args.request)) { + throw new BridgeStoreError( + 'ASSIGNMENT_INVALID', + 'Invalid workspace tool request', + ); + } + const settlement = (await this.dispatch({ + ...args, + body: {} as t.PayloadBody, + headers: {}, + workspaceRequest: args.request, + })) as unknown as CodeBridgeWorkspaceSettlement; + if ( + settlement.status === 'fulfilled' && + !isWorkspaceToolResult(args.request, settlement.result) + ) { + throw new BridgeStoreError( + 'RESULT_INVALID', + 'Bridge worker returned an invalid workspace tool result', + ); + } + return settlement; + } + async dispatch(args: { workerId: string; tenantId?: string; requireTenantBinding?: boolean; body: t.PayloadBody; headers: Record; + workspaceRequest?: WorkspaceToolRequest; runtimeSessionId?: string; deadlineAtMs: number; signal: AbortSignal; @@ -489,6 +548,15 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not provide a stateful workspace`, ); } + if ( + args.workspaceRequest != null && + !supportsWorkspaceTool(registration, args.workspaceRequest) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} does not advertise the requested workspace tool`, + ); + } if ( args.runtimeSessionId !== undefined && (await this.dispatchCommand( @@ -549,10 +617,17 @@ export class RedisBridgeStore { : {}), expiresAt: new Date(args.deadlineAtMs).toISOString(), runtimeSessionId: args.runtimeSessionId, - request: { - body: args.body, - headers: args.headers, - }, + ...(args.workspaceRequest != null + ? { + executionKind: 'workspace_tool' as const, + request: args.workspaceRequest, + } + : { + request: { + body: args.body, + headers: args.headers, + }, + }), }; let queued = false; for (let attempt = 0; attempt < 8 && !queued; attempt += 1) { @@ -589,6 +664,15 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not provide a stateful workspace`, ); } + if ( + args.workspaceRequest != null && + !supportsWorkspaceTool(replacement.registration, args.workspaceRequest) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} no longer advertises the requested workspace tool`, + ); + } registration = replacement.registration; readyToken = replacement.readyToken; } @@ -948,7 +1032,7 @@ export class RedisBridgeStore { async settle( workerId: string, assignmentId: string, - settlement: CodeBridgeSettlement, + settlement: AnyCodeBridgeSettlement, signal?: AbortSignal, identityId?: string, ): Promise { @@ -1430,7 +1514,7 @@ export class RedisBridgeStore { private async commitPendingWorkspace( assignment: StoredAssignment, - settlement: CodeBridgeSettlement, + settlement: AnyCodeBridgeSettlement, deadlineAtMs: number, signal: AbortSignal, ): Promise { diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts new file mode 100644 index 00000000..182a2da7 --- /dev/null +++ b/service/src/bridge/workspace-store.test.ts @@ -0,0 +1,159 @@ +import { afterEach, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; + +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const store = new RedisBridgeStore(redis); +const incarnationId = 'incarnation-00000001'; + +afterEach(async () => { + await redis.flushall(); +}); + +test('dispatches a workspace tool only to a worker advertising its workspace and operation', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + }, + }); + const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file' as const, + workspaceId: 'primary', + path: 'README.md', + }; + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ + executionKind: 'workspace_tool', + request, + }); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }, + }); + + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { content: '# LibreChat' }, + }); +}); + +test('rejects a workspace tool that the selected worker did not advertise', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + +test('rejects a fulfilled workspace settlement that violates the result contract', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: 'safe', + startLine: 1, + endLine: 1, + truncated: false, + root: '/Users/operator/private', + } as never, + }); + + await expect(completion).rejects.toMatchObject({ + code: 'RESULT_INVALID', + }); +}); diff --git a/service/src/local-api.ts b/service/src/local-api.ts index df35cb5e..871bc356 100644 --- a/service/src/local-api.ts +++ b/service/src/local-api.ts @@ -11,6 +11,7 @@ import express, { json, Router } from 'express'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; import bridgeRouter from './bridge'; +import workspaceToolsRouter from './workspace-tools'; import { requestErrorLogger, requestNotFoundLogger } from './middleware/request-error-logger'; import { executionProfileMiddleware } from './middleware/execution-profile'; import { localAuth } from './auth/local'; @@ -52,6 +53,7 @@ app.get('/v1/health', async (_, res) => { v1.use('/bridge', bridgeRouter); v1.use(localAuth); +v1.use(workspaceToolsRouter); v1.use(serviceRouter); v1.use(programmaticRouter); app.use('/v1', v1); diff --git a/service/src/service-api.ts b/service/src/service-api.ts index 79db08d5..b5b7d52e 100644 --- a/service/src/service-api.ts +++ b/service/src/service-api.ts @@ -6,6 +6,7 @@ import { executionProfileMiddleware } from './middleware/execution-profile'; import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; import bridgeRouter from './bridge'; +import workspaceToolsRouter from './workspace-tools'; import { connection } from './queue'; import { env } from './config'; import logger from './logger'; @@ -32,6 +33,7 @@ app.get('/v1/health', async (_, res) => { v1.use('/bridge', bridgeRouter); v1.use(apiKeyAuth); +v1.use(workspaceToolsRouter); v1.use(serviceRouter); v1.use(programmaticRouter); diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts new file mode 100644 index 00000000..6f105f16 --- /dev/null +++ b/service/src/workspace-tools/index.ts @@ -0,0 +1,20 @@ +import { Router } from 'express'; + +import { bridgeStore } from '../bridge'; +import { env } from '../config'; +import { executionLimiter } from '../middleware/limits'; +import { createWorkspaceToolsRouter } from './router'; + +const router = Router(); +router.use('/workspace-tools/execute', executionLimiter); +router.use( + createWorkspaceToolsRouter({ + store: bridgeStore, + backend: env.SANDBOX_BACKEND, + configuredWorkerId: env.BRIDGE_WORKER_ID, + dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + timeoutMs: env.JOB_TIMEOUT, + }), +); + +export default router; diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts new file mode 100644 index 00000000..aa9c2102 --- /dev/null +++ b/service/src/workspace-tools/router.test.ts @@ -0,0 +1,203 @@ +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; + +import { afterEach, expect, test } from 'bun:test'; +import express, { json } from 'express'; + +import { applyPrincipal } from '../auth/principal'; +import { BridgeStoreError } from '../bridge/store'; +import { bridgeStoreStatus, createWorkspaceToolsRouter } from './router'; + +let server: Server | undefined; + +afterEach(() => { + server?.close(); + server = undefined; +}); + +test('maps invalid worker results to an upstream failure', () => { + expect(bridgeStoreStatus(new BridgeStoreError('RESULT_INVALID', 'invalid worker result'))).toBe(502); +}); + +test('rejects new workspace dispatches while the service is shutting down', async () => { + let dispatched = false; + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { + userId: 'user-1', + tenantId: 'tenant-1', + principalSource: 'librechat_jwt', + codeWorkerId: 'user-worker', + }); + next(); + }); + app.use( + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'shared-worker', + dynamicWorkers: true, + isShuttingDown: () => true, + store: { + async dispatchWorkspaceTool() { + dispatched = true; + throw new Error('must not dispatch'); + }, + }, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }), + }); + + expect(response.status).toBe(503); + expect(dispatched).toBe(false); +}); + +test.each([ + ['SEARCH_TIMEOUT', 504], + ['SEARCH_UNAVAILABLE', 503], +] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { + userId: 'user-1', + tenantId: 'tenant-1', + principalSource: 'librechat_jwt', + codeWorkerId: 'user-worker', + }); + next(); + }); + app.use( + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'shared-worker', + dynamicWorkers: true, + store: { + async dispatchWorkspaceTool() { + return { + protocolVersion: 1, + generation: 1, + leaseToken: 'lease-token', + incarnationId: 'incarnation-1', + status: 'rejected', + error: 'search failed', + errorCode, + }; + }, + }, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }), + }); + + expect(response.status).toBe(expectedStatus); + await expect(response.json()).resolves.toMatchObject({ code: errorCode }); +}); + +test('dispatches an authenticated workspace tool request to the principal-bound worker', async () => { + let dispatchArgs: Record | undefined; + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { + userId: 'user-1', + tenantId: 'tenant-1', + principalSource: 'librechat_jwt', + codeWorkerId: 'user-worker', + }); + next(); + }); + app.use( + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'shared-worker', + dynamicWorkers: true, + store: { + async dispatchWorkspaceTool(args) { + dispatchArgs = args as unknown as Record; + return { + protocolVersion: 1, + generation: 1, + leaseToken: 'lease-token', + incarnationId: 'incarnation-1', + status: 'fulfilled', + result: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }, + }; + }, + }, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + + const request = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }; + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-LibreChat-Code-Worker-ID': 'user-worker', + }, + body: JSON.stringify(request), + }); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + operation: 'read_file', + content: '# LibreChat', + }); + expect(dispatchArgs).toMatchObject({ + workerId: 'user-worker', + tenantId: 'tenant-1', + requireTenantBinding: true, + request, + }); +}); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts new file mode 100644 index 00000000..13a4bd9d --- /dev/null +++ b/service/src/workspace-tools/router.ts @@ -0,0 +1,130 @@ +import { Router } from 'express'; + +import type { RequestHandler, Response } from 'express'; +import type { AuthenticatedRequest } from '../types'; +import type { RedisBridgeStore } from '../bridge/store'; + +import { getPrincipalOrReject } from '../auth/principal'; +import { BridgeStoreError } from '../bridge/store'; +import { checkServiceShutDown } from '../lifecycle'; +import { isWorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import { + CODEAPI_BRIDGE_WORKER_HEADER, + BridgeWorkerSelectionError, + resolveBridgeWorkerSelection, +} from '../bridge/selection'; + +interface WorkspaceToolsRouterOptions { + store: Pick; + backend: 'http' | 'lambda-microvm' | 'remote-bridge'; + configuredWorkerId: string; + dynamicWorkers: boolean; + timeoutMs?: number; + isShuttingDown?: () => boolean; +} + +function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promise): RequestHandler { + return (req, res, next) => { + void handler(req as AuthenticatedRequest, res).catch(next); + }; +} + +export function bridgeStoreStatus(error: BridgeStoreError): number { + if (error.code === 'WORKER_UNAUTHORIZED') return 403; + if (error.code === 'ASSIGNMENT_INVALID') return 400; + if (error.code === 'RESULT_INVALID') return 502; + if (error.code === 'ASSIGNMENT_EXPIRED') return 504; + if (error.code === 'WORKER_OFFLINE' || error.code === 'WORKER_BUSY') { + return 503; + } + return 409; +} + +export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions): Router { + const router = Router(); + + router.post( + '/workspace-tools/execute', + asyncRoute(async (req, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + if ((options.isShuttingDown ?? checkServiceShutDown)()) { + res.status(503).json({ error: 'Service is shutting down' }); + return; + } + if (!isWorkspaceToolRequest(req.body)) { + res.status(400).json({ + error: 'Invalid workspace tool request', + }); + return; + } + + let selection: { workerId: string; explicit: boolean } | undefined; + try { + selection = resolveBridgeWorkerSelection({ + backend: options.backend, + configuredWorkerId: options.configuredWorkerId, + dynamicWorkers: options.dynamicWorkers, + requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), + trustedWorkerId: principal.codeWorkerId, + }); + } catch (error) { + if (error instanceof BridgeWorkerSelectionError) { + res.status(error.status).json({ error: error.message }); + return; + } + throw error; + } + if (selection == null) { + res.status(503).json({ + error: 'Workspace tools require the remote-bridge backend', + }); + return; + } + + const controller = new AbortController(); + const abort = () => controller.abort(); + req.once('aborted', abort); + const abortClosedResponse = () => { + if (!res.writableEnded) abort(); + }; + res.once('close', abortClosedResponse); + try { + const settlement = await options.store.dispatchWorkspaceTool({ + workerId: selection.workerId, + tenantId: principal.tenantId, + requireTenantBinding: + selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), + request: req.body, + deadlineAtMs: Date.now() + Math.max(1, options.timeoutMs ?? 30_000), + signal: controller.signal, + }); + if (settlement.status === 'rejected') { + let status = 422; + if (settlement.errorCode === 'SEARCH_TIMEOUT') status = 504; + if (settlement.errorCode === 'SEARCH_UNAVAILABLE') status = 503; + res.status(status).json({ + error: settlement.error, + code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', + }); + return; + } + res.status(200).json(settlement.result); + } catch (error) { + if (error instanceof BridgeStoreError) { + res.status(bridgeStoreStatus(error)).json({ + error: error.message, + code: error.code, + }); + return; + } + throw error; + } finally { + req.removeListener('aborted', abort); + res.removeListener('close', abortClosedResponse); + } + }), + ); + + return router; +} From adf2adf26eda1a322fd3c7ac8e2bc2b377431883 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 21:04:40 -0400 Subject: [PATCH 025/116] =?UTF-8?q?=F0=9F=97=BA=EF=B8=8F=20feat:=20List=20?= =?UTF-8?q?Attached=20Workspace=20Files=20(#92)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): list attached workspace files * fix: preserve canonical workspace listings * fix: preserve exact UTF-8 workspace paths * fix: preserve workspace listing policy --- docs/remote-bridge/README.md | 11 +- packages/code/README.md | 25 +- packages/code/src/protocol.test.ts | 59 ++++ packages/code/src/protocol.ts | 92 +++++- packages/code/src/worker.ts | 88 +++++- packages/code/src/workspace-worker.test.ts | 140 ++++++++++ packages/code/src/workspace.test.ts | 261 +++++++++++++++++- packages/code/src/workspace.ts | 307 ++++++++++++++++++++- service/src/bridge/router.test.ts | 5 + service/src/bridge/router.ts | 5 + service/src/workspace-tools/router.test.ts | 2 + service/src/workspace-tools/router.ts | 14 +- 12 files changed, 970 insertions(+), 39 deletions(-) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 5fbb5b37..60882dce 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -118,14 +118,15 @@ The endpoint uses the same authenticated principal-bound worker selection, tenant fence, lease deadline, cancellation, and settlement lifecycle as remote sandbox execution. Requests must name a workspace and operation advertised by that worker. Results are validated against the originating request before they -leave Code API, and are bounded to 1 MiB/500 lines for reads or 200 matches for -searches. Absolute paths, traversal, backslashes, symlink escapes, unexpected -fields, and host roots are rejected. +leave Code API, and are bounded to 1 MiB/500 lines for reads, 200 matches for +searches, or 500 relative paths for file listings. Absolute paths, traversal, +backslashes, symlink escapes, unexpected fields, and host roots are rejected. The workspace root can be an existing project, a Git repository, or an empty directory; Git is not required. This boundary keeps that directory local to the -operator's machine, but the selected file contents, search matches, and later -tool results necessarily cross the outbound bridge to Code API and the model. +operator's machine, but selected file contents, search matches, relative file +listings, and later tool results necessarily cross the outbound bridge to Code +API and the model. Treat them as explicit tool outputs, apply the same retention and audit policy as chat content, and do not register a directory containing secrets. The default operations are read-only; future mutation and shell operations must be diff --git a/packages/code/README.md b/packages/code/README.md index 076f831d..be421760 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -195,7 +195,8 @@ coding-agent access to workspace directories on the worker machine. A workspace may be an existing project, a Git repository, or a newly created empty directory; Git is optional. `LocalWorkspaceTools` registers opaque workspace IDs with optional display -names and exposes bounded `read_file` and literal `search_text` operations. +names and exposes bounded `read_file`, literal `search_text`, and deterministic +`list_files` operations. Only IDs, names, protocol version, and supported operations appear in worker capabilities; absolute host paths remain local to the worker process. @@ -204,10 +205,11 @@ and files larger than 1 MiB. The opened file is checked against its canonical in-workspace inode before it is read. Text search uses `rg` only to enumerate a bounded set of ignored-aware candidates with configuration and symlink following disabled. It then opens and verifies each candidate through the same confined -1 MiB read boundary before matching locally. Search limits returned line length -and stops after a bounded global result count. The worker process still belongs -inside the trusted BYOM boundary and should receive filesystem access only to -roots the operator intentionally registers. +1 MiB read boundary before matching locally. File listing invokes `rg` without +a shell, with configuration and symlink following disabled. Both operations +stop after bounded global result counts. The worker process still belongs inside +the trusted BYOM boundary and should receive filesystem access only to roots the +operator intentionally registers. Register one directory already present on the worker machine with the worker-directory option: @@ -220,17 +222,18 @@ The default public workspace ID is `primary` and the default display name is the directory basename. Operators can use `--workspace-id` and `--workspace-name`, or `LIBRECHAT_CODE_WORKER_DIR`, `LIBRECHAT_CODE_WORKSPACE_ID`, and `LIBRECHAT_CODE_WORKSPACE_NAME`, to set them -explicitly. `rg` must be installed on the worker for `search_text`. +explicitly. `rg` must be installed on the worker for `search_text` and +`list_files`. The worker advertises these capabilities only when a directory is configured and executes matching assignments under the bridge's existing lease, deadline, cancellation, credential-refresh, and settlement fencing. The workspace itself remains on the worker. As with Cursor's self-hosted agents, -text deliberately selected by `read_file` or `search_text` crosses the outbound -bridge so the remote agent/model can reason over it. Host paths are never part -of that payload. The Code API workspace-tool endpoint is delivered as a -dependent layer; deployments without it continue to use sandbox assignments -unchanged. +text and relative paths deliberately selected by `read_file`, `search_text`, or +`list_files` cross the outbound bridge so the remote agent/model can reason over +them. Host paths are never part of that payload. The Code API workspace-tool +endpoint is delivered as a dependent layer; deployments without it continue to +use sandbox assignments unchanged. After discarding or resetting that session's local runner, acknowledge recovery with `librechat-code reset-workspace `. The command uses the diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 793c7d0f..55b67b2f 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -4,6 +4,8 @@ import { bridgeWorkerPath, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceToolRequest, + isWorkspaceToolResult, } from './protocol.js'; test('bridgeWorkerPath encodes worker-controlled path segments', () => { @@ -89,3 +91,60 @@ test('bridge worker capabilities accept only bounded public workspace descriptor false, ); }); + +test('workspace file listing accepts only bounded portable requests and results', () => { + const request = { + protocolVersion: 1 as const, + operation: 'list_files' as const, + workspaceId: 'primary', + path: 'src', + maxResults: 20, + }; + assert.equal(isWorkspaceToolRequest(request), true); + assert.equal( + isWorkspaceToolRequest({ ...request, path: '../outside' }), + false, + ); + assert.equal(isWorkspaceToolRequest({ ...request, maxResults: 501 }), false); + + const result = { + protocolVersion: 1 as const, + operation: 'list_files' as const, + workspaceId: 'primary', + paths: ['src/app.ts', 'src/worker.ts'], + truncated: false, + }; + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + paths: ['/Users/operator/private'], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { ...result, paths: ['outside.txt'] }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + paths: ['src/app.ts', 'src/app.ts'], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + paths: ['src/app.ts', 'src/./app.ts'], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + root: '/private/workspace', + }), + false, + ); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 286b8125..9d065238 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -11,10 +11,14 @@ export const BRIDGE_WORKSPACE_READ_MAX_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_READ_MAX_LINES = 500; export const BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS = 200; export const BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH = 2000; +export const BRIDGE_WORKSPACE_LIST_MAX_RESULTS = 500; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; -export type BridgeWorkspaceToolOperation = 'read_file' | 'search_text'; +export type BridgeWorkspaceToolOperation = + | 'read_file' + | 'search_text' + | 'list_files'; export interface BridgeWorkspaceDescriptor { id: string; @@ -72,12 +76,30 @@ export interface WorkspaceSearchTextResult { truncated: boolean; } +export interface WorkspaceListFilesRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'list_files'; + workspaceId: string; + path?: string; + maxResults?: number; +} + +export interface WorkspaceListFilesResult { + protocolVersion: BridgeProtocolVersion; + operation: 'list_files'; + workspaceId: string; + paths: string[]; + truncated: boolean; +} + export type WorkspaceToolRequest = | WorkspaceReadFileRequest - | WorkspaceSearchTextRequest; + | WorkspaceSearchTextRequest + | WorkspaceListFilesRequest; export type WorkspaceToolResult = | WorkspaceReadFileResult - | WorkspaceSearchTextResult; + | WorkspaceSearchTextResult + | WorkspaceListFilesResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -95,6 +117,13 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'path', 'maxResults', ]); +const WORKSPACE_LIST_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'maxResults', +]); const WORKSPACE_READ_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -113,6 +142,13 @@ const WORKSPACE_SEARCH_RESULT_KEYS = new Set([ 'matches', 'truncated', ]); +const WORKSPACE_LIST_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'paths', + 'truncated', +]); const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ 'path', 'line', @@ -144,6 +180,8 @@ export interface BridgeWorkerRegistrationResponse { registrationGeneration?: number; registeredAt: string; leaseTtlMs: number; + /** Operations this Code API can dispatch after the worker advertises them. */ + supportedWorkspaceToolOperations?: BridgeWorkspaceToolOperation[]; } export interface BridgePairingRedemption { @@ -212,6 +250,8 @@ export type WorkspaceToolErrorCode = | 'READ_LIMIT_EXCEEDED' | 'REGISTRATION_INVALID' | 'EXECUTION_ABORTED' + | 'LIST_TIMEOUT' + | 'LIST_UNAVAILABLE' | 'SEARCH_TIMEOUT' | 'SEARCH_UNAVAILABLE'; @@ -221,6 +261,8 @@ const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'READ_LIMIT_EXCEEDED', 'REGISTRATION_INVALID', 'EXECUTION_ABORTED', + 'LIST_TIMEOUT', + 'LIST_UNAVAILABLE', 'SEARCH_TIMEOUT', 'SEARCH_UNAVAILABLE', ]); @@ -266,7 +308,7 @@ export function isValidBridgeWorkerId(workerId: string): boolean { return BRIDGE_WORKER_ID_PATTERN.test(workerId); } -function isSafePortableRelativePath(value: unknown): value is string { +export function isSafePortableRelativePath(value: unknown): value is string { if ( typeof value !== 'string' || value.length === 0 || @@ -354,6 +396,17 @@ export function isWorkspaceToolRequest( Number(request.maxResults) <= BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) ); } + if (request.operation === 'list_files') { + return ( + hasOnlyKeys(request, WORKSPACE_LIST_REQUEST_KEYS) && + (request.path === undefined || + isSafePortableRelativePath(request.path)) && + (request.maxResults === undefined || + (Number.isSafeInteger(request.maxResults) && + Number(request.maxResults) >= 1 && + Number(request.maxResults) <= BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) + ); + } return false; } @@ -405,6 +458,30 @@ export function isWorkspaceToolResult( ); } + if (request.operation === 'list_files') { + const maxResults = request.maxResults ?? 100; + if ( + !hasOnlyKeys(result, WORKSPACE_LIST_RESULT_KEYS) || + !Array.isArray(result.paths) || + result.paths.length > maxResults + ) { + return false; + } + const normalizedPaths = new Set(); + for (const path of result.paths) { + if ( + !isSafePortableRelativePath(path) || + !isWithinRequestedPath(path, request.path) + ) { + return false; + } + const normalizedPath = normalizePortableRelativePath(path); + if (normalizedPaths.has(normalizedPath)) return false; + normalizedPaths.add(normalizedPath); + } + return true; + } + if (!Array.isArray(result.matches)) return false; const maxResults = request.maxResults ?? 50; return ( @@ -438,9 +515,12 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !Array.isArray(capabilities.operations) || capabilities.operations.length < 1 || - capabilities.operations.length > 2 || + capabilities.operations.length > 3 || !capabilities.operations.every( - (operation) => operation === 'read_file' || operation === 'search_text', + (operation) => + operation === 'read_file' || + operation === 'search_text' || + operation === 'list_files', ) || new Set(capabilities.operations).size !== capabilities.operations.length || !Array.isArray(capabilities.workspaces) || diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 27526747..98eb9976 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -137,6 +137,42 @@ function workspaceCapabilitiesMatch( ); } +function registrationCompatibleCapabilities( + capabilities: BridgeWorkerCapabilities, +): BridgeWorkerCapabilities { + const workspaceTools = capabilities.workspaceTools; + if ( + workspaceTools == null || + !workspaceTools.operations.includes('list_files') + ) { + return capabilities; + } + const operations = workspaceTools.operations.filter( + (operation) => operation !== 'list_files', + ); + if (operations.length === 0) { + const { workspaceTools: _workspaceTools, ...compatible } = capabilities; + return compatible; + } + return { + ...capabilities, + workspaceTools: { ...workspaceTools, operations }, + }; +} + +function supportsDesiredWorkspaceTools( + registration: BridgeWorkerRegistrationResponse, + capabilities: BridgeWorkerCapabilities, +): boolean { + const desired = capabilities.workspaceTools?.operations; + const supported = registration.supportedWorkspaceToolOperations; + return ( + desired != null && + Array.isArray(supported) && + desired.every((operation) => supported.includes(operation)) + ); +} + export class BridgeWorkspaceQuarantinedError extends Error { constructor( message: string, @@ -152,6 +188,8 @@ export class BridgeWorker { private readonly codeApiUrl: string; private readonly runtimeSupervisor: RuntimeSupervisor; private readonly incarnationId: string; + private readonly compatibleCapabilities: BridgeWorkerCapabilities; + private registrationCapabilities: BridgeWorkerCapabilities; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; @@ -194,6 +232,10 @@ export class BridgeWorker { }); this.incarnationId = options.incarnationId ?? randomBytes(18).toString('base64url'); + this.compatibleCapabilities = registrationCompatibleCapabilities( + options.capabilities, + ); + this.registrationCapabilities = this.compatibleCapabilities; } async register( @@ -218,16 +260,42 @@ export class BridgeWorker { const registrationStartedAtMs = Date.now(); let registration: BridgeWorkerRegistrationResponse; try { - registration = await this.request( - `${this.codeApiUrl}/bridge/workers/register`, - { - protocolVersion: BRIDGE_PROTOCOL_VERSION, - workerId: this.options.workerId, - incarnationId: this.incarnationId, - capabilities: this.options.capabilities, - }, - registrationController.signal, - ); + const register = (capabilities: BridgeWorkerCapabilities) => + this.request( + `${this.codeApiUrl}/bridge/workers/register`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: this.options.workerId, + incarnationId: this.incarnationId, + capabilities, + }, + registrationController.signal, + ); + try { + registration = await register(this.registrationCapabilities); + } catch (error) { + if ( + !(error instanceof BridgeProtocolError) || + error.status !== 400 || + this.registrationCapabilities === this.compatibleCapabilities + ) { + throw error; + } + this.registrationCapabilities = this.compatibleCapabilities; + registration = await register(this.registrationCapabilities); + } + if ( + this.registrationCapabilities !== this.options.capabilities && + supportsDesiredWorkspaceTools(registration, this.options.capabilities) + ) { + this.registrationCapabilities = this.options.capabilities; + try { + registration = await register(this.registrationCapabilities); + } catch (error) { + this.registrationCapabilities = this.compatibleCapabilities; + if (signal?.aborted) throw error; + } + } } finally { clearTimeout(timeout); signal?.removeEventListener('abort', abortRegistration); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 352cbe33..742808ea 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -6,6 +6,146 @@ import { WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +const listWorkspaceCapabilities = { + protocolVersion: 1 as const, + operations: [ + 'read_file' as const, + 'search_text' as const, + 'list_files' as const, + ], + workspaces: [{ id: 'primary' }], +}; + +function registrationResponse(supportsList: boolean): Response { + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + ...(supportsList + ? { + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + ], + } + : {}), + }); +} + +function listWorkspaceExecutor() { + return { + capabilities: listWorkspaceCapabilities, + async execute() { + return { + protocolVersion: 1 as const, + operation: 'list_files' as const, + workspaceId: 'primary', + paths: [], + truncated: false, + }; + }, + }; +} + +test('worker keeps v1 registration compatible until list_files support is advertised', async () => { + const registrations: string[][] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: listWorkspaceCapabilities, + }, + workspaceTools: listWorkspaceExecutor(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { workspaceTools?: { operations: string[] } }; + }; + registrations.push(body.capabilities.workspaceTools?.operations ?? []); + return registrationResponse(false); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [['read_file', 'search_text']]); +}); + +test('worker re-registers list_files after the Code API advertises support', async () => { + const registrations: string[][] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: listWorkspaceCapabilities, + }, + workspaceTools: listWorkspaceExecutor(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { workspaceTools?: { operations: string[] } }; + }; + registrations.push(body.capabilities.workspaceTools?.operations ?? []); + return registrationResponse(true); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + ['read_file', 'search_text'], + ['read_file', 'search_text', 'list_files'], + ]); +}); + +test('worker retains a compatible registration when list_files promotion times out', async () => { + let registrationRequests = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + registrationTransportTimeoutMs: 20, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: listWorkspaceCapabilities, + }, + workspaceTools: listWorkspaceExecutor(), + fetchImpl: async (_input, init) => { + registrationRequests += 1; + if (registrationRequests === 1) return registrationResponse(true); + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(init.signal?.reason ?? new Error('aborted')), + { once: true }, + ); + }); + }, + }); + + const registration = await worker.register(); + + assert.equal(registration.workerId, 'vm-1'); + assert.equal(registrationRequests, 2); +}); + test('worker preserves bounded workspace rejection codes', async () => { let settlement: Record | undefined; const workspaceCapabilities = { diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index e8ee53bf..b0335507 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, sep } from 'node:path'; import test from 'node:test'; import { promisify } from 'node:util'; @@ -159,6 +159,263 @@ test('searches workspace text with a hard global result bound', async (t) => { ); }); +test('lists workspace files deterministically with a hard result bound', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + await mkdir(join(root, 'docs')); + await writeFile(join(root, 'src', 'app.ts'), 'export {}'); + await writeFile(join(root, 'src', 'worker.ts'), 'export {}'); + await writeFile(join(root, 'docs', 'guide.md'), '# Guide'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 2, + }), + { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['docs/guide.md', 'src/app.ts'], + truncated: true, + }, + ); + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + path: 'src', + maxResults: 10, + }), + { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['src/app.ts', 'src/worker.ts'], + truncated: false, + }, + ); +}); + +test('rejects listing through a directory symlink that leaves the workspace', async (t) => { + const parent = await mkdtemp( + join(tmpdir(), 'librechat-code-workspace-parent-'), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'workspace'); + const outside = join(parent, 'outside'); + await mkdir(root); + await mkdir(outside); + await writeFile(join(outside, 'secret.txt'), 'host secret'); + await symlink(outside, join(root, 'linked-outside')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + path: 'linked-outside', + }), + /invalid workspace path/i, + ); +}); + +test('listing preserves an in-workspace symlink namespace', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + await writeFile(join(root, 'src', 'app.ts'), 'export const app = true;'); + await symlink(join(root, 'src'), join(root, 'alias')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + path: 'alias', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.deepEqual(result.paths, ['alias/app.ts']); +}); + +test('listing ignores ripgrep config that follows escaping symlinks', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-list-parent-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'workspace'); + const outside = join(parent, 'outside'); + await mkdir(root); + await mkdir(outside); + await writeFile(join(root, 'safe.txt'), 'safe'); + await writeFile(join(outside, 'secret.txt'), 'secret'); + await symlink(outside, join(root, 'linked-outside')); + const config = join(parent, 'ripgrep.conf'); + await writeFile(config, '--follow\n'); + const previousConfig = process.env.RIPGREP_CONFIG_PATH; + process.env.RIPGREP_CONFIG_PATH = config; + t.after(() => { + if (previousConfig === undefined) delete process.env.RIPGREP_CONFIG_PATH; + else process.env.RIPGREP_CONFIG_PATH = previousConfig; + }); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.deepEqual(result.paths, ['safe.txt']); +}); + +test('listing preserves ignore rules for an explicitly requested subtree', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'vendor')); + await writeFile(join(root, 'vendor', 'dependency.js'), 'ignored'); + await writeFile(join(root, '.ignore'), 'vendor/\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + path: 'vendor', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.deepEqual(result.paths, []); +}); + +test('listing excludes an explicitly requested symlink file alias', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'target.txt'), 'target'); + await symlink(join(root, 'target.txt'), join(root, 'alias.txt')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + path: 'alias.txt', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.deepEqual(result.paths, []); +}); + +test('listing skips filenames the portable protocol cannot represent', async (t) => { + if (sep === '\\') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'invalid\\name.txt'), 'invalid'); + await writeFile(join(root, 'safe.txt'), 'safe'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.deepEqual(result.paths, ['safe.txt']); +}); + +test('listing rejects invalid UTF-8 bytes instead of aliasing a valid filename', async (t) => { + if (sep === '\\') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const invalidPath = Buffer.concat([ + Buffer.from(`${root}${sep}`), + Buffer.from([0xff]), + Buffer.from('.txt'), + ]); + try { + await writeFile(invalidPath, 'invalid'); + } catch { + t.skip('filesystem does not support non-UTF-8 filenames'); + return; + } + await writeFile(join(root, '\ufffd.txt'), 'valid but ignored'); + await writeFile(join(root, '.ignore'), '\ufffd.txt\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.equal(result.paths.includes('\ufffd.txt'), false); +}); + +test('listing preserves a leading UTF-8 BOM in a filename', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, '\ufefffoo.txt'), 'bom filename'); + await writeFile(join(root, 'foo.txt'), 'ignored sibling'); + await writeFile(join(root, '.ignore'), 'foo.txt\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.equal(result.paths.includes('\ufefffoo.txt'), true); + assert.equal(result.paths.includes('foo.txt'), false); +}); + +test('listing excludes a non-regular explicit target', async (t) => { + if (sep === '\\') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await execFileAsync('mkfifo', [join(root, 'events.pipe')]); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + path: 'events.pipe', + }); + + if (result.operation !== 'list_files') assert.fail('expected list result'); + assert.deepEqual(result.paths, []); +}); + test('search ignores ripgrep config that follows escaping symlinks', async (t) => { const parent = await mkdtemp(join(tmpdir(), 'librechat-code-search-parent-')); t.after(() => rm(parent, { recursive: true, force: true })); @@ -491,7 +748,7 @@ test('advertises workspace IDs and names without exposing host roots', async (t) assert.deepEqual(tools.capabilities, { protocolVersion: 1, - operations: ['read_file', 'search_text'], + operations: ['read_file', 'search_text', 'list_files'], workspaces: [{ id: 'primary', name: 'LibreChat' }], }); assert.equal(JSON.stringify(tools.capabilities).includes(root), false); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 3e6c27c0..43114516 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process'; import { constants } from 'node:fs'; -import { open, realpath, stat } from 'node:fs/promises'; +import { lstat, open, realpath, stat } from 'node:fs/promises'; import { isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -9,8 +9,10 @@ import { BRIDGE_PROTOCOL_VERSION, BRIDGE_WORKSPACE_READ_MAX_BYTES, BRIDGE_WORKSPACE_READ_MAX_LINES, + BRIDGE_WORKSPACE_LIST_MAX_RESULTS, BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS, BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, + isSafePortableRelativePath, isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, isWorkspaceToolResult, @@ -21,6 +23,8 @@ import type { BridgeWorkspaceToolCapabilities, WorkspaceReadFileRequest, WorkspaceReadFileResult, + WorkspaceListFilesRequest, + WorkspaceListFilesResult, WorkspaceSearchMatch, WorkspaceSearchTextRequest, WorkspaceSearchTextResult, @@ -33,6 +37,8 @@ export { isWorkspaceToolRequest, isWorkspaceToolResult }; export type { WorkspaceReadFileRequest, WorkspaceReadFileResult, + WorkspaceListFilesRequest, + WorkspaceListFilesResult, WorkspaceSearchMatch, WorkspaceSearchTextRequest, WorkspaceSearchTextResult, @@ -61,6 +67,7 @@ export interface WorkspaceToolExecutor { const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; +const LIST_TIMEOUT_MS = 10_000; function isUtf8ScalarString(value: string): boolean { return Buffer.from(value).toString('utf8') === value; @@ -499,6 +506,297 @@ async function searchWorkspace( }; } +async function listWorkspaceFiles( + root: string, + request: WorkspaceListFilesRequest, + signal?: AbortSignal, +): Promise { + const deadline = Date.now() + LIST_TIMEOUT_MS; + const maxResults = request.maxResults ?? 100; + if ( + !Number.isSafeInteger(maxResults) || + maxResults < 1 || + maxResults > BRIDGE_WORKSPACE_LIST_MAX_RESULTS + ) { + throw new WorkspaceToolError( + 'Invalid workspace listing', + 'INVALID_REQUEST', + ); + } + + const listPath = request.path ?? '.'; + const target = resolveWorkspacePath(root, listPath); + let canonicalTarget: string; + try { + canonicalTarget = await withinListDeadline( + realpath(target), + signal, + deadline, + ); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + if (!isWithinRoot(root, canonicalTarget)) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + const canonicalListPath = relative(root, canonicalTarget) || '.'; + const portableCanonicalListPath = canonicalListPath.split(sep).join('/'); + let canonicalTargetIsDirectory = false; + try { + canonicalTargetIsDirectory = ( + await withinListDeadline(stat(canonicalTarget), signal, deadline) + ).isDirectory(); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + const normalizedRequestedResultPath = request.path + ?.split('/') + .filter((segment) => segment.length > 0 && segment !== '.') + .join('/'); + const requestedResultPath = normalizedRequestedResultPath || undefined; + + const candidates: Array<{ filesystemPath: string; resultPath: string }> = []; + let truncated = false; + let pending: Buffer = Buffer.alloc(0); + let stoppedForLimit = false; + await new Promise((resolvePromise, reject) => { + const args = [ + '--files', + '--no-config', + '--no-follow', + '--no-messages', + '--sort', + 'path', + '--null', + ]; + if (portableCanonicalListPath !== '.') { + args.push( + '--glob', + canonicalTargetIsDirectory + ? `${portableCanonicalListPath}/**` + : portableCanonicalListPath, + ); + } + args.push('--', '.'); + const child = spawn( + 'rg', + args, + { cwd: root, stdio: ['ignore', 'pipe', 'ignore'] }, + ); + let aborted = false; + let timedOut = false; + const abort = () => { + aborted = true; + child.kill(); + }; + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, Math.max(0, deadline - Date.now())); + const cleanup = () => { + clearTimeout(timeout); + signal?.removeEventListener('abort', abort); + }; + const pathDecoder = new TextDecoder('utf-8', { + fatal: true, + ignoreBOM: true, + }); + const consumePath = (rawPath: Buffer) => { + if (rawPath.length === 0 || stoppedForLimit) return; + let path: string; + try { + path = pathDecoder.decode(rawPath); + } catch { + return; + } + if (!Buffer.from(path).equals(rawPath)) return; + if (candidates.length === maxResults + BRIDGE_WORKSPACE_LIST_MAX_RESULTS) { + truncated = true; + stoppedForLimit = true; + child.kill(); + return; + } + const portablePath = sep === '\\' ? path.split(sep).join('/') : path; + const normalizedPath = portablePath.startsWith('./') + ? portablePath.slice(2) + : portablePath; + const resultPath = + requestedResultPath == null + ? normalizedPath + : portableCanonicalListPath === '.' + ? `${requestedResultPath}/${normalizedPath}` + : normalizedPath === portableCanonicalListPath || + normalizedPath.startsWith(`${portableCanonicalListPath}/`) + ? `${requestedResultPath}${normalizedPath.slice(portableCanonicalListPath.length)}` + : normalizedPath; + if (!isSafePortableRelativePath(resultPath)) return; + candidates.push({ filesystemPath: normalizedPath, resultPath }); + }; + + child.stdout.on('data', (chunk: Buffer) => { + pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]); + let delimiter = pending.indexOf(0); + while (delimiter >= 0) { + consumePath(pending.subarray(0, delimiter)); + pending = pending.subarray(delimiter + 1); + delimiter = pending.indexOf(0); + } + }); + child.once('error', () => { + cleanup(); + reject( + new WorkspaceToolError( + 'Workspace listing unavailable', + 'LIST_UNAVAILABLE', + ), + ); + }); + child.once('close', (code) => { + cleanup(); + consumePath(pending); + if (aborted) { + reject( + new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ); + } else if (timedOut) { + reject( + new WorkspaceToolError('Workspace listing timed out', 'LIST_TIMEOUT'), + ); + } else if (stoppedForLimit || code === 0 || code === 1) { + resolvePromise(); + } else { + reject( + new WorkspaceToolError( + 'Workspace listing unavailable', + 'LIST_UNAVAILABLE', + ), + ); + } + }); + }); + + const paths: string[] = []; + const seenPaths = new Set(); + for (const candidate of candidates) { + let canonicalPath: string; + try { + canonicalPath = await withinListDeadline( + realpath(resolveWorkspacePath(root, candidate.filesystemPath)), + signal, + deadline, + ); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + continue; + } + if (!isWithinRoot(root, canonicalPath)) { + continue; + } + const reportedPath = resolveWorkspacePath(root, candidate.resultPath); + try { + const reportedPathStat = await withinListDeadline( + lstat(reportedPath), + signal, + deadline, + ); + if (reportedPathStat.isSymbolicLink()) continue; + const canonicalReportedPath = await withinListDeadline( + realpath(reportedPath), + signal, + deadline, + ); + if (canonicalReportedPath !== canonicalPath) continue; + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + continue; + } + let regularFile = false; + try { + regularFile = ( + await withinListDeadline(stat(canonicalPath), signal, deadline) + ).isFile(); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + continue; + } + if (!regularFile || seenPaths.has(candidate.resultPath)) continue; + if (paths.length === maxResults) { + truncated = true; + break; + } + seenPaths.add(candidate.resultPath); + paths.push(candidate.resultPath); + } + + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: request.workspaceId, + paths, + truncated, + }; +} + +async function withinListDeadline( + operation: Promise, + signal: AbortSignal | undefined, + deadline: number, +): Promise { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw new WorkspaceToolError('Workspace listing timed out', 'LIST_TIMEOUT'); + } + return new Promise((resolvePromise, reject) => { + let settled = false; + const settle = (callback: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + signal?.removeEventListener('abort', abort); + callback(); + }; + const abort = () => + settle(() => + reject( + new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ), + ); + const timeout = setTimeout( + () => + settle(() => + reject( + new WorkspaceToolError( + 'Workspace listing timed out', + 'LIST_TIMEOUT', + ), + ), + ), + remainingMs, + ); + signal?.addEventListener('abort', abort, { once: true }); + operation.then( + (value) => settle(() => resolvePromise(value)), + (error: unknown) => settle(() => reject(error)), + ); + }); +} + export class LocalWorkspaceTools implements WorkspaceToolExecutor { readonly capabilities: BridgeWorkspaceToolCapabilities; @@ -508,7 +806,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ) { this.capabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, - operations: ['read_file', 'search_text'], + operations: ['read_file', 'search_text', 'list_files'], workspaces, }; } @@ -525,7 +823,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ); const capabilities: BridgeWorkspaceToolCapabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, - operations: ['read_file', 'search_text'], + operations: ['read_file', 'search_text', 'list_files'], workspaces, }; if (!isValidBridgeWorkspaceToolCapabilities(capabilities)) { @@ -574,6 +872,9 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { if (request.operation === 'search_text') { return searchWorkspace(root, request, signal); } + if (request.operation === 'list_files') { + return listWorkspaceFiles(root, request, signal); + } const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 7e6e0fc5..2e5343f7 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -279,6 +279,11 @@ describe('paired bridge HTTP API', () => { workerId: 'vm-1', incarnationId: 'incarnation-00000001', registrationGeneration: 1, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + ], }); const crossDeploymentRevoke = await fetch( diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 0905993a..f4c2b693 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -379,6 +379,11 @@ router.post( registrationGeneration, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + ], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index aa9c2102..28893910 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -71,6 +71,8 @@ test('rejects new workspace dispatches while the service is shutting down', asyn test.each([ ['SEARCH_TIMEOUT', 504], ['SEARCH_UNAVAILABLE', 503], + ['LIST_TIMEOUT', 504], + ['LIST_UNAVAILABLE', 503], ] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { const app = express(); app.use(json()); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 13a4bd9d..5daf83ea 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -101,8 +101,18 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) }); if (settlement.status === 'rejected') { let status = 422; - if (settlement.errorCode === 'SEARCH_TIMEOUT') status = 504; - if (settlement.errorCode === 'SEARCH_UNAVAILABLE') status = 503; + if ( + settlement.errorCode === 'SEARCH_TIMEOUT' || + settlement.errorCode === 'LIST_TIMEOUT' + ) { + status = 504; + } + if ( + settlement.errorCode === 'SEARCH_UNAVAILABLE' || + settlement.errorCode === 'LIST_UNAVAILABLE' + ) { + status = 503; + } res.status(status).json({ error: settlement.error, code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', From 578c8a1d7d9f87312d27d26cf0c0cb7b16dfa6b2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 21:10:25 -0400 Subject: [PATCH 026/116] =?UTF-8?q?=F0=9F=8C=B1=20feat:=20Create=20an=20Ex?= =?UTF-8?q?plicit=20Default=20Workspace=20(#93)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/code/README.md | 21 +++++++- packages/code/src/cli.ts | 47 ++++++++++++++--- packages/code/src/storage.test.ts | 61 ++++++++++++++++++++++ packages/code/src/storage.ts | 56 ++++++++++++++++++-- packages/code/src/workspace-cli.test.ts | 68 ++++++++++++++++++++++++- 5 files changed, 241 insertions(+), 12 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index be421760..f5483cf8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -218,12 +218,31 @@ worker-directory option: librechat-code run --worker-dir /path/to/workspace ``` +To start without an existing project or Git repository, explicitly ask the +worker to create and reuse an application-owned workspace: + +```bash +librechat-code run --default-workspace +``` + +The directory is created with owner-only permissions below +`~/.local/share/librechat/code/workspaces/`, using stable digests of the worker +and workspace IDs so distinct IDs cannot alias on case-insensitive filesystems. +The deployment and paired bridge identity are also part of the namespace, so +re-pairing or switching Code API deployments cannot expose the previous +identity's files. It persists across worker restarts. The current workspace +tools are read-only, so an empty directory must be populated by a local process +until write-capable coding tools are enabled. The worker never registers its +process working directory implicitly, and `--default-workspace` cannot be +combined with `--worker-dir`. + The default public workspace ID is `primary` and the default display name is the directory basename. Operators can use `--workspace-id` and `--workspace-name`, or `LIBRECHAT_CODE_WORKER_DIR`, `LIBRECHAT_CODE_WORKSPACE_ID`, and `LIBRECHAT_CODE_WORKSPACE_NAME`, to set them explicitly. `rg` must be installed on the worker for `search_text` and -`list_files`. +`list_files`. `LIBRECHAT_CODE_DEFAULT_WORKSPACE=true` is the environment +equivalent of `--default-workspace`. The worker advertises these capabilities only when a directory is configured and executes matching assignments under the bridge's existing lease, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index a8cb1f93..3f68a9d3 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -8,6 +8,8 @@ import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { defaultBridgeIdentityPath, + defaultWorkspacePath, + ensurePrivateWorkspaceDirectory, loadBridgeIdentity, saveBridgeIdentity, } from './storage.js'; @@ -67,6 +69,10 @@ function option(args: string[], name: string): string | undefined { return args.find((value) => value.startsWith(`${name}=`))?.slice(name.length + 1); } +function nonEmpty(value: string | undefined): string | undefined { + return value?.trim().length ? value : undefined; +} + function defaultWorkspaceName(workerDirectory: string, workspaceId: string): string { const directoryName = basename(resolve(workerDirectory)); return directoryName.trim().length > 0 && @@ -197,15 +203,42 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise 0; - const workerDirectory = - runtimeSessionId == null - ? option(args, '--worker-dir') ?? - process.env.LIBRECHAT_CODE_WORKER_DIR?.trim() - : undefined; const workspaceId = option(args, '--workspace-id') ?? process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; + const explicitWorkerDirectory = + runtimeSessionId == null + ? nonEmpty( + option(args, '--worker-dir') ?? + process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), + ) + : undefined; + const useDefaultWorkspace = + runtimeSessionId == null && + (args.includes('--default-workspace') || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === + 'true'); + if (explicitWorkerDirectory && useDefaultWorkspace) { + throw new Error( + '--worker-dir and --default-workspace cannot be used together', + ); + } + const workerDirectory = + explicitWorkerDirectory ?? + (useDefaultWorkspace + ? defaultWorkspacePath({ + codeApiUrl, + securityIdentity: + pairedIdentity?.publicKey ?? + required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken), + workerId, + workspaceId, + }) + : undefined); + if (useDefaultWorkspace && workerDirectory) { + await ensurePrivateWorkspaceDirectory(workerDirectory); + } const workspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: [ @@ -214,7 +247,9 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise ); }); +test('default workspace paths are stable and collision resistant', () => { + const home = '/home/tester'; + const options = { + codeApiUrl: 'https://code.example/v1', + securityIdentity: 'bridge-public-key', + workerId: 'vm-1', + workspaceId: 'primary', + homeDirectory: home, + }; + assert.equal( + defaultWorkspacePath(options), + defaultWorkspacePath({ ...options, codeApiUrl: 'https://code.example/v1/' }), + ); + assert.notEqual( + defaultWorkspacePath({ ...options, workerId: 'vm:a' }), + defaultWorkspacePath({ ...options, workerId: 'vm_a' }), + ); + assert.notEqual( + defaultWorkspacePath({ ...options, workerId: 'vm:a' }), + defaultWorkspacePath({ + ...options, + workerId: 'vm_a-2d4fcea9e21e004d', + }), + ); + assert.notEqual( + defaultWorkspacePath({ ...options, workerId: 'VM-1' }).toLowerCase(), + defaultWorkspacePath({ ...options, workerId: 'vm-1' }).toLowerCase(), + ); + assert.notEqual( + defaultWorkspacePath(options), + defaultWorkspacePath({ + ...options, + securityIdentity: 'new-pairing-public-key', + }), + ); + assert.notEqual( + defaultWorkspacePath(options), + defaultWorkspacePath({ + ...options, + codeApiUrl: 'https://other-code.example/v1', + }), + ); +}); + +test('default workspace directories are created with owner-only permissions', async () => { + const directory = await mkdtemp( + join(tmpdir(), 'librechat-code-workspace-home-'), + ); + const path = join(directory, 'workspaces', 'primary'); + try { + await ensurePrivateWorkspaceDirectory(path); + const metadata = await stat(path); + assert.equal(metadata.isDirectory(), true); + assert.equal(metadata.mode & 0o777, 0o700); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + test('paired identity is persisted atomically with owner-only permissions', async () => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-')); const path = join(directory, 'identity.json'); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index a26c3eb0..adeab1bd 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes } from 'node:crypto'; -import { chmod, mkdir, open, readFile, rename, rm } from 'node:fs/promises'; +import { chmod, lstat, mkdir, open, readFile, rename, rm } from 'node:fs/promises'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -27,12 +27,60 @@ function isPairedIdentity(value: unknown): value is PairedBridgeWorkerIdentity { export function defaultBridgeIdentityPath(workerId: string): string { const readableName = workerId.replace(/[^A-Za-z0-9._-]/g, '_'); - const fileName = readableName === workerId - ? readableName - : `${readableName}-${createHash('sha256').update(workerId).digest('hex').slice(0, 16)}`; + const fileName = + readableName === workerId + ? readableName + : `${readableName}-${createHash('sha256') + .update(workerId) + .digest('hex') + .slice(0, 16)}`; return join(homedir(), '.config', 'librechat', 'code', `${fileName}.json`); } +function workspaceStorageName(value: string): string { + return `id-${createHash('sha256').update(value).digest('hex')}`; +} + +export interface DefaultWorkspacePathOptions { + codeApiUrl: string; + securityIdentity: string; + workerId: string; + workspaceId: string; + homeDirectory?: string; +} + +export function defaultWorkspacePath({ + codeApiUrl, + securityIdentity, + workerId, + workspaceId, + homeDirectory = homedir(), +}: DefaultWorkspacePathOptions): string { + const deploymentIdentity = `${codeApiUrl.replace(/\/+$/, '')}\0${securityIdentity}`; + return join( + homeDirectory, + '.local', + 'share', + 'librechat', + 'code', + 'workspaces', + workspaceStorageName(deploymentIdentity), + workspaceStorageName(workerId), + workspaceStorageName(workspaceId), + ); +} + +export async function ensurePrivateWorkspaceDirectory( + path: string, +): Promise { + await mkdir(path, { recursive: true, mode: 0o700 }); + const metadata = await lstat(path); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new BridgeProtocolError('Default workspace path must be a directory'); + } + await chmod(path, 0o700); +} + export async function saveBridgeIdentity( path: string, identity: PairedBridgeWorkerIdentity, diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 4ac486ee..67692cab 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -1,13 +1,15 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { once } from 'node:events'; -import { mkdtemp, mkdir, rm } from 'node:fs/promises'; +import { mkdtemp, mkdir, rm, stat } from 'node:fs/promises'; import { createServer } from 'node:http'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; +import { defaultWorkspacePath } from './storage.js'; + test('CLI validates a configured worker directory before registration', () => { const result = spawnSync( process.execPath, @@ -32,6 +34,30 @@ test('CLI validates a configured worker directory before registration', () => { assert.match(result.stderr, /invalid workspace registration/i); }); +test('CLI trims an environment-configured worker directory', async (t) => { + const workspaceRoot = await mkdtemp( + join(tmpdir(), 'librechat-code-env-workspace-'), + ); + t.after(() => rm(workspaceRoot, { recursive: true, force: true })); + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url)), 'run'], + { + encoding: 'utf8', + timeout: 500, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: ` ${workspaceRoot} `, + }, + }, + ); + + assert.doesNotMatch(result.stderr, /invalid workspace registration/i); +}); + test('CLI falls back to the workspace ID when the directory basename is invalid', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); @@ -116,3 +142,43 @@ test('CLI falls back to the workspace ID when the directory basename is invalid' }, ); }); + +test('CLI explicitly creates and registers an application-owned default workspace', async () => { + const testHome = await mkdtemp(join(tmpdir(), 'librechat-code-home-')); + try { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--default-workspace', + ], + { + encoding: 'utf8', + timeout: 500, + env: { + ...process.env, + HOME: testHome, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: ' ', + }, + }, + ); + + assert.doesNotMatch(result.stderr, /invalid workspace registration/i); + const workspace = defaultWorkspacePath({ + codeApiUrl: 'http://127.0.0.1:1/v1', + securityIdentity: 'worker-secret', + workerId: 'engineering-vm', + workspaceId: 'primary', + homeDirectory: testHome, + }); + const metadata = await stat(workspace); + assert.equal(metadata.isDirectory(), true); + assert.equal(metadata.mode & 0o777, 0o700); + } finally { + await rm(testHome, { recursive: true, force: true }); + } +}); From e116d1095291151f85d9e566a4ab5e6948f68888 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 2 Sep 2026 21:56:30 -0400 Subject: [PATCH 027/116] =?UTF-8?q?=F0=9F=8F=A1=20feat:=20Add=20Hosted=20A?= =?UTF-8?q?pp=20Runner=20(#57)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add dedicated Lambda hosted-app runner * fix: serialize hosted app cgroup cleanup * fix: serialize hosted app workspace lifecycle * fix: fence hosted app replacement cleanup * fix: preserve hosted app lifecycle invariants * fix: complete hosted app workspace coordination --- .github/workflows/ci.yml | 17 +- api/Dockerfile | 14 +- api/src/api/hosted-app.routes.test.ts | 115 ++++ api/src/api/lifecycle.ts | 8 +- api/src/api/v2.ts | 95 ++- api/src/config.ts | 25 + api/src/hosted-app-launcher.sh | 26 + api/src/hosted-app.test.ts | 565 ++++++++++++++++ api/src/hosted-app.ts | 796 +++++++++++++++++++++++ api/src/index.ts | 5 + docs/lambda-microvm/README.md | 58 ++ scripts/build-lambda-microvm-artifact.sh | 53 +- 12 files changed, 1750 insertions(+), 27 deletions(-) create mode 100644 api/src/api/hosted-app.routes.test.ts create mode 100644 api/src/hosted-app-launcher.sh create mode 100644 api/src/hosted-app.test.ts create mode 100644 api/src/hosted-app.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ba2ef1c..fd89d5da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,12 +158,17 @@ jobs: shellcheck scripts/build-lambda-microvm-artifact.sh - name: Validate runner Dockerfile - run: >- - docker buildx build --check - --platform linux/arm64 - --target lambda-microvm-runner - -f api/Dockerfile - . + run: | + docker buildx build --check \ + --platform linux/arm64 \ + --target lambda-microvm-runner \ + -f api/Dockerfile \ + . + docker buildx build --check \ + --platform linux/arm64 \ + --target lambda-microvm-app-host \ + -f api/Dockerfile \ + . - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 with: diff --git a/api/Dockerfile b/api/Dockerfile index 61a0bd08..26877009 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -148,7 +148,8 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \ 2>/dev/null || true COPY api/src/entrypoint.sh ./entrypoint.sh -RUN chmod +x ./entrypoint.sh +COPY api/src/hosted-app-launcher.sh /usr/local/bin/codeapi-hosted-app-launcher +RUN chmod +x ./entrypoint.sh /usr/local/bin/codeapi-hosted-app-launcher # ============================================================================ # Stage 2b: AWS Lambda MicroVM container base image @@ -188,6 +189,17 @@ ENV PORT=2000 \ EXPOSE 2000/tcp ENTRYPOINT ["/sandbox_api/entrypoint.sh"] +# Dedicated resident-process host. This image deliberately keeps the runner +# control listener on 8080 and exposes one fixed user-app port separately. +# `/execute` is disabled when hosted-app mode is enabled; the control plane +# restores a checkpoint and starts the app through `/api/v2/hosted-app/start`. +FROM lambda-microvm-runner AS lambda-microvm-app-host + +ENV SANDBOX_HOSTED_APPS_ENABLED=true \ + SANDBOX_HOSTED_APP_PORT=3000 + +EXPOSE 3000/tcp + # ============================================================================ # Stage 3: Build the Rust launcher binary (Fedora for libkrun ABI) # ============================================================================ diff --git a/api/src/api/hosted-app.routes.test.ts b/api/src/api/hosted-app.routes.test.ts new file mode 100644 index 00000000..f55ecac2 --- /dev/null +++ b/api/src/api/hosted-app.routes.test.ts @@ -0,0 +1,115 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test'; +import express from 'express'; +import type { Server } from 'node:http'; +import { config } from '../config'; +import { HostedAppError, hostedAppSupervisor } from '../hosted-app'; +import { resetSessionWorkspaceStateForTests } from '../session-workspace'; +import v2Router from './v2'; + +let server: Server; +let baseUrl: string; +const savedHostedAppsEnabled = config.hosted_apps_enabled; +const savedSessionWorkspaceEnabled = config.session_workspace_enabled; + +beforeAll(async () => { + const app = express(); + app.use(express.urlencoded({ extended: true })); + app.use('/api/v2', v2Router); + await new Promise(resolve => { + server = app.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + baseUrl = `http://127.0.0.1:${typeof address === 'object' && address ? address.port : 0}`; +}); + +afterEach(() => { + config.hosted_apps_enabled = false; + config.session_workspace_enabled = false; + resetSessionWorkspaceStateForTests(); +}); + +afterAll(async () => { + await new Promise(resolve => server.close(() => resolve())); + config.hosted_apps_enabled = savedHostedAppsEnabled; + config.session_workspace_enabled = savedSessionWorkspaceEnabled; + resetSessionWorkspaceStateForTests(); +}); + +const post = (path: string, body: unknown = {}) => fetch(`${baseUrl}/api/v2${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), +}); + +describe('hosted-app route isolation', () => { + test('ordinary runner images do not expose hosted-app controls', async () => { + config.hosted_apps_enabled = false; + config.session_workspace_enabled = true; + + const response = await post('/hosted-app/start'); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ message: 'Not Found' }); + }); + + test('dedicated app hosts require the authenticated runtime-session binding', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + + const response = await post('/hosted-app/start'); + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ message: 'Missing runtime session header' }); + }); + + test('dedicated app hosts refuse ordinary execution requests', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + + const response = await post('/execute', {}); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ message: 'Not Found' }); + }); + + test('status is session-bound and reports absence without starting a process', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + + const response = await fetch(`${baseUrl}/api/v2/hosted-app/status`, { + headers: { 'X-Runtime-Session-Id': 'rt_hosted_demo' }, + }); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ + error: 'hosted_app_not_running', + message: 'No hosted app has been started', + }); + }); + + test('checkpoint creation uses the hosted-app workspace gate', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + const original = hostedAppSupervisor.withQuiescedWorkspace; + hostedAppSupervisor.withQuiescedWorkspace = async () => { + throw new HostedAppError( + 'hosted_app_workspace_busy', + 'the hosted app must be stopped before accessing its workspace', + 409, + ); + }; + + try { + const response = await fetch(`${baseUrl}/api/v2/session/checkpoint`, { + headers: { 'X-Runtime-Session-Id': 'rt_hosted_checkpoint' }, + }); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: 'hosted_app_workspace_busy', + message: 'the hosted app must be stopped before accessing its workspace', + }); + } finally { + hostedAppSupervisor.withQuiescedWorkspace = original; + } + }); +}); diff --git a/api/src/api/lifecycle.ts b/api/src/api/lifecycle.ts index 28977eda..1963f7f6 100644 --- a/api/src/api/lifecycle.ts +++ b/api/src/api/lifecycle.ts @@ -1,6 +1,7 @@ import express, { Router, type Request, type Response } from 'express'; import { logger } from '../logger'; import { bindSessionWorkspace, parseSessionBinding, unbindSessionWorkspace } from '../session-workspace'; +import { hostedAppSupervisor } from '../hosted-app'; /** * AWS Lambda MicroVM hook endpoints. The platform POSTs to @@ -97,7 +98,12 @@ lifecycleRouter.post('/suspend', ackHook('suspend')); lifecycleRouter.post('/terminate', (_req: Request, res: Response) => { logger.info({ hook: 'terminate' }, 'MicroVM lifecycle hook invoked'); - void unbindSessionWorkspace().catch((err) => logger.error({ err }, 'Failed to unbind session workspace on terminate')); + /* Stop before resetting the workspace so a resident process cannot race the + * recursive session cleanup. The platform does not need to wait for this + * best-effort cleanup before destroying the whole MicroVM. */ + void hostedAppSupervisor.shutdown() + .then(() => unbindSessionWorkspace()) + .catch((err) => logger.error({ err }, 'Failed to stop hosted app on terminate')); return res.status(200).json({ hook: 'terminate', status: 'ok' }); }); diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index b4895735..b0cc1a21 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -32,6 +32,10 @@ import { pruneInputCache, storeCachedInputs, } from '../session-inputs'; +import { + HostedAppError, + hostedAppSupervisor, +} from '../hosted-app'; const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; @@ -388,6 +392,16 @@ router.use((req: Request, res: Response, next: NextFunction) => { next(); }); +/* A hosted-app image is a dedicated process host, not an execution runner. + * Keeping `/execute` structurally unavailable prevents a resident app and an + * NsJail job from sharing the pinned session UID/workspace concurrently. */ +router.use('/execute', (_req: Request, res: Response, next: NextFunction) => { + if (config.hosted_apps_enabled) { + return res.status(404).json({ message: 'Not Found' }); + } + next(); +}); + /** Replay PTC payloads (user code + tool definitions + inlined * `_ptc_history.json` + pyplot assets) can far exceed Express's default * ~100kb body limit. The parser is installed *here* rather than globally @@ -677,15 +691,92 @@ router.get('/session/checkpoint', (req: Request, res: Response, next: NextFuncti if (failure) { return res.status(failure.status).json(failure.body); } - return streamSessionCheckpoint(res).catch(next); + const checkpoint = (): Promise => streamSessionCheckpoint(res); + return (config.hosted_apps_enabled + ? hostedAppSupervisor.withQuiescedWorkspace(checkpoint) + : checkpoint() + ).catch(error => hostedAppFailure(error, res, next)); }); router.post('/session/restore', (req: Request, res: Response, next: NextFunction) => { const failure = bindSessionFromHeader(req); if (failure) { return res.status(failure.status).json(failure.body); } - return restoreSessionCheckpoint(req, res).catch(next); + const restore = (): Promise => restoreSessionCheckpoint(req, res); + return (config.hosted_apps_enabled + ? hostedAppSupervisor.withQuiescedWorkspace(restore) + : restore() + ).catch(error => hostedAppFailure(error, res, next)); }); + +function requireHostedAppTarget( + _req: Request, + res: Response, + next: NextFunction, +): Response | void { + if (!config.hosted_apps_enabled) { + return res.status(404).json({ message: 'Not Found' }); + } + next(); +} + +function hostedAppFailure( + error: unknown, + res: Response, + next: NextFunction, +): Response | void { + if (error instanceof HostedAppError) { + return res.status(error.status).json({ error: error.code, message: error.message }); + } + next(error); +} + +/* Dedicated Lambda MicroVM resident-server adapter. The control plane first + * restores an immutable session checkpoint into this VM, then starts exactly + * one foreground process. Preview traffic uses a separate AWS token restricted + * to `hosted_app_port`; these control routes stay on the runner port. */ +router.post( + '/hosted-app/start', + requireHostedAppTarget, + express.json({ limit: '64kb' }), + (req: Request, res: Response, next: NextFunction) => { + const failure = bindSessionFromHeader(req); + if (failure) return res.status(failure.status).json(failure.body); + return hostedAppSupervisor.start(req.body) + .then(status => res.status(200).json(status)) + .catch(error => hostedAppFailure(error, res, next)); + }, +); + +router.get( + '/hosted-app/status', + requireHostedAppTarget, + (req: Request, res: Response) => { + const failure = bindSessionFromHeader(req); + if (failure) return res.status(failure.status).json(failure.body); + const status = hostedAppSupervisor.status(); + if (!status) { + return res.status(404).json({ + error: 'hosted_app_not_running', + message: 'No hosted app has been started', + }); + } + return res.status(200).json(status); + }, +); + +router.post( + '/hosted-app/stop', + requireHostedAppTarget, + express.json({ limit: '1kb' }), + (req: Request, res: Response, next: NextFunction) => { + const failure = bindSessionFromHeader(req); + if (failure) return res.status(failure.status).json(failure.body); + return hostedAppSupervisor.stop() + .then(status => status ? res.status(200).json(status) : res.status(204).send()) + .catch(error => hostedAppFailure(error, res, next)); + }, +); /** * Input delivery for backends whose sandbox cannot reach the file server. * diff --git a/api/src/config.ts b/api/src/config.ts index d824c0db..79c0e078 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -71,6 +71,31 @@ export const config = { * session mode. An enabled runner additionally binds each request to a * workspace through the authenticated X-Runtime-Session-Id header. */ session_workspace_enabled: (process.env.SANDBOX_SESSION_WORKSPACE_ENABLED ?? 'false') === 'true', + /** + * Enables the Lambda-only hosted-app runner surface. This must only be set + * on a dedicated app-host MicroVM image: user application processes share + * that VM's network namespace and are therefore intentionally never started + * by the ordinary stateless/session execution runner. + */ + hosted_apps_enabled: (process.env.SANDBOX_HOSTED_APPS_ENABLED ?? 'false') === 'true', + hosted_app_port: safeInt(process.env.SANDBOX_HOSTED_APP_PORT, 3000), + hosted_app_start_timeout_ms: safeInt( + process.env.SANDBOX_HOSTED_APP_START_TIMEOUT_MS, + 30_000, + ), + hosted_app_stop_timeout_ms: safeInt( + process.env.SANDBOX_HOSTED_APP_STOP_TIMEOUT_MS, + 5_000, + ), + hosted_app_log_max_bytes: safeInt( + process.env.SANDBOX_HOSTED_APP_LOG_MAX_BYTES, + 64 * 1024, + ), + hosted_app_memory_max_bytes: safeInt( + process.env.SANDBOX_HOSTED_APP_MEMORY_MAX_BYTES, + 2 * 1024 * 1024 * 1024, + ), + hosted_app_pids_max: safeInt(process.env.SANDBOX_HOSTED_APP_PIDS_MAX, 128), job_uid_base: safeInt(process.env.SANDBOX_JOB_UID_BASE, 200000), job_gid_base: safeInt(process.env.SANDBOX_JOB_GID_BASE, 200000), job_uid_count: safeInt( diff --git a/api/src/hosted-app-launcher.sh b/api/src/hosted-app-launcher.sh new file mode 100644 index 00000000..ea866ed3 --- /dev/null +++ b/api/src/hosted-app-launcher.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -lt 5 ]; then + echo "usage: hosted-app-launcher [args...]" >&2 + exit 64 +fi + +CGROUP_PATH="$1" +APP_UID="$2" +APP_GID="$3" +shift 3 + +# This wrapper starts as root, joins the root-owned cgroup before any user code +# can fork, then irreversibly drops identity and capabilities. App descendants +# inherit the cgroup even if they daemonize or create a new process group. +printf '%s' "$$" > "${CGROUP_PATH}/cgroup.procs" +exec /usr/bin/setpriv \ + --no-new-privs \ + --reuid "$APP_UID" \ + --regid "$APP_GID" \ + --clear-groups \ + --inh-caps=-all \ + --ambient-caps=-all \ + --bounding-set=-all \ + -- "$@" diff --git a/api/src/hosted-app.test.ts b/api/src/hosted-app.test.ts new file mode 100644 index 00000000..727803d1 --- /dev/null +++ b/api/src/hosted-app.test.ts @@ -0,0 +1,565 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import * as fsp from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { PassThrough } from 'node:stream'; +import type { SpawnOptions } from 'node:child_process'; +import type { Runtime } from './runtime'; +import type { SessionWorkspace } from './session-workspace'; +import { + HostedAppError, + HostedAppSupervisor, + prepareHostedAppRuntimeWorkspace, + type HostedAppDependencies, + type HostedAppStartRequest, +} from './hosted-app'; +import { config } from './config'; + +interface FakeChild extends EventEmitter { + pid: number; + stdout: PassThrough; + stderr: PassThrough; +} + +const savedConfig = { + port: config.hosted_app_port, + start: config.hosted_app_start_timeout_ms, + stop: config.hosted_app_stop_timeout_ms, + logs: config.hosted_app_log_max_bytes, + packages: config.packages_directory, +}; + +let roots: string[] = []; + +beforeEach(() => { + config.hosted_app_port = 3123; + config.hosted_app_start_timeout_ms = 25; + config.hosted_app_stop_timeout_ms = 25; + config.hosted_app_log_max_bytes = 64; +}); + +afterEach(async () => { + config.hosted_app_port = savedConfig.port; + config.hosted_app_start_timeout_ms = savedConfig.start; + config.hosted_app_stop_timeout_ms = savedConfig.stop; + config.hosted_app_log_max_bytes = savedConfig.logs; + config.packages_directory = savedConfig.packages; + await Promise.all(roots.map(root => fsp.rm(root, { recursive: true, force: true }))); + roots = []; +}); + +async function workspace(): Promise { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), 'hosted-app-')); + roots.push(root); + await fsp.writeFile(path.join(root, 'server.js'), 'serve();'); + await fsp.mkdir(path.join(root, 'app')); + return root; +} + +function fakeRuntime(): Runtime { + return { + language: 'node', + version: { raw: '22.0.0' } as Runtime['version'], + aliases: [], + pkgdir: '/pkgs/node/22', + compiled: false, + env_vars: { PATH: '/pkgs/node/22/bin:/usr/bin' }, + timeouts: { compile: 0, run: 0 }, + cpu_times: { compile: 0, run: 0 }, + memory_limits: { compile: 0, run: 0 }, + max_process_count: 64, + max_open_files: 2048, + max_file_size: 10_000_000, + output_max_size: 1024, + }; +} + +function fakeChild(pid = 4242): FakeChild { + const child = new EventEmitter() as FakeChild; + child.pid = pid; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + return child; +} + +function request(overrides: Partial = {}): HostedAppStartRequest { + return { + app_id: 'demo', + revision: 'rev-1', + language: 'node', + version: '>=22', + entrypoint: 'server.js', + ...overrides, + }; +} + +function dependencies( + root: string, + options: { + probe?: boolean; + probePort?: () => Promise; + guardError?: Error; + killCgroup?: () => Promise; + runtime?: Runtime; + runtimes?: Runtime[]; + } = {}, +): { + deps: HostedAppDependencies; + spawns: Array<{ command: string; args: readonly string[]; options: SpawnOptions }>; + guards: number[]; + cgroupKills: string[]; + kills: Array<{ pid: number; signal: NodeJS.Signals }>; + children: FakeChild[]; + preparations: Array<{ runtime: Runtime; nodeModulesPath?: string }>; +} { + const spawns: Array<{ command: string; args: readonly string[]; options: SpawnOptions }> = []; + const guards: number[] = []; + const cgroupKills: string[] = []; + const kills: Array<{ pid: number; signal: NodeJS.Signals }> = []; + const children: FakeChild[] = []; + const preparations: Array<{ runtime: Runtime; nodeModulesPath?: string }> = []; + const getSession = () => ({ + ownership: async () => ({ dir: root, uid: 200123, gid: 200123 }), + } as SessionWorkspace); + const deps: HostedAppDependencies = { + getSession, + resolveRuntime: () => options.runtime === undefined ? fakeRuntime() : options.runtime, + listRuntimes: () => options.runtimes ?? [options.runtime ?? fakeRuntime()], + prepareRuntimeWorkspace: async (_workspaceDir, runtime, nodeModulesPath) => { + preparations.push({ runtime, nodeModulesPath }); + }, + spawnApp: ((command: string, args: readonly string[], spawnOptions: SpawnOptions) => { + spawns.push({ command, args, options: spawnOptions }); + const child = fakeChild(4242 + children.length); + children.push(child); + return child as unknown as ReturnType; + }) as HostedAppDependencies['spawnApp'], + prepareCgroup: async () => {}, + killCgroup: async () => { + cgroupKills.push('kill'); + await options.killCgroup?.(); + }, + installNetworkGuard: async uid => { + guards.push(uid); + if (options.guardError) throw options.guardError; + }, + probePort: options.probePort ?? (async () => options.probe ?? true), + killProcessGroup: (pid, signal) => { + kills.push({ pid, signal }); + const child = children.find(candidate => candidate.pid === pid); + queueMicrotask(() => child?.emit('exit', null, signal)); + }, + now: () => new Date('2026-08-21T12:00:00.000Z'), + }; + return { deps, spawns, guards, cgroupKills, kills, children, preparations }; +} + +describe('HostedAppSupervisor', () => { + test('starts a runtime as the session UID with a curated fixed-port environment', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + + const status = await supervisor.start(request({ + args: ['--production'], + env: { + APP_NAME: 'demo', + HOME: '/attacker', + port: '9999', + HOST: 'attacker.invalid', + LD_PRELOAD: '/tmp/evil.so', + }, + })); + + expect(status).toMatchObject({ + app_id: 'demo', + revision: 'rev-1', + state: 'running', + port: 3123, + pid: 4242, + }); + expect(fixture.guards).toEqual([200123]); + expect(fixture.spawns).toHaveLength(1); + const launch = fixture.spawns[0]; + expect(launch.command).toBe('/usr/local/bin/codeapi-hosted-app-launcher'); + const realRoot = await fsp.realpath(root); + expect(launch.args).toEqual([ + '/sys/fs/cgroup/codeapi_hosted_app', + '200123', + '200123', + '/bin/bash', + '/pkgs/node/22/run', + path.join(realRoot, 'server.js'), + '--production', + ]); + expect(launch.options).toMatchObject({ + cwd: realRoot, + detached: true, + }); + expect(launch.options.env).toMatchObject({ + APP_NAME: 'demo', + HOME: root, + HOST: '0.0.0.0', + PORT: '3123', + PATH: '/pkgs/node/22/bin:/usr/bin', + }); + expect(launch.options.env).not.toHaveProperty('port'); + expect(launch.options.env).not.toHaveProperty('LD_PRELOAD'); + await supervisor.shutdown(); + }); + + test('gives Bash hosted apps access to curated packaged runtimes', async () => { + const root = await workspace(); + const bash = { + ...fakeRuntime(), + language: 'bash', + pkgdir: '/pkgs/bash/5', + env_vars: { PATH: '/pkgs/bash/5/bin:/usr/bin' }, + }; + const node = { + ...fakeRuntime(), + env_vars: { + PATH: '/pkgs/node/22/bin:/usr/bin', + NODE_PATH: '/pkgs/node/22/node_modules', + }, + }; + const fixture = dependencies(root, { runtime: bash, runtimes: [bash, node] }); + const supervisor = new HostedAppSupervisor(fixture.deps); + + await supervisor.start(request({ language: 'bash', version: '>=5' })); + + expect(fixture.spawns[0].options.env).toMatchObject({ + PATH: '/pkgs/node/22/bin:/pkgs/bash/5/bin:/usr/bin', + NODE_PATH: '/pkgs/node/22/node_modules', + }); + expect(fixture.preparations).toEqual([{ + runtime: bash, + nodeModulesPath: '/pkgs/node/22/node_modules', + }]); + await supervisor.shutdown(); + }); + + test('is idempotent for the exact same immutable revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + const spec = request({ env: { B: '2', A: '1' } }); + + const first = await supervisor.start(spec); + const second = await supervisor.start(request({ env: { A: '1', B: '2' } })); + + expect(second).toEqual(first); + expect(fixture.spawns).toHaveLength(1); + await supervisor.shutdown(); + }); + + test('rejects changed launch settings under an existing revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + const error = await supervisor.start(request({ args: ['changed'] })).catch(value => value); + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_revision_conflict'); + expect(fixture.spawns).toHaveLength(1); + await supervisor.shutdown(); + }); + + test('preserves revision immutability after stop and replacement', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + await supervisor.stop(); + + const afterStop = await supervisor.start(request({ args: ['changed'] })).catch(value => value); + expect(afterStop).toBeInstanceOf(HostedAppError); + expect(afterStop.code).toBe('hosted_app_revision_conflict'); + + await supervisor.start(request()); + await supervisor.start(request({ revision: 'rev-2' })); + const afterReplacement = await supervisor.start( + request({ args: ['changed'] }), + ).catch(value => value); + expect(afterReplacement).toBeInstanceOf(HostedAppError); + expect(afterReplacement.code).toBe('hosted_app_revision_conflict'); + expect(fixture.spawns).toHaveLength(3); + await supervisor.shutdown(); + }); + + test('stops the old process group before launching a new revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + const status = await supervisor.start(request({ revision: 'rev-2' })); + + expect(status.revision).toBe('rev-2'); + expect(fixture.kills).toEqual([{ pid: 4242, signal: 'SIGTERM' }]); + expect(fixture.cgroupKills.length).toBeGreaterThan(0); + expect(fixture.spawns).toHaveLength(2); + await supervisor.shutdown(); + }); + + test('validates a replacement before stopping the running revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + fixture.deps.resolveRuntime = () => undefined; + + const error = await supervisor.start(request({ revision: 'rev-2' })).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_runtime_not_found'); + expect(supervisor.status()?.state).toBe('running'); + expect(fixture.kills).toEqual([]); + await supervisor.shutdown(); + }); + + test('does not report running when the child exits during its readiness probe', async () => { + const root = await workspace(); + let finishProbe!: (ready: boolean) => void; + const probe = new Promise(resolve => { finishProbe = resolve; }); + const fixture = dependencies(root, { probePort: () => probe }); + const supervisor = new HostedAppSupervisor(fixture.deps); + const started = supervisor.start(request()); + while (fixture.children.length === 0) { + await new Promise(resolve => setTimeout(resolve, 0)); + } + + fixture.children[0].emit('exit', 1, null); + finishProbe(true); + const error = await started.catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_start_failed'); + expect(supervisor.status()?.state).toBe('failed'); + }); + + test('serializes quiesced workspace access and rejects it while an app is running', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + let mutated = false; + + const error = await supervisor.withQuiescedWorkspace(async () => { + mutated = true; + }).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_workspace_busy'); + expect(mutated).toBe(false); + await supervisor.shutdown(); + }); + + test('links bundled JavaScript packages into the hosted workspace', async () => { + const root = await workspace(); + const packages = await fsp.mkdtemp(path.join(os.tmpdir(), 'hosted-app-packages-')); + roots.push(packages); + config.packages_directory = packages; + const packageRoot = path.join(packages, 'node', '22'); + await fsp.mkdir(path.join(packageRoot, 'node_modules'), { recursive: true }); + const runtime = { ...fakeRuntime(), pkgdir: packageRoot }; + + await prepareHostedAppRuntimeWorkspace(root, runtime); + + expect(await fsp.realpath(path.join(root, 'node_modules'))).toBe( + await fsp.realpath(path.join(packageRoot, 'node_modules')), + ); + }); + + test('refreshes a supervisor-managed package link for a new runtime', async () => { + const root = await workspace(); + const packages = await fsp.mkdtemp(path.join(os.tmpdir(), 'hosted-app-packages-')); + roots.push(packages); + config.packages_directory = packages; + const firstRoot = path.join(packages, 'node', '22'); + const secondRoot = path.join(packages, 'bun', '1'); + await fsp.mkdir(path.join(firstRoot, 'node_modules'), { recursive: true }); + await fsp.mkdir(path.join(secondRoot, 'node_modules'), { recursive: true }); + + await prepareHostedAppRuntimeWorkspace(root, { ...fakeRuntime(), pkgdir: firstRoot }); + await prepareHostedAppRuntimeWorkspace(root, { ...fakeRuntime(), pkgdir: secondRoot }); + + expect(await fsp.realpath(path.join(root, 'node_modules'))).toBe( + await fsp.realpath(path.join(secondRoot, 'node_modules')), + ); + }); + + test('waits for confirmed cgroup cleanup before completing stop', async () => { + const root = await workspace(); + let releaseCleanup!: () => void; + const cleanupBlocked = new Promise(resolve => { releaseCleanup = resolve; }); + const fixture = dependencies(root, { killCgroup: () => cleanupBlocked }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + let stopped = false; + const stopping = supervisor.stop().then(() => { stopped = true; }); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(stopped).toBe(false); + + releaseCleanup(); + await stopping; + expect(stopped).toBe(true); + }); + + test('classifies a stop cleanup failure as a retryable server error', async () => { + const root = await workspace(); + let permitCleanup = false; + const fixture = dependencies(root, { + killCgroup: async () => { + if (!permitCleanup) throw new Error('cgroup remains populated'); + }, + }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + const error = await supervisor.stop().catch(value => value); + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_cleanup_failed'); + expect(error.status).toBe(503); + + permitCleanup = true; + await supervisor.shutdown(); + }); + + test('fails workspace mutation closed until a failed app cgroup is drained', async () => { + const root = await workspace(); + let permitCleanup = false; + const fixture = dependencies(root, { + killCgroup: async () => { + if (!permitCleanup) throw new Error('cgroup remains populated'); + }, + }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + fixture.children[0].emit('exit', 1, null); + await new Promise(resolve => setTimeout(resolve, 0)); + let mutated = false; + + const error = await supervisor.withQuiescedWorkspace(async () => { + mutated = true; + }).catch(value => value); + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_cleanup_failed'); + expect(mutated).toBe(false); + + permitCleanup = true; + await supervisor.withQuiescedWorkspace(async () => { mutated = true; }); + expect(mutated).toBe(true); + }); + + test('skips queued exit cleanup after a replacement becomes active', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + let releaseOwnership!: (value: { dir: string; uid: number; gid: number }) => void; + const ownershipBlocked = new Promise<{ dir: string; uid: number; gid: number }>( + resolve => { releaseOwnership = resolve; }, + ); + fixture.deps.getSession = () => ({ + ownership: () => ownershipBlocked, + } as SessionWorkspace); + + const replacement = supervisor.start(request({ revision: 'rev-2' })); + await new Promise(resolve => setTimeout(resolve, 0)); + fixture.children[0].emit('exit', 1, null); + releaseOwnership({ dir: root, uid: 200123, gid: 200123 }); + + expect((await replacement).revision).toBe('rev-2'); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(supervisor.status()?.state).toBe('running'); + expect(fixture.cgroupKills).toHaveLength(1); + await supervisor.shutdown(); + }); + + test('fails closed before spawning when the network guard cannot be installed', async () => { + const root = await workspace(); + const fixture = dependencies(root, { guardError: new Error('iptables unavailable') }); + const supervisor = new HostedAppSupervisor(fixture.deps); + + const error = await supervisor.start(request()).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_isolation_failed'); + expect(fixture.spawns).toHaveLength(0); + }); + + test('rejects symlink entrypoints even when the target is a regular file', async () => { + const root = await workspace(); + const outside = await fsp.mkdtemp(path.join(os.tmpdir(), 'hosted-app-outside-')); + roots.push(outside); + await fsp.writeFile(path.join(outside, 'outside.js'), 'steal();'); + await fsp.symlink(path.join(outside, 'outside.js'), path.join(root, 'linked.js')); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + + const error = await supervisor.start(request({ entrypoint: 'linked.js' })).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_path_escape'); + expect(fixture.guards).toHaveLength(0); + expect(fixture.spawns).toHaveLength(0); + }); + + test('retains only the bounded tail of process logs', async () => { + const root = await workspace(); + config.hosted_app_log_max_bytes = 8; + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + fixture.children[0].stdout.write('0123456789'); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(supervisor.status()?.stdout).toBe('23456789'); + await supervisor.shutdown(); + }); + + test('reaps the cgroup when the tracked parent exits unexpectedly', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + fixture.children[0].emit('exit', 1, null); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(supervisor.status()?.state).toBe('failed'); + expect(fixture.cgroupKills.length).toBeGreaterThan(0); + await supervisor.shutdown(); + }); + + test('waits for unexpected-exit cleanup before launching a replacement revision', async () => { + const root = await workspace(); + let releaseCleanup!: () => void; + const cleanupBlocked = new Promise(resolve => { releaseCleanup = resolve; }); + let cleanupCalls = 0; + const fixture = dependencies(root, { + killCgroup: async () => { + cleanupCalls += 1; + if (cleanupCalls === 1) await cleanupBlocked; + }, + }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + fixture.children[0].emit('exit', 1, null); + await Promise.resolve(); + const replacement = supervisor.start(request({ revision: 'rev-2' })); + await Promise.resolve(); + expect(fixture.spawns).toHaveLength(1); + + releaseCleanup(); + expect((await replacement).revision).toBe('rev-2'); + expect(fixture.spawns).toHaveLength(2); + await supervisor.shutdown(); + }); +}); diff --git a/api/src/hosted-app.ts b/api/src/hosted-app.ts new file mode 100644 index 00000000..d8cbfc50 --- /dev/null +++ b/api/src/hosted-app.ts @@ -0,0 +1,796 @@ +import { execFile, spawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import * as fsp from 'node:fs/promises'; +import * as net from 'node:net'; +import * as path from 'node:path'; +import type { Readable } from 'node:stream'; +import { promisify } from 'node:util'; +import { config } from './config'; +import { aggregateBashExtras, filterExtraEnvVars } from './job'; +import { logger } from './logger'; +import { getLatestRuntimeMatchingLanguageVersion, getRuntimes, type Runtime } from './runtime'; +import { getBoundSessionWorkspace, type SessionWorkspace } from './session-workspace'; +import { ValidationError, validateFilePath } from './validation'; + +const execFileAsync = promisify(execFile); +const APP_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +const REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const HOSTED_APP_EGRESS_CHAIN = 'CODEAPI_HOSTED_APP_EGRESS'; +/* Sibling of sandbox_api, not its child: the API process lives in sandbox_api + * and cgroup v2 forbids enabling domain controllers below a populated parent. */ +const HOSTED_APP_CGROUP = '/sys/fs/cgroup/codeapi_hosted_app'; +const HOSTED_APP_LAUNCHER = '/usr/local/bin/codeapi-hosted-app-launcher'; +const MAX_ARGS = 64; +const MAX_ARG_BYTES = 4096; +const MAX_ENV_VARS = 64; +const MAX_ENV_VALUE_BYTES = 4096; +const MAX_ENV_BYTES = 32 * 1024; +const MAX_TRACKED_APP_REVISIONS = 1024; +const PROBE_INTERVAL_MS = 100; + +export interface HostedAppStartRequest { + app_id: string; + revision: string; + language: string; + version: string; + entrypoint: string; + cwd?: string; + args?: string[]; + env?: Record; +} + +interface NormalizedHostedAppRequest extends HostedAppStartRequest { + cwd: string; + args: string[]; + env: Record; +} + +export type HostedAppState = 'starting' | 'running' | 'stopping' | 'stopped' | 'failed'; + +export interface HostedAppStatus { + app_id: string; + revision: string; + state: HostedAppState; + port: number; + pid?: number; + started_at: string; + exited_at?: string; + exit_code?: number; + signal?: NodeJS.Signals; + message?: string; + stdout: string; + stderr: string; +} + +export class HostedAppError extends Error { + constructor( + readonly code: string, + message: string, + readonly status: number, + ) { + super(message); + this.name = 'HostedAppError'; + } +} + +interface ActiveHostedApp { + request: NormalizedHostedAppRequest; + specKey: string; + cgroupDrained: boolean; + process?: HostedAppChild; + status: HostedAppStatus; +} + +interface HostedAppChild extends ChildProcess { + readonly pid: number; + readonly stdout: Readable; + readonly stderr: Readable; +} + +type SpawnApp = (command: string, args: readonly string[], options: SpawnOptions) => HostedAppChild; +type ResolveRuntime = typeof getLatestRuntimeMatchingLanguageVersion; + +export interface HostedAppDependencies { + getSession: () => SessionWorkspace | undefined; + resolveRuntime: ResolveRuntime; + listRuntimes: () => Runtime[]; + prepareRuntimeWorkspace: ( + workspaceDir: string, + runtime: Runtime, + nodeModulesPath?: string, + ) => Promise; + spawnApp: SpawnApp; + prepareCgroup: () => Promise; + killCgroup: () => Promise; + installNetworkGuard: (uid: number) => Promise; + probePort: (port: number) => Promise; + killProcessGroup: (pid: number, signal: NodeJS.Signals) => void; + now: () => Date; +} + +export async function prepareHostedAppRuntimeWorkspace( + workspaceDir: string, + runtime: Runtime, + nodeModulesPath?: string, +): Promise { + const packageModules = nodeModulesPath ?? path.join(runtime.pkgdir, 'node_modules'); + const packageStat = await fsp.stat(packageModules).catch(error => { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + }); + if (!packageStat?.isDirectory()) return; + + const workspaceModules = path.join(workspaceDir, 'node_modules'); + let workspaceModulesStat: Awaited> | undefined; + try { + workspaceModulesStat = await fsp.lstat(workspaceModules); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + if (workspaceModulesStat && !workspaceModulesStat.isSymbolicLink()) return; + if (workspaceModulesStat?.isSymbolicLink()) { + const linkTarget = await fsp.readlink(workspaceModules); + const resolvedTarget = path.resolve(workspaceDir, linkTarget); + const packageRoot = path.resolve(config.packages_directory); + if (!isInside(packageRoot, resolvedTarget)) return; + if (resolvedTarget === packageModules) return; + await fsp.unlink(workspaceModules); + } + try { + await fsp.symlink(packageModules, workspaceModules, 'dir'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } +} + +function byteLength(value: string): number { + return Buffer.byteLength(value, 'utf8'); +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const proto = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +function normalizeHostedAppRequest(value: unknown): NormalizedHostedAppRequest { + if (!isPlainObject(value)) { + throw new HostedAppError('invalid_hosted_app_request', 'request body must be an object', 400); + } + + const stringField = (name: string): string => { + const field = value[name]; + if (typeof field !== 'string' || field.length === 0) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `${name} must be a non-empty string`, + 400, + ); + } + if (field.includes('\0')) { + throw new HostedAppError('invalid_hosted_app_request', `${name} must not contain NUL`, 400); + } + return field; + }; + + const appId = stringField('app_id'); + const revision = stringField('revision'); + const language = stringField('language'); + const version = stringField('version'); + const entrypoint = stringField('entrypoint'); + if (!APP_ID_PATTERN.test(appId)) { + throw new HostedAppError('invalid_hosted_app_request', 'app_id is malformed', 400); + } + if (!REVISION_PATTERN.test(revision)) { + throw new HostedAppError('invalid_hosted_app_request', 'revision is malformed', 400); + } + try { + validateFilePath(entrypoint, '/tmp/codeapi-hosted-app-validation'); + } catch (error) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `entrypoint is invalid: ${error instanceof Error ? error.message : 'invalid path'}`, + 400, + ); + } + + const cwdValue = value.cwd ?? '.'; + if (typeof cwdValue !== 'string' || cwdValue.includes('\0')) { + throw new HostedAppError('invalid_hosted_app_request', 'cwd must be a string', 400); + } + if (cwdValue !== '.') { + try { + /* validateFilePath is also the canonical relative-path validator. A cwd + * is allowed to name a directory; append a sentinel so trailing-slash + * and directory-root cases retain the same traversal checks. */ + validateFilePath(path.posix.join(cwdValue, '.codeapi-cwd'), '/tmp/codeapi-hosted-app-validation'); + if (path.posix.normalize(cwdValue) !== cwdValue || cwdValue.endsWith('/')) { + throw new ValidationError('cwd must be a canonical relative path'); + } + } catch (error) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `cwd is invalid: ${error instanceof Error ? error.message : 'invalid path'}`, + 400, + ); + } + } + + const argsValue = value.args ?? []; + if ( + !Array.isArray(argsValue) + || argsValue.length > MAX_ARGS + || argsValue.some(arg => ( + typeof arg !== 'string' + || arg.includes('\0') + || byteLength(arg) > MAX_ARG_BYTES + )) + ) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `args must contain at most ${MAX_ARGS} bounded strings`, + 400, + ); + } + + const envValue = value.env ?? {}; + if (!isPlainObject(envValue) || Object.keys(envValue).length > MAX_ENV_VARS) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `env must be an object with at most ${MAX_ENV_VARS} entries`, + 400, + ); + } + const env: Record = {}; + let envBytes = 0; + for (const [key, raw] of Object.entries(envValue)) { + if ( + !ENV_NAME_PATTERN.test(key) + || typeof raw !== 'string' + || raw.includes('\0') + || byteLength(raw) > MAX_ENV_VALUE_BYTES + ) { + throw new HostedAppError('invalid_hosted_app_request', `env.${key} is invalid`, 400); + } + envBytes += byteLength(key) + byteLength(raw); + if (envBytes > MAX_ENV_BYTES) { + throw new HostedAppError('invalid_hosted_app_request', 'env is too large', 400); + } + env[key] = raw; + } + + return { + app_id: appId, + revision, + language, + version, + entrypoint, + cwd: cwdValue, + args: [...argsValue] as string[], + env, + }; +} + +function canonicalSpecKey(request: NormalizedHostedAppRequest): string { + const canonical = JSON.stringify({ + ...request, + env: Object.fromEntries(Object.entries(request.env).sort(([a], [b]) => a.localeCompare(b))), + }); + return createHash('sha256').update(canonical).digest('hex'); +} + +function isInside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`)); +} + +async function resolveWorkspacePaths( + workspaceDir: string, + request: NormalizedHostedAppRequest, +): Promise<{ cwd: string; entrypoint: string }> { + const realRoot = await fsp.realpath(workspaceDir); + const cwd = await fsp.realpath(path.resolve(realRoot, request.cwd)).catch(() => { + throw new HostedAppError('hosted_app_cwd_missing', 'cwd does not exist', 400); + }); + if (!isInside(realRoot, cwd)) { + throw new HostedAppError('hosted_app_path_escape', 'cwd escapes the session workspace', 400); + } + const cwdStat = await fsp.stat(cwd); + if (!cwdStat.isDirectory()) { + throw new HostedAppError('hosted_app_cwd_missing', 'cwd is not a directory', 400); + } + + const requestedEntrypoint = path.resolve(realRoot, request.entrypoint); + const entrypointLstat = await fsp.lstat(requestedEntrypoint).catch(() => { + throw new HostedAppError('hosted_app_entrypoint_missing', 'entrypoint does not exist', 400); + }); + if (entrypointLstat.isSymbolicLink()) { + throw new HostedAppError('hosted_app_path_escape', 'entrypoint must not be a symbolic link', 400); + } + const entrypoint = await fsp.realpath(requestedEntrypoint); + if (!isInside(realRoot, entrypoint)) { + throw new HostedAppError('hosted_app_path_escape', 'entrypoint escapes the session workspace', 400); + } + const entrypointStat = await fsp.stat(entrypoint); + if (!entrypointStat.isFile()) { + throw new HostedAppError('hosted_app_entrypoint_missing', 'entrypoint is not a file', 400); + } + return { cwd, entrypoint }; +} + +function appendBounded(current: string, chunk: Buffer | string): string { + const next = current + chunk.toString(); + const bytes = Buffer.byteLength(next); + if (bytes <= config.hosted_app_log_max_bytes) return next; + return Buffer.from(next).subarray(bytes - config.hosted_app_log_max_bytes).toString(); +} + +async function commandSucceeds(binary: string, args: string[]): Promise { + try { + await execFileAsync(binary, args); + return true; + } catch { + return false; + } +} + +/** + * Hosted apps receive inbound preview traffic, but may not initiate network + * connections. Besides blocking internet egress, this prevents the untrusted + * app UID from calling the root-owned control listener on localhost. Reply + * packets for accepted inbound connections remain allowed by conntrack. + */ +export async function installHostedAppNetworkGuard(uid: number): Promise { + for (const binary of ['/usr/sbin/iptables', '/usr/sbin/ip6tables']) { + if (!(await commandSucceeds(binary, ['-w', '5', '-L', HOSTED_APP_EGRESS_CHAIN]))) { + await execFileAsync(binary, ['-w', '5', '-N', HOSTED_APP_EGRESS_CHAIN]); + } + await execFileAsync(binary, ['-w', '5', '-F', HOSTED_APP_EGRESS_CHAIN]); + await execFileAsync(binary, [ + '-w', '5', '-A', HOSTED_APP_EGRESS_CHAIN, + '-m', 'conntrack', '--ctstate', 'ESTABLISHED,RELATED', '-j', 'ACCEPT', + ]); + await execFileAsync(binary, ['-w', '5', '-A', HOSTED_APP_EGRESS_CHAIN, '-j', 'REJECT']); + const jump = [ + '-m', 'owner', '--uid-owner', String(uid), '-j', HOSTED_APP_EGRESS_CHAIN, + ]; + if (!(await commandSucceeds(binary, ['-w', '5', '-C', 'OUTPUT', ...jump]))) { + await execFileAsync(binary, ['-w', '5', '-I', 'OUTPUT', '1', ...jump]); + } + } +} + +/** Create a process-tree boundary owned only by the root runner. The launcher + * moves itself here before dropping to the session UID, so every descendant + * inherits the cgroup and cannot escape it by daemonizing or calling setsid. */ +export async function prepareHostedAppCgroup(): Promise { + await fsp.mkdir(HOSTED_APP_CGROUP, { recursive: true }); + /* cgroup.kill (Linux 5.14+) is required, not an optional optimization: it is + * the primitive that prevents a setsid()/double-fork descendant escaping + * revision replacement. Fail closed on kernels that do not expose it. */ + await fsp.access(path.join(HOSTED_APP_CGROUP, 'cgroup.kill')); + await killHostedAppCgroup(); + await fsp.writeFile(path.join(HOSTED_APP_CGROUP, 'memory.max'), String( + config.hosted_app_memory_max_bytes, + )); + await fsp.writeFile(path.join(HOSTED_APP_CGROUP, 'pids.max'), String( + config.hosted_app_pids_max, + )); +} + +/** `cgroup.kill` reaches descendants that changed session/process group. */ +export async function killHostedAppCgroup(): Promise { + try { + await fsp.writeFile(path.join(HOSTED_APP_CGROUP, 'cgroup.kill'), '1'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + const deadline = Date.now() + config.hosted_app_stop_timeout_ms; + while (true) { + const events = await fsp.readFile( + path.join(HOSTED_APP_CGROUP, 'cgroup.events'), + 'utf8', + ); + if (/^populated 0$/m.test(events)) return; + if (Date.now() >= deadline) { + throw new Error('hosted-app cgroup did not become empty'); + } + await new Promise(resolve => setTimeout(resolve, 25)); + } +} + +export function probeHostedAppPort(port: number): Promise { + return new Promise(resolve => { + const socket = net.createConnection({ host: '127.0.0.1', port }); + socket.setTimeout(500); + socket.once('connect', () => { + socket.destroy(); + resolve(true); + }); + const fail = (): void => { + socket.destroy(); + resolve(false); + }; + socket.once('error', fail); + socket.once('timeout', fail); + }); +} + +function killHostedAppProcessGroup(pid: number, signal: NodeJS.Signals): void { + try { + process.kill(-pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + } +} + +function publicStatus(active: ActiveHostedApp): HostedAppStatus { + return { ...active.status }; +} + +export class HostedAppSupervisor { + private active: ActiveHostedApp | undefined; + private readonly revisionSpecs = new Map(); + private transition: Promise = Promise.resolve(); + + constructor(private readonly deps: HostedAppDependencies = { + getSession: getBoundSessionWorkspace, + resolveRuntime: getLatestRuntimeMatchingLanguageVersion, + listRuntimes: getRuntimes, + prepareRuntimeWorkspace: prepareHostedAppRuntimeWorkspace, + spawnApp: (command, args, options) => spawn(command, args, options) as unknown as HostedAppChild, + prepareCgroup: prepareHostedAppCgroup, + killCgroup: killHostedAppCgroup, + installNetworkGuard: installHostedAppNetworkGuard, + probePort: probeHostedAppPort, + killProcessGroup: killHostedAppProcessGroup, + now: () => new Date(), + }) {} + + status(): HostedAppStatus | undefined { + return this.active ? publicStatus(this.active) : undefined; + } + + async start(rawRequest: unknown): Promise { + return this.serialize(() => this.startImpl(rawRequest)); + } + + async stop(): Promise { + return this.serialize(async () => { + try { + return await this.stopImpl(); + } catch (error) { + logger.error({ err: error }, 'Hosted-app stop cleanup failed'); + throw new HostedAppError( + 'hosted_app_cleanup_failed', + 'the hosted app could not be stopped safely', + 503, + ); + } + }); + } + + async shutdown(): Promise { + await this.stop(); + } + + async withQuiescedWorkspace(operation: () => Promise): Promise { + return this.serialize(async () => { + const state = this.active?.status.state; + if (state === 'starting' || state === 'running' || state === 'stopping') { + throw new HostedAppError( + 'hosted_app_workspace_busy', + 'the hosted app must be stopped before accessing its workspace', + 409, + ); + } + if (this.active && !this.active.cgroupDrained) { + try { + await this.deps.killCgroup(); + this.active.cgroupDrained = true; + } catch (error) { + logger.error( + { err: error, appId: this.active.request.app_id }, + 'Hosted-app cgroup cleanup failed before workspace mutation', + ); + throw new HostedAppError( + 'hosted_app_cleanup_failed', + 'the hosted app workspace is not safe to replace', + 503, + ); + } + } + return operation(); + }); + } + + private async serialize(operation: () => Promise): Promise { + const previous = this.transition; + let release!: () => void; + this.transition = new Promise(resolve => { release = resolve; }); + await previous; + try { + return await operation(); + } finally { + release(); + } + } + + private async startImpl(rawRequest: unknown): Promise { + const request = normalizeHostedAppRequest(rawRequest); + const specKey = canonicalSpecKey(request); + const revisionKey = `${request.app_id}\0${request.revision}`; + const rememberedSpec = this.revisionSpecs.get(revisionKey); + if (rememberedSpec !== undefined && rememberedSpec !== specKey) { + throw new HostedAppError( + 'hosted_app_revision_conflict', + 'an app revision is immutable; use a new revision for changed launch settings', + 409, + ); + } + if (this.active?.status.state === 'running' && this.active.specKey === specKey) { + return publicStatus(this.active); + } + const session = this.deps.getSession(); + if (!session) { + throw new HostedAppError( + 'hosted_app_session_required', + 'a bound stateful runtime session is required', + 409, + ); + } + const runtime = this.deps.resolveRuntime(request.language, request.version); + if (!runtime) { + throw new HostedAppError( + 'hosted_app_runtime_not_found', + `runtime ${request.language}@${request.version} is not installed`, + 400, + ); + } + if (runtime.compiled) { + throw new HostedAppError( + 'hosted_app_runtime_unsupported', + 'compiled runtimes are not supported by the resident-server adapter', + 400, + ); + } + + const ownership = await session.ownership(); + await resolveWorkspacePaths(ownership.dir, request); + if (rememberedSpec === undefined) { + if (this.revisionSpecs.size >= MAX_TRACKED_APP_REVISIONS) { + throw new HostedAppError( + 'hosted_app_revision_limit', + 'the hosted app revision limit for this runtime session was reached', + 409, + ); + } + this.revisionSpecs.set(revisionKey, specKey); + } + await this.stopImpl(); + const runtimeEnv = { ...runtime.env_vars }; + let nodeModulesPath: string | undefined; + if (runtime.language === 'bash') { + const linkTarget: { nodeModulesPath?: string } = {}; + aggregateBashExtras(runtime.pkgdir, runtimeEnv, this.deps.listRuntimes(), linkTarget); + nodeModulesPath = linkTarget.nodeModulesPath; + } + await this.deps.prepareRuntimeWorkspace(ownership.dir, runtime, nodeModulesPath); + /* The previous process shared this workspace UID and could have changed a + * validated path before it exited. Re-resolve after the process/cgroup are + * gone; the preflight above exists to avoid stopping it for ordinary + * configuration errors, while this check is authoritative for launch. */ + const workspace = await resolveWorkspacePaths(ownership.dir, request); + try { + await this.deps.prepareCgroup(); + await this.deps.installNetworkGuard(ownership.uid); + } catch (error) { + logger.error({ err: error, uid: ownership.uid }, 'Hosted-app isolation setup failed'); + throw new HostedAppError( + 'hosted_app_isolation_failed', + 'hosted app could not be started safely', + 503, + ); + } + + const status: HostedAppStatus = { + app_id: request.app_id, + revision: request.revision, + state: 'starting', + port: config.hosted_app_port, + started_at: this.deps.now().toISOString(), + stdout: '', + stderr: '', + }; + const active: ActiveHostedApp = { request, specKey, cgroupDrained: false, status }; + this.active = active; + + const callerEnv = filterExtraEnvVars(request.env); + for (const key of Object.keys(callerEnv)) { + if (key.toUpperCase() === 'PORT' || key.toUpperCase() === 'HOST') { + delete callerEnv[key]; + } + } + const env: NodeJS.ProcessEnv = { + ...callerEnv, + ...runtimeEnv, + HOME: ownership.dir, + HOST: '0.0.0.0', + PORT: String(config.hosted_app_port), + SANDBOX_LANGUAGE: runtime.language, + }; + const command = HOSTED_APP_LAUNCHER; + const args = [ + HOSTED_APP_CGROUP, + String(ownership.uid), + String(ownership.gid), + '/bin/bash', + path.join(runtime.pkgdir, 'run'), + workspace.entrypoint, + ...request.args, + ]; + + let child: HostedAppChild; + try { + child = this.deps.spawnApp(command, args, { + cwd: workspace.cwd, + env, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + active.status.state = 'failed'; + active.status.exited_at = this.deps.now().toISOString(); + active.status.message = 'hosted app process could not be spawned'; + throw new HostedAppError('hosted_app_spawn_failed', active.status.message, 500); + } + active.process = child; + active.status.pid = child.pid; + child.stdout.on('data', chunk => { + active.status.stdout = appendBounded(active.status.stdout, chunk); + }); + child.stderr.on('data', chunk => { + active.status.stderr = appendBounded(active.status.stderr, chunk); + }); + child.once('error', error => { + active.status.state = 'failed'; + active.status.exited_at = this.deps.now().toISOString(); + active.status.message = error.message; + }); + child.once('exit', (code, signal) => { + active.process = undefined; + active.status.exited_at = this.deps.now().toISOString(); + if (code !== null) active.status.exit_code = code; + if (signal !== null) active.status.signal = signal; + if (active.status.state === 'stopping') { + active.status.state = 'stopped'; + } else if (active.status.state !== 'stopped') { + active.status.state = 'failed'; + active.status.message ??= 'hosted app exited'; + } + /* A daemonized descendant can outlive the tracked launcher process and + * process group. Queue unexpected cleanup in the same transition chain + * as start/stop: a detached sweep must never land after a new revision + * has entered this shared cgroup. */ + if (active.status.state !== 'stopped') { + void this.serialize(async () => { + if (this.active !== active || active.status.state === 'stopped') return; + await this.deps.killCgroup(); + active.cgroupDrained = true; + }).catch(error => { + logger.error( + { err: error, appId: active.request.app_id }, + 'Hosted-app cgroup cleanup failed', + ); + }); + } + }); + + const deadline = Date.now() + config.hosted_app_start_timeout_ms; + while (Date.now() < deadline) { + if (active.status.state === 'failed') { + await this.stopImpl(true); + active.status.state = 'failed'; + throw new HostedAppError( + 'hosted_app_start_failed', + active.status.message ?? 'hosted app exited before becoming ready', + 502, + ); + } + const ready = await this.deps.probePort(config.hosted_app_port); + if (active.process !== child || active.status.state === 'failed') { + await this.stopImpl(true); + active.status.state = 'failed'; + throw new HostedAppError( + 'hosted_app_start_failed', + active.status.message ?? 'hosted app exited before becoming ready', + 502, + ); + } + if (ready) { + active.status.state = 'running'; + logger.info( + { appId: request.app_id, revision: request.revision, pid: child.pid }, + 'Hosted app started', + ); + return publicStatus(active); + } + await new Promise(resolve => setTimeout(resolve, PROBE_INTERVAL_MS)); + } + + active.status.message = `hosted app did not listen on port ${config.hosted_app_port}`; + await this.stopImpl(true); + active.status.state = 'failed'; + throw new HostedAppError('hosted_app_start_timeout', active.status.message, 504); + } + + private async stopImpl(preserveActive = false): Promise { + const active = this.active; + if (!active) return undefined; + const child = active.process; + if (!child?.pid) { + await this.deps.killCgroup(); + active.cgroupDrained = true; + active.status.state = 'stopped'; + active.status.exited_at ??= this.deps.now().toISOString(); + if (!preserveActive) this.active = undefined; + return publicStatus(active); + } + + active.status.state = 'stopping'; + this.deps.killProcessGroup(child.pid, 'SIGTERM'); + const exited = new Promise(resolve => child.once('exit', () => resolve(true))); + let timer: ReturnType | undefined; + const timedOut = new Promise(resolve => { + timer = setTimeout(() => resolve(false), config.hosted_app_stop_timeout_ms); + timer.unref?.(); + }); + const stopped = await Promise.race([exited, timedOut]); + if (timer) clearTimeout(timer); + if (!stopped && active.process?.pid) { + await this.deps.killCgroup(); + await Promise.race([ + new Promise(resolve => child.once('exit', () => resolve())), + new Promise(resolve => setTimeout(resolve, config.hosted_app_stop_timeout_ms)), + ]); + } + /* Always sweep the cgroup: the tracked parent may have exited cleanly + * while a daemonized descendant stayed alive in a different process group. */ + await this.deps.killCgroup(); + active.cgroupDrained = true; + active.status.state = 'stopped'; + active.status.exited_at ??= this.deps.now().toISOString(); + const status = publicStatus(active); + if (!preserveActive) this.active = undefined; + return status; + } +} + +export function validateHostedAppStartup(): void { + if (!config.hosted_apps_enabled) return; + const bindParts = config.bind_address.split(':'); + const runnerPort = Number(bindParts[bindParts.length - 1]); + const failures: string[] = []; + if (!config.session_workspace_enabled) { + failures.push('SANDBOX_SESSION_WORKSPACE_ENABLED must be true'); + } + if (config.hosted_app_port < 1024 || config.hosted_app_port > 65535) { + failures.push('SANDBOX_HOSTED_APP_PORT must be between 1024 and 65535'); + } + if (config.hosted_app_port === runnerPort) { + failures.push('SANDBOX_HOSTED_APP_PORT must differ from PORT'); + } + if (!config.use_cgroupv2) { + failures.push('SANDBOX_USE_CGROUPV2 must be true'); + } + if (process.getuid?.() !== 0) { + failures.push('the dedicated hosted-app runner must start as root'); + } + if (failures.length > 0) { + throw new Error(`Invalid hosted-app runner configuration: ${failures.join('; ')}`); + } +} + +export const hostedAppSupervisor = new HostedAppSupervisor(); diff --git a/api/src/index.ts b/api/src/index.ts index 629436e2..cfbc936c 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -8,6 +8,7 @@ import { httpMetricsMiddleware, metricsHandler } from './metrics'; import { positiveInt, shutdownTelemetry, traceHttpRequest } from './telemetry'; import { startWarmupCommand } from './warmup'; import { stopToolCallSocketProxy } from './tool-call-socket-process'; +import { hostedAppSupervisor, validateHostedAppStartup } from './hosted-app'; import v2Router from './api/v2'; import lifecycleRouter, { LIFECYCLE_HOOK_BASE_PATH } from './api/lifecycle'; @@ -66,6 +67,7 @@ app.use((err: HttpError, _req: express.Request, res: express.Response, _next: ex }); async function main(): Promise { + validateHostedAppStartup(); validateHardenedSandboxStartup(); await initializeSandboxWorkspaceIsolation(); await startWarmupCommand(); @@ -111,6 +113,9 @@ async function main(): Promise { shuttingDown = true; stopWorkspaceReaper(); await closeHttpServerWithTimeout(); + await hostedAppSupervisor.shutdown().catch((err) => { + logger.warn({ err }, 'Hosted-app process shutdown failed'); + }); await stopToolCallSocketProxy().catch((err) => { logger.warn({ err }, 'Tool-call socket proxy shutdown failed'); }); diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 309d6eca..2283bbd7 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -142,6 +142,64 @@ IMAGE_DIGEST=sha256:<64-hex-digest> \ scripts/build-lambda-microvm-artifact.sh zip upload ``` +### Dedicated hosted-app image (experimental) + +Resident web servers use a separate `lambda-microvm-app-host` image. They do +not run as background children of `/execute`: an execution's PID/network +namespaces end with that execution, while the app host intentionally keeps one +supervised foreground process alive for the MicroVM lease. Build its immutable +artifact through the same provenance-checked pipeline: + +```bash +MICROVM_IMAGE_TARGET=lambda-microvm-app-host \ + ECR_URI="$ECR_URI" S3_URI="$S3_URI" IMAGE_TAG="$IMAGE_TAG" \ + scripts/build-lambda-microvm-artifact.sh build push zip upload +# → app-host tags/artifacts are distinct from the normal runner +``` + +The app-host contract is deliberately narrow: + +1. Launch a MicroVM from the dedicated app-host image and wait for port 8080. +2. Mint a control token restricted to port 8080. +3. Restore an immutable stateful-session checkpoint with + `X-Runtime-Session-Id`, exactly as for a replacement session runner. +4. `POST /api/v2/hosted-app/start` on port 8080 with the same session header: + + ```json + { + "app_id": "my-app", + "revision": "rev-1", + "language": "node", + "version": ">=22", + "entrypoint": "server.js", + "cwd": ".", + "args": [], + "env": {} + } + ``` + + The process must listen on `HOST=0.0.0.0` and the injected `PORT` (3000 by + default). Start is idempotent for an identical revision; changed launch + settings require a new revision. +5. Mint a separate preview token restricted to port 3000. Keep the endpoint and + both AWS credentials behind a CodeAPI preview gateway; never put a raw AWS + proxy token in browser-visible HTML or JavaScript. + +Hosted mode disables ordinary `/api/v2/execute`. The app runs as the +session-workspace UID with a curated environment, and the runner installs +fail-closed IPv4 and IPv6 OUTPUT rules before spawn: responses to inbound +preview traffic are allowed, but new outbound connections (including calls to +the root-owned control listener on localhost) are rejected. One app runs per +MicroVM. A restored checkpoint is a revision copy, not a live shared filesystem +with the coding VM. + +Lambda suspend/resume preserves the resident process, but the eight-hour hard +lifetime does not. The higher-level hosted-app control plane must therefore +retain the immutable revision/checkpoint identity, relaunch, restore, and start +again after expiry. Static assets and request-shaped handlers should remain on +cheaper stateless delivery paths; this target is only the resident-server +adapter. + ### 3. Generate the split execution-manifest keys The worker signs each execution manifest; the runner only receives the public diff --git a/scripts/build-lambda-microvm-artifact.sh b/scripts/build-lambda-microvm-artifact.sh index 8e8869bd..6ad13b7a 100755 --- a/scripts/build-lambda-microvm-artifact.sh +++ b/scripts/build-lambda-microvm-artifact.sh @@ -7,7 +7,7 @@ # in a same-account ECR repo (Lambda's build infra can pull it there). # # Stages (each optional, in order): -# build docker buildx the arm64 lambda-microvm-runner target (no AWS) +# build docker buildx the selected arm64 MicroVM target (no AWS) # push push to ECR (needs AWS_PROFILE + repo) # zip render the code-artifact Dockerfile and zip it (no AWS) # upload upload the zip to S3 (needs AWS_PROFILE + bucket) @@ -24,6 +24,8 @@ # S3_URI e.g. s3://codeapi-microvm-artifacts/runner # AWS_PROFILE e.g. librechat-dev # AWS_REGION required for push/upload +# MICROVM_IMAGE_TARGET lambda-microvm-runner (default) or +# lambda-microvm-app-host set -euo pipefail cd "$(dirname "$0")/.." @@ -37,8 +39,25 @@ if [ -z "${IMAGE_TAG:-}" ]; then fi ECR_URI="${ECR_URI:-}" S3_URI="${S3_URI:-}" -OUT_DIR="${OUT_DIR:-.build-lambda-microvm}" -LOCAL_TAG="codeapi-lambda-microvm-runner:${IMAGE_TAG}" +MICROVM_IMAGE_TARGET="${MICROVM_IMAGE_TARGET:-lambda-microvm-runner}" +case "$MICROVM_IMAGE_TARGET" in + lambda-microvm-runner) + ARTIFACT_KIND="runner" + PUBLISHED_TAG="$IMAGE_TAG" + DEFAULT_OUT_DIR=".build-lambda-microvm" + ;; + lambda-microvm-app-host) + ARTIFACT_KIND="app-host" + PUBLISHED_TAG="app-host-${IMAGE_TAG}" + DEFAULT_OUT_DIR=".build-lambda-microvm-app-host" + ;; + *) + echo "MICROVM_IMAGE_TARGET must be lambda-microvm-runner or lambda-microvm-app-host" >&2 + exit 1 + ;; +esac +OUT_DIR="${OUT_DIR:-$DEFAULT_OUT_DIR}" +LOCAL_TAG="codeapi-${MICROVM_IMAGE_TARGET}:${IMAGE_TAG}" IMAGE_DIGEST="${IMAGE_DIGEST:-}" require_ecr() { @@ -58,8 +77,8 @@ resolve_image_digest() { local cached_repository cached_tag cached_repository="$(sed -n '1p' "$OUT_DIR/image-repository" 2>/dev/null || true)" cached_tag="$(sed -n '1p' "$OUT_DIR/image-tag" 2>/dev/null || true)" - if [ "$cached_repository" != "$ECR_URI" ] || [ "$cached_tag" != "$IMAGE_TAG" ]; then - echo "Cached image digest does not belong to ECR_URI=$ECR_URI IMAGE_TAG=$IMAGE_TAG; run push first or set IMAGE_DIGEST explicitly." >&2 + if [ "$cached_repository" != "$ECR_URI" ] || [ "$cached_tag" != "$PUBLISHED_TAG" ]; then + echo "Cached image digest does not belong to ECR_URI=$ECR_URI PUBLISHED_TAG=$PUBLISHED_TAG; run push first or set IMAGE_DIGEST explicitly." >&2 exit 1 fi IMAGE_DIGEST="$(sed -n '1p' "$OUT_DIR/image-digest")" @@ -76,12 +95,12 @@ resolve_image_digest() { do_build() { local tags=(-t "$LOCAL_TAG") if [ -n "$ECR_URI" ]; then - tags+=(-t "$ECR_URI:$IMAGE_TAG") + tags+=(-t "$ECR_URI:$PUBLISHED_TAG") fi - echo ">> buildx arm64 lambda-microvm-runner (${LOCAL_TAG})" + echo ">> buildx arm64 ${MICROVM_IMAGE_TARGET} (${LOCAL_TAG})" docker buildx build \ --platform linux/arm64 \ - --target lambda-microvm-runner \ + --target "$MICROVM_IMAGE_TARGET" \ -f api/Dockerfile \ "${tags[@]}" \ --load \ @@ -91,13 +110,13 @@ do_build() { do_push() { require_ecr mkdir -p "$OUT_DIR" - echo ">> pushing $ECR_URI:$IMAGE_TAG" + echo ">> pushing $ECR_URI:$PUBLISHED_TAG" aws ecr get-login-password --region "${AWS_REGION:?AWS_REGION required}" \ | docker login --username AWS --password-stdin "${ECR_URI%%/*}" # `build` is intentionally usable without AWS/ECR configuration. Tag here as # well so a later, separately invoked `push` stage still has the remote tag. - docker image tag "$LOCAL_TAG" "$ECR_URI:$IMAGE_TAG" - docker push "$ECR_URI:$IMAGE_TAG" | tee "$OUT_DIR/push.log" + docker image tag "$LOCAL_TAG" "$ECR_URI:$PUBLISHED_TAG" + docker push "$ECR_URI:$PUBLISHED_TAG" | tee "$OUT_DIR/push.log" IMAGE_DIGEST="$(sed -n 's/^.*digest: \(sha256:[0-9a-f]\{64\}\).*$/\1/p' "$OUT_DIR/push.log" | tail -n 1)" [ -n "$IMAGE_DIGEST" ] || { echo "Could not determine the pushed ECR digest; refusing to render a mutable artifact." >&2 @@ -105,8 +124,8 @@ do_push() { } printf '%s\n' "$IMAGE_DIGEST" > "$OUT_DIR/image-digest" printf '%s\n' "$ECR_URI" > "$OUT_DIR/image-repository" - printf '%s\n' "$IMAGE_TAG" > "$OUT_DIR/image-tag" - echo ">> immutable runner ref: $ECR_URI@$IMAGE_DIGEST" + printf '%s\n' "$PUBLISHED_TAG" > "$OUT_DIR/image-tag" + echo ">> immutable ${ARTIFACT_KIND} ref: $ECR_URI@$IMAGE_DIGEST" } do_zip() { @@ -118,7 +137,7 @@ FROM ${ECR_URI}@${IMAGE_DIGEST} EOF (cd "$OUT_DIR" && rm -f artifact.zip && zip -q artifact.zip Dockerfile) printf '%s\n' "$ECR_URI" > "$OUT_DIR/artifact-image-repository" - printf '%s\n' "$IMAGE_TAG" > "$OUT_DIR/artifact-image-tag" + printf '%s\n' "$PUBLISHED_TAG" > "$OUT_DIR/artifact-image-tag" printf '%s\n' "$IMAGE_DIGEST" > "$OUT_DIR/artifact-image-digest" file_sha256 "$OUT_DIR/artifact.zip" > "$OUT_DIR/artifact-sha256" echo ">> wrote $OUT_DIR/artifact.zip (FROM ${ECR_URI}@${IMAGE_DIGEST})" @@ -145,13 +164,13 @@ do_upload() { artifact_hash="$(sed -n '1p' "$OUT_DIR/artifact-sha256")" actual_hash="$(file_sha256 "$OUT_DIR/artifact.zip")" if [ "$artifact_repository" != "$ECR_URI" ] \ - || [ "$artifact_tag" != "$IMAGE_TAG" ] \ + || [ "$artifact_tag" != "$PUBLISHED_TAG" ] \ || [ "$artifact_digest" != "$IMAGE_DIGEST" ] \ || [ "$artifact_hash" != "$actual_hash" ]; then echo "artifact.zip provenance does not match the current repository, tag, digest, or bytes; run zip again before upload." >&2 exit 1 fi - local key="$S3_URI/runner-${IMAGE_TAG}.zip" + local key="$S3_URI/${ARTIFACT_KIND}-${IMAGE_TAG}.zip" aws s3 cp "$OUT_DIR/artifact.zip" "$key" --region "${AWS_REGION:?AWS_REGION required}" echo ">> uploaded $key" cat < \\ --region \${AWS_REGION} \\ From f210c9070dca6c75e58f6adac43cc9791871480f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:20:28 -0400 Subject: [PATCH 028/116] build(deps): bump qs to 6.16.0 and express to 5.2.1 in /api (#94) Redo of the Dependabot bump, which only updated `api/package.json` and `api/package-lock.json`: - `api/bun.lock` is the lockfile CI and `api/Dockerfile` actually install from (`bun ci` / `bun install --frozen-lockfile`), and it was left on express 4, so every api job failed with "lockfile had changes, but lockfile is frozen". - The `path-to-regexp: 0.1.13` override pinned express 4's router dependency. Express 5 routes through `router@2`, which needs path-to-regexp ^8, so the override had to go rather than be carried forward. - qs stayed on 6.15.3: it already satisfied express 5's `qs@^6.14.0`, so resolution never moved it to the 6.16.0 the bump was for. Both lockfiles are regenerated and agree on express 5.2.1, qs 6.16.0, path-to-regexp 8.4.2 and router 2.2.0. The api's express surface is v5-clean: all route paths are literal (no wildcards or optional params for path-to-regexp 8 to reject), `req.query` is never read (so the query-parser default change is inert), and the only `req.body` reads are behind a JSON parser, an `application/json` guard, or already undefined-tolerant. `@types/express` was already ^5.0.0, so the types now match the runtime instead of being a version ahead of it. Co-authored-by: Danny Avila --- api/bun.lock | 73 ++- api/package-lock.json | 1050 ++++++++++++++++++++--------------------- api/package.json | 5 +- api/src/api/v2.ts | 8 +- 4 files changed, 558 insertions(+), 578 deletions(-) diff --git a/api/bun.lock b/api/bun.lock index 4341186d..b2003e30 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -10,7 +10,7 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "express": "^4.22.2", + "express": "^5.2.1", "nanoid": "^5.1.16", "pino": "^10.3.0", "prom-client": "^15.1.3", @@ -24,9 +24,6 @@ }, }, }, - "overrides": { - "path-to-regexp": "0.1.13", - }, "packages": { "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], @@ -76,15 +73,13 @@ "@types/serve-static": ["@types/serve-static@2.2.0", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*" } }, "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ=="], - "accepts": ["accepts@1.3.8", "", { "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" } }, "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw=="], - - "array-flatten": ["array-flatten@1.1.1", "", {}, "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="], + "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], "bintrees": ["bintrees@1.0.2", "", {}, "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw=="], - "body-parser": ["body-parser@1.20.6", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.15.1", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g=="], + "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], @@ -94,20 +89,18 @@ "call-bound": ["call-bound@1.0.3", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "get-intrinsic": "^1.2.6" } }, "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA=="], - "content-disposition": ["content-disposition@0.5.4", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ=="], + "content-disposition": ["content-disposition@1.1.0", "", {}, "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g=="], "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], "cookie": ["cookie@0.7.1", "", {}, "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w=="], - "cookie-signature": ["cookie-signature@1.0.6", "", {}, "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="], + "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], - "debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], - "destroy": ["destroy@1.2.0", "", {}, "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="], - "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], @@ -124,13 +117,13 @@ "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], - "express": ["express@4.22.2", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", "qs": "~6.15.1", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q=="], + "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], - "finalhandler": ["finalhandler@1.3.1", "", { "dependencies": { "debug": "2.6.9", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "2.4.1", "parseurl": "~1.3.3", "statuses": "2.0.1", "unpipe": "~1.0.0" } }, "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ=="], + "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], - "fresh": ["fresh@0.5.2", "", {}, "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q=="], + "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], @@ -146,31 +139,29 @@ "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], + "iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], - "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - - "media-typer": ["media-typer@0.3.0", "", {}, "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="], + "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], - "merge-descriptors": ["merge-descriptors@1.0.3", "", {}, "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ=="], + "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - "methods": ["methods@1.1.2", "", {}, "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="], + "media-typer": ["media-typer@1.1.1", "", {}, "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ=="], - "mime": ["mime@1.6.0", "", { "bin": "cli.js" }, "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="], + "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], - "mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], + "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], - "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], + "mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="], - "ms": ["ms@2.0.0", "", {}, "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "nanoid": ["nanoid@5.1.16", "", { "bin": { "nanoid": "bin/nanoid.js" } }, "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ=="], - "negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="], + "negotiator": ["negotiator@1.1.0", "", { "dependencies": { "content-type": "^2.1.0" } }, "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -178,9 +169,11 @@ "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="], + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], - "path-to-regexp": ["path-to-regexp@0.1.13", "", {}, "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA=="], + "path-to-regexp": ["path-to-regexp@8.4.2", "", {}, "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA=="], "pino": ["pino@10.3.0", "", { "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^3.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^4.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-0GNPNzHXBKw6U/InGe79A3Crzyk9bcSyObF9/Gfo9DLEf5qj5RF50RSjsu0W1rZ6ZqRGdzDFCRBQvi9/rSGPtA=="], @@ -194,17 +187,17 @@ "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], - "qs": ["qs@6.15.3", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A=="], + "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], "quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="], "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], - "raw-body": ["raw-body@2.5.3", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "unpipe": "~1.0.0" } }, "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA=="], + "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], "real-require": ["real-require@0.2.0", "", {}, "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="], - "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], "safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="], @@ -212,9 +205,9 @@ "semver": ["semver@7.8.0", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA=="], - "send": ["send@0.19.0", "", { "dependencies": { "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", "http-errors": "2.0.0", "mime": "1.6.0", "ms": "2.1.3", "on-finished": "2.4.1", "range-parser": "~1.2.1", "statuses": "2.0.1" } }, "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw=="], + "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], - "serve-static": ["serve-static@1.16.2", "", { "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", "send": "0.19.0" } }, "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw=="], + "serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="], "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], @@ -238,7 +231,7 @@ "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], - "type-is": ["type-is@1.6.18", "", { "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.24" } }, "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g=="], + "type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], @@ -246,18 +239,14 @@ "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], - "utils-merge": ["utils-merge@1.0.1", "", {}, "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="], - "vary": ["vary@1.1.2", "", {}, "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="], - "finalhandler/statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="], - - "send/encodeurl": ["encodeurl@1.0.2", "", {}, "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], - "send/http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="], + "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], - "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "negotiator/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], - "send/statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="], + "type-is/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], } } diff --git a/api/package-lock.json b/api/package-lock.json index 998bdca7..8cf48ae4 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -14,7 +14,7 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "express": "^4.22.2", + "express": "^5.2.1", "nanoid": "^5.1.16", "pino": "^10.3.0", "prom-client": "^15.1.3", @@ -304,22 +304,18 @@ } }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -335,56 +331,40 @@ "license": "MIT" }, "node_modules/body-parser": { - "version": "1.20.6", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", - "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "license": "MIT", "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" + "node": ">=18" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" } }, - "node_modules/body-parser/node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/bun-types": { @@ -436,14 +416,16 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "dependencies": { - "safe-buffer": "5.2.1" - }, + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/content-type": { @@ -464,35 +446,40 @@ } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } }, "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", "dependencies": { - "ms": "2.0.0" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -510,12 +497,14 @@ "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -553,56 +542,55 @@ "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" }, "funding": { "type": "opencollective", @@ -610,20 +598,24 @@ } }, "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/forwarded": { @@ -635,11 +627,12 @@ } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" } }, "node_modules/function-bind": { @@ -725,36 +718,46 @@ } }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" }, "node_modules/ipaddr.js": { "version": "1.9.1", @@ -764,6 +767,12 @@ "node": ">= 0.10" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -774,64 +783,60 @@ } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", "license": "MIT", "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "node": ">= 0.8" + }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "bin": { - "mime": "cli.js" + "node": ">=18" }, - "engines": { - "node": ">=4" + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", "dependencies": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" }, "node_modules/nanoid": { "version": "5.1.16", @@ -852,11 +857,32 @@ } }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/object-inspect": { @@ -883,6 +909,7 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", "dependencies": { "ee-first": "1.1.1" }, @@ -890,19 +917,33 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/pino": { "version": "10.3.1", @@ -979,9 +1020,9 @@ } }, "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "license": "BSD-3-Clause", "dependencies": { "es-define-property": "^1.0.1", @@ -1000,55 +1041,31 @@ "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", "engines": { "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "license": "MIT", "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", + "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" - } - }, - "node_modules/raw-body/node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/raw-body/node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" + "node": ">= 0.10" } }, "node_modules/real-require": { @@ -1059,24 +1076,21 @@ "node": ">= 12.13.0" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } }, "node_modules/safe-stable-stringify": { "version": "2.5.0", @@ -1105,59 +1119,55 @@ } }, "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", - "engines": { - "node": ">= 0.8" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" }, "node_modules/side-channel": { "version": "1.1.1", @@ -1248,9 +1258,10 @@ } }, "node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", "engines": { "node": ">= 0.8" } @@ -1279,21 +1290,40 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", "engines": { "node": ">=0.6" } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, "engines": { - "node": ">= 0.6" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/typescript": { @@ -1319,18 +1349,11 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -1338,6 +1361,12 @@ "engines": { "node": ">= 0.8" } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" } }, "dependencies": { @@ -1549,19 +1578,14 @@ } }, "accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" } }, - "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, "atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -1573,40 +1597,25 @@ "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" }, "body-parser": { - "version": "1.20.6", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", - "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "requires": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" }, "dependencies": { - "http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "requires": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - } - }, - "statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==" + "content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==" } } }, @@ -1643,12 +1652,9 @@ } }, "content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "requires": { - "safe-buffer": "5.2.1" - } + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==" }, "content-type": { "version": "1.0.5", @@ -1661,16 +1667,16 @@ "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==" }, "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==" }, "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { - "ms": "2.0.0" + "ms": "^2.1.3" } }, "depd": { @@ -1678,11 +1684,6 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, - "destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" - }, "dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1732,55 +1733,51 @@ "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "requires": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.13", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" } }, "finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "requires": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" } }, "forwarded": { @@ -1789,9 +1786,9 @@ "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==" }, "function-bind": { "version": "1.1.2", @@ -1843,23 +1840,23 @@ } }, "http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "requires": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" } }, "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "requires": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "inherits": { @@ -1872,48 +1869,43 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, + "is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==" + }, "math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" }, "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==" }, "merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==" - }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" - }, - "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==" }, "mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==" }, "mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "requires": { - "mime-db": "1.52.0" + "mime-db": "^1.54.0" } }, "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nanoid": { "version": "5.1.16", @@ -1921,9 +1913,19 @@ "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==" }, "negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "requires": { + "content-type": "^2.1.0" + }, + "dependencies": { + "content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==" + } + } }, "object-inspect": { "version": "1.13.4", @@ -1943,15 +1945,23 @@ "ee-first": "1.1.1" } }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==" + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==" }, "pino": { "version": "10.3.1", @@ -2008,9 +2018,9 @@ } }, "qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", "requires": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" @@ -2022,38 +2032,19 @@ "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, "range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==" }, "raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "requires": { "bytes": "~3.1.2", "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", + "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" - }, - "dependencies": { - "http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "requires": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - } - }, - "statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==" - } } }, "real-require": { @@ -2061,10 +2052,17 @@ "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==" }, - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "requires": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + } }, "safe-stable-stringify": { "version": "2.5.0", @@ -2082,46 +2080,32 @@ "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==" }, "send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", "requires": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "dependencies": { - "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" } }, "serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "requires": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" } }, "setprototypeof": { @@ -2187,9 +2171,9 @@ "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" }, "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==" }, "tdigest": { "version": "0.1.2", @@ -2213,12 +2197,20 @@ "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, "type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "dependencies": { + "content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==" + } } }, "typescript": { @@ -2238,15 +2230,15 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, - "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" } } } diff --git a/api/package.json b/api/package.json index 8ed06d85..de62fb55 100644 --- a/api/package.json +++ b/api/package.json @@ -15,7 +15,7 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "express": "^4.22.2", + "express": "^5.2.1", "nanoid": "^5.1.16", "pino": "^10.3.0", "prom-client": "^15.1.3", @@ -27,8 +27,5 @@ "@types/semver": "^7.5.8", "typescript": "^5.7.3" }, - "overrides": { - "path-to-regexp": "0.1.13" - }, "license": "Apache-2.0" } diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index b0cc1a21..9656c257 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -683,9 +683,11 @@ function bindSessionFromHeader(req: Request): SessionBindFailure | null { return null; } -/* Express 4 (pinned) does NOT auto-forward rejected route-handler promises, so - * `.catch(next)` is required or a rejection (e.g. session.ownership()) hangs the - * request and surfaces as an unhandled rejection instead of a 5xx. */ +/* `.catch(next)` hands rejections (e.g. session.ownership()) to the error + * middleware so they surface as a 5xx. Express 5 auto-forwards rejected + * route-handler promises too, so this is now belt-and-braces; it was strictly + * required under the previously pinned Express 4, where a rejection instead + * hung the request and surfaced as an unhandled rejection. */ router.get('/session/checkpoint', (req: Request, res: Response, next: NextFunction) => { const failure = bindSessionFromHeader(req); if (failure) { From b3db89cf317569a1e8953a12ac87b262b4a84ed7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 3 Sep 2026 09:33:02 -0400 Subject: [PATCH 029/116] =?UTF-8?q?=E2=9C=8D=EF=B8=8F=20feat:=20Add=20Opt-?= =?UTF-8?q?In=20BYOM=20Workspace=20Mutations=20(#95)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): add opt-in workspace mutations * fix(code): fence workspace mutation commits * fix(code): harden workspace replacement invariants * fix(code): close workspace mutation lifecycle gaps * fix(code): revalidate edits after metadata restore * fix(code): persist mutation quarantine across restarts * fix(code): fence uncertain mutation executor failures * fix(code): close durable mutation guard races * fix(code): bind durable guards to workspace roots * fix(code): harden mutation durability * fix(code): fence concurrent workspace mutations --- docs/remote-bridge/README.md | 14 +- packages/code/README.md | 40 +- packages/code/src/cli.ts | 162 ++- packages/code/src/protocol.test.ts | 78 ++ packages/code/src/protocol.ts | 164 ++- packages/code/src/storage.test.ts | 115 +- packages/code/src/storage.ts | 174 ++- packages/code/src/worker.ts | 262 +++- packages/code/src/workspace-cli.test.ts | 158 ++- packages/code/src/workspace-worker.test.ts | 1396 ++++++++++++++++++-- packages/code/src/workspace.test.ts | 543 +++++++- packages/code/src/workspace.ts | 524 +++++++- service/src/bridge/router.test.ts | 2 + service/src/bridge/router.ts | 2 + service/src/bridge/store.ts | 9 +- service/src/bridge/workspace-store.test.ts | 37 + service/src/workspace-tools/router.test.ts | 4 + service/src/workspace-tools/router.ts | 4 + 18 files changed, 3497 insertions(+), 191 deletions(-) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 60882dce..d8725c28 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -122,6 +122,14 @@ leave Code API, and are bounded to 1 MiB/500 lines for reads, 200 matches for searches, or 500 relative paths for file listings. Absolute paths, traversal, backslashes, symlink escapes, unexpected fields, and host roots are rejected. +Workspace mutation remains disabled unless the operator starts the worker with +`--allow-workspace-writes` (or +`LIBRECHAT_CODE_ALLOW_WORKSPACE_WRITES=true`). That adds bounded `write_file` +and exact-match `edit_file` operations. Writes are limited to 1 MiB of UTF-8 +text, require an existing in-workspace parent directory, reject symlinks, and +commit atomically. The worker capability is an enforcement boundary; LibreChat +should still route every mutation through its configurable tool-approval hooks. + The workspace root can be an existing project, a Git repository, or an empty directory; Git is not required. This boundary keeps that directory local to the operator's machine, but selected file contents, search matches, relative file @@ -129,9 +137,9 @@ listings, and later tool results necessarily cross the outbound bridge to Code API and the model. Treat them as explicit tool outputs, apply the same retention and audit policy as chat content, and do not register a directory containing secrets. The -default operations are read-only; future mutation and shell operations must be -gated by LibreChat's tool-approval hooks in addition to worker capability -checks. +default operations are read-only. Shell execution remains a separate future +capability because it requires a sandboxed process boundary in addition to +LibreChat's tool-approval hooks and worker capability checks. Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, diff --git a/packages/code/README.md b/packages/code/README.md index f5483cf8..db223994 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -187,6 +187,19 @@ stateful result remains ambiguous, it exits with a quarantine error instead of accepting another assignment. Reset or discard that session's local runner before restarting the worker; its workspace may contain mutations that Code API did not commit. +Likewise, if a local `write_file` or `edit_file` completes but its fulfilled +settlement cannot be acknowledged, the worker exits before accepting more +workspace operations and writes a deployment/worker/workspace-scoped +quarantine marker that survives process restarts. The marker is armed before +each mutation with exclusive, incarnation-owned creation and removed only after +Code API accepts its settlement. Overlapping workers cannot replace or clear +one another's marker. The worker refuses to register writable workspace tools +while that marker exists. Inspect or restore the registered directory, then +explicitly clear the marker with +`librechat-code clear-workspace-quarantine --worker-dir ` +before restarting it. Use `--default-workspace --workspace-id ` instead for +an application-owned default directory. `LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE` +may override the marker path for managed deployments. ## Local workspace tools (bridge preview) @@ -196,7 +209,10 @@ may be an existing project, a Git repository, or a newly created empty directory; Git is optional. `LocalWorkspaceTools` registers opaque workspace IDs with optional display names and exposes bounded `read_file`, literal `search_text`, and deterministic -`list_files` operations. +`list_files` operations. Workspace mutation is disabled by default. Operators +can explicitly add confined `write_file` and exact-match `edit_file` operations +with `--allow-workspace-writes` or +`LIBRECHAT_CODE_ALLOW_WORKSPACE_WRITES=true`. Only IDs, names, protocol version, and supported operations appear in worker capabilities; absolute host paths remain local to the worker process. @@ -211,6 +227,15 @@ stop after bounded global result counts. The worker process still belongs inside the trusted BYOM boundary and should receive filesystem access only to roots the operator intentionally registers. +Writes are limited to 1 MiB of UTF-8 text and require an existing directory +inside the registered root. They reject traversal, symlink targets, and +non-regular files, and commit through an owner-only temporary file followed by +an atomic rename. The worker syncs the containing directory and verifies that +the installed inode still contains the requested bytes before reporting +success. Edits replace text only when the requested old text occurs exactly +once and reject if the file changes before commit. These operations do not +create directories or execute commands. + Register one directory already present on the worker machine with the worker-directory option: @@ -231,10 +256,9 @@ and workspace IDs so distinct IDs cannot alias on case-insensitive filesystems. The deployment and paired bridge identity are also part of the namespace, so re-pairing or switching Code API deployments cannot expose the previous identity's files. It persists across worker restarts. The current workspace -tools are read-only, so an empty directory must be populated by a local process -until write-capable coding tools are enabled. The worker never registers its -process working directory implicitly, and `--default-workspace` cannot be -combined with `--worker-dir`. +tools are read-only unless writes are explicitly enabled. The worker never +registers its process working directory implicitly, and `--default-workspace` +cannot be combined with `--worker-dir`. The default public workspace ID is `primary` and the default display name is the directory basename. Operators can use `--workspace-id` and @@ -244,6 +268,12 @@ explicitly. `rg` must be installed on the worker for `search_text` and `list_files`. `LIBRECHAT_CODE_DEFAULT_WORKSPACE=true` is the environment equivalent of `--default-workspace`. +The write flag is an operator capability boundary, not an approval bypass. +LibreChat should allow read, search, and list operations by default and route +write and edit operations through its configurable tool-approval hooks before +dispatch. A worker that was started without write capability rejects mutations +even if a remote caller tries to send one. + The worker advertises these capabilities only when a directory is configured and executes matching assignments under the bridge's existing lease, deadline, cancellation, credential-refresh, and settlement fencing. The diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 3f68a9d3..d52ed957 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,27 +1,56 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; +import { realpath } from 'node:fs/promises'; import { basename, resolve } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { + assertWorkspaceMutationQuarantineOwner, + clearWorkspaceMutationQuarantine, defaultBridgeIdentityPath, + defaultWorkspaceQuarantinePath, defaultWorkspacePath, ensurePrivateWorkspaceDirectory, loadBridgeIdentity, + loadWorkspaceMutationQuarantine, saveBridgeIdentity, + saveWorkspaceMutationQuarantine, } from './storage.js'; import { BridgeWorker } from './worker.js'; import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; import { LocalWorkspaceTools } from './workspace.js'; import { BRIDGE_WORKSPACE_NAME_MAX_LENGTH, + BridgeProtocolError, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, } from './protocol.js'; +function workspaceSecurityIdentity( + pairedPublicKey: string | undefined, + configuredToken: string | undefined, +): string { + return ( + pairedPublicKey ?? required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) + ); +} + +function workspaceQuarantinePath(options: { + codeApiUrl: string; + workerId: string; + workspaceRoot?: string; +}): string { + const override = process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); + if (override) return override; + return defaultWorkspaceQuarantinePath({ + ...options, + workspaceRoot: required('workspace directory', options.workspaceRoot), + }); +} + function required(name: string, value = process.env[name]): string { const normalized = value?.trim(); if (!normalized) throw new Error(`${name} is required`); @@ -219,6 +248,11 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise { @@ -448,6 +538,68 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise { + const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); + const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); + const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); + const identityPath = + configuredIdentityPath ?? + (configuredWorkerId && !configuredToken + ? defaultBridgeIdentityPath(configuredWorkerId) + : undefined); + const pairedIdentity = identityPath + ? await loadBridgeIdentity(identityPath) + : undefined; + const workerId = required( + 'LIBRECHAT_CODE_WORKER_ID', + configuredWorkerId ?? pairedIdentity?.workerId, + ); + const codeApiUrl = required( + 'LIBRECHAT_CODE_URL', + process.env.LIBRECHAT_CODE_URL ?? pairedIdentity?.codeApiUrl, + ); + const workspaceId = + option(args, '--workspace-id') ?? + process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? + 'primary'; + const explicitWorkerDirectory = nonEmpty( + option(args, '--worker-dir') ?? process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), + ); + const useDefaultWorkspace = + args.includes('--default-workspace') || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === + 'true'; + if (explicitWorkerDirectory && useDefaultWorkspace) { + throw new Error( + '--worker-dir and --default-workspace cannot be used together', + ); + } + const workerDirectory = + explicitWorkerDirectory ?? + (useDefaultWorkspace + ? defaultWorkspacePath({ + codeApiUrl, + securityIdentity: workspaceSecurityIdentity( + pairedIdentity?.publicKey, + configuredToken, + ), + workerId, + workspaceId, + }) + : undefined); + const path = workspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: workerDirectory + ? await realpath(workerDirectory) + : undefined, + }); + await clearWorkspaceMutationQuarantine(path); + process.stdout.write( + `librechat-code: cleared workspace mutation quarantine for ${workspaceId}\n`, + ); +} + async function main(): Promise { const args = process.argv.slice(2); if (args[0] === 'relay') { @@ -468,6 +620,10 @@ async function main(): Promise { await run(runtimeSessionId); return; } + if (args[0] === 'clear-workspace-quarantine') { + await clearMutationQuarantine(args.slice(1)); + return; + } if (args[0] && args[0] !== 'run') { throw new Error(`Unknown command: ${args[0]}`); } diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 55b67b2f..10e59105 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -148,3 +148,81 @@ test('workspace file listing accepts only bounded portable requests and results' false, ); }); + +test('workspace mutations accept bounded UTF-8 requests and exact result shapes', () => { + const writeRequest = { + protocolVersion: 1 as const, + operation: 'write_file' as const, + workspaceId: 'primary', + path: 'notes.txt', + content: 'hello', + }; + assert.equal(isWorkspaceToolRequest(writeRequest), true); + assert.equal( + isWorkspaceToolRequest({ + ...writeRequest, + content: 'x'.repeat(1024 * 1024 + 1), + }), + false, + ); + assert.equal( + isWorkspaceToolResult(writeRequest, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + created: true, + bytesWritten: 5, + }), + true, + ); + + const editRequest = { + protocolVersion: 1 as const, + operation: 'edit_file' as const, + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'hello', + newText: 'goodbye', + }; + assert.equal(isWorkspaceToolRequest(editRequest), true); + assert.equal(isWorkspaceToolRequest({ ...editRequest, oldText: '' }), false); + assert.equal( + isWorkspaceToolResult(editRequest, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + replacements: 1, + bytesWritten: 7, + }), + true, + ); +}); + +test('workspace capabilities allow per-workspace operation restrictions', () => { + const capabilities = { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'write_file'], + workspaces: [ + { id: 'readonly', operations: ['read_file'] }, + { id: 'writable', operations: ['read_file', 'write_file'] }, + ], + }, + }; + assert.equal(isValidBridgeWorkerCapabilities(capabilities), true); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...capabilities, + workspaceTools: { + ...capabilities.workspaceTools, + workspaces: [{ id: 'invalid', operations: ['edit_file'] }], + }, + }), + false, + ); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 9d065238..cae6e59e 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -8,6 +8,7 @@ export const BRIDGE_WORKSPACE_NAME_MAX_LENGTH = 128; export const BRIDGE_WORKSPACE_PATH_MAX_LENGTH = 4096; export const BRIDGE_WORKSPACE_QUERY_MAX_LENGTH = 4096; export const BRIDGE_WORKSPACE_READ_MAX_BYTES = 1024 * 1024; +export const BRIDGE_WORKSPACE_WRITE_MAX_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_READ_MAX_LINES = 500; export const BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS = 200; export const BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH = 2000; @@ -18,11 +19,15 @@ export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; export type BridgeWorkspaceToolOperation = | 'read_file' | 'search_text' - | 'list_files'; + | 'list_files' + | 'write_file' + | 'edit_file'; export interface BridgeWorkspaceDescriptor { id: string; name?: string; + /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ + operations?: BridgeWorkspaceToolOperation[]; } export interface BridgeWorkspaceToolCapabilities { @@ -92,14 +97,53 @@ export interface WorkspaceListFilesResult { truncated: boolean; } +export interface WorkspaceWriteFileRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'write_file'; + workspaceId: string; + path: string; + content: string; +} + +export interface WorkspaceWriteFileResult { + protocolVersion: BridgeProtocolVersion; + operation: 'write_file'; + workspaceId: string; + path: string; + created: boolean; + bytesWritten: number; +} + +export interface WorkspaceEditFileRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'edit_file'; + workspaceId: string; + path: string; + oldText: string; + newText: string; +} + +export interface WorkspaceEditFileResult { + protocolVersion: BridgeProtocolVersion; + operation: 'edit_file'; + workspaceId: string; + path: string; + replacements: 1; + bytesWritten: number; +} + export type WorkspaceToolRequest = | WorkspaceReadFileRequest | WorkspaceSearchTextRequest - | WorkspaceListFilesRequest; + | WorkspaceListFilesRequest + | WorkspaceWriteFileRequest + | WorkspaceEditFileRequest; export type WorkspaceToolResult = | WorkspaceReadFileResult | WorkspaceSearchTextResult - | WorkspaceListFilesResult; + | WorkspaceListFilesResult + | WorkspaceWriteFileResult + | WorkspaceEditFileResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -124,6 +168,21 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'path', 'maxResults', ]); +const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'content', +]); +const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'oldText', + 'newText', +]); const WORKSPACE_READ_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -149,6 +208,22 @@ const WORKSPACE_LIST_RESULT_KEYS = new Set([ 'paths', 'truncated', ]); +const WORKSPACE_WRITE_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'created', + 'bytesWritten', +]); +const WORKSPACE_EDIT_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'replacements', + 'bytesWritten', +]); const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ 'path', 'line', @@ -248,6 +323,10 @@ export type WorkspaceToolErrorCode = | 'INVALID_PATH' | 'INVALID_REQUEST' | 'READ_LIMIT_EXCEEDED' + | 'WRITE_LIMIT_EXCEEDED' + | 'WRITE_DISABLED' + | 'WRITE_UNAVAILABLE' + | 'EDIT_CONFLICT' | 'REGISTRATION_INVALID' | 'EXECUTION_ABORTED' | 'LIST_TIMEOUT' @@ -259,6 +338,10 @@ const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'INVALID_PATH', 'INVALID_REQUEST', 'READ_LIMIT_EXCEEDED', + 'WRITE_LIMIT_EXCEEDED', + 'WRITE_DISABLED', + 'WRITE_UNAVAILABLE', + 'EDIT_CONFLICT', 'REGISTRATION_INVALID', 'EXECUTION_ABORTED', 'LIST_TIMEOUT', @@ -407,6 +490,31 @@ export function isWorkspaceToolRequest( Number(request.maxResults) <= BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) ); } + if (request.operation === 'write_file') { + return ( + hasOnlyKeys(request, WORKSPACE_WRITE_REQUEST_KEYS) && + isSafePortableRelativePath(request.path) && + typeof request.content === 'string' && + Buffer.from(request.content).toString('utf8') === request.content && + new TextEncoder().encode(request.content).byteLength <= + BRIDGE_WORKSPACE_WRITE_MAX_BYTES + ); + } + if (request.operation === 'edit_file') { + return ( + hasOnlyKeys(request, WORKSPACE_EDIT_REQUEST_KEYS) && + isSafePortableRelativePath(request.path) && + typeof request.oldText === 'string' && + request.oldText.length > 0 && + Buffer.from(request.oldText).toString('utf8') === request.oldText && + new TextEncoder().encode(request.oldText).byteLength <= + BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + typeof request.newText === 'string' && + Buffer.from(request.newText).toString('utf8') === request.newText && + new TextEncoder().encode(request.newText).byteLength <= + BRIDGE_WORKSPACE_WRITE_MAX_BYTES + ); + } return false; } @@ -420,7 +528,11 @@ export function isWorkspaceToolResult( result.protocolVersion !== BRIDGE_PROTOCOL_VERSION || result.operation !== request.operation || result.workspaceId !== request.workspaceId || - typeof result.truncated !== 'boolean' + (request.operation === 'read_file' || + request.operation === 'search_text' || + request.operation === 'list_files' + ? typeof result.truncated !== 'boolean' + : false) ) { return false; } @@ -482,6 +594,28 @@ export function isWorkspaceToolResult( return true; } + if (request.operation === 'write_file') { + return ( + hasOnlyKeys(result, WORKSPACE_WRITE_RESULT_KEYS) && + result.path === request.path && + typeof result.created === 'boolean' && + Number.isSafeInteger(result.bytesWritten) && + Number(result.bytesWritten) === + new TextEncoder().encode(request.content).byteLength + ); + } + + if (request.operation === 'edit_file') { + return ( + hasOnlyKeys(result, WORKSPACE_EDIT_RESULT_KEYS) && + result.path === request.path && + result.replacements === 1 && + Number.isSafeInteger(result.bytesWritten) && + Number(result.bytesWritten) >= 0 && + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + ); + } + if (!Array.isArray(result.matches)) return false; const maxResults = request.maxResults ?? 50; return ( @@ -515,12 +649,14 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !Array.isArray(capabilities.operations) || capabilities.operations.length < 1 || - capabilities.operations.length > 3 || + capabilities.operations.length > 5 || !capabilities.operations.every( (operation) => operation === 'read_file' || operation === 'search_text' || - operation === 'list_files', + operation === 'list_files' || + operation === 'write_file' || + operation === 'edit_file', ) || new Set(capabilities.operations).size !== capabilities.operations.length || !Array.isArray(capabilities.workspaces) || @@ -535,14 +671,26 @@ export function isValidBridgeWorkspaceToolCapabilities( if (typeof workspace !== 'object' || workspace === null) return false; const descriptor = workspace as Record; if ( - Object.keys(descriptor).some((key) => key !== 'id' && key !== 'name') || + Object.keys(descriptor).some( + (key) => key !== 'id' && key !== 'name' && key !== 'operations', + ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || - descriptor.name.length > BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) + descriptor.name.length > BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || + (descriptor.operations !== undefined && + (!Array.isArray(descriptor.operations) || + descriptor.operations.length < 1 || + descriptor.operations.length > + (capabilities.operations as unknown[]).length || + descriptor.operations.some( + (operation) => + !(capabilities.operations as unknown[]).includes(operation), + ) || + new Set(descriptor.operations).size !== descriptor.operations.length)) ) { return false; } diff --git a/packages/code/src/storage.test.ts b/packages/code/src/storage.test.ts index 4aa890aa..3a17705a 100644 --- a/packages/code/src/storage.test.ts +++ b/packages/code/src/storage.test.ts @@ -1,15 +1,21 @@ import assert from 'node:assert/strict'; -import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { mkdtemp, open, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; +import type { FileHandle } from 'node:fs/promises'; + import { + clearWorkspaceMutationQuarantine, defaultBridgeIdentityPath, + defaultWorkspaceQuarantinePath, defaultWorkspacePath, ensurePrivateWorkspaceDirectory, loadBridgeIdentity, + loadWorkspaceMutationQuarantine, saveBridgeIdentity, + saveWorkspaceMutationQuarantine, } from './storage.js'; test('default identity paths do not collide after worker ID sanitization', () => { @@ -63,6 +69,40 @@ test('default workspace paths are stable and collision resistant', () => { ); }); +test('default mutation quarantine paths are stable and worker scoped', () => { + const options = { + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + workspaceRoot: '/srv/workspaces/project', + homeDirectory: '/home/tester', + }; + assert.equal( + defaultWorkspaceQuarantinePath(options), + defaultWorkspaceQuarantinePath({ + ...options, + codeApiUrl: 'https://code.example/v1/', + }), + ); + assert.equal( + defaultWorkspaceQuarantinePath(options), + defaultWorkspaceQuarantinePath({ + ...options, + codeApiUrl: 'https://CODE.EXAMPLE:443/v1', + }), + ); + assert.notEqual( + defaultWorkspaceQuarantinePath(options), + defaultWorkspaceQuarantinePath({ + ...options, + workspaceRoot: '/srv/workspaces/secondary', + }), + ); + assert.notEqual( + defaultWorkspaceQuarantinePath(options), + defaultWorkspaceQuarantinePath({ ...options, workerId: 'vm-2' }), + ); +}); + test('default workspace directories are created with owner-only permissions', async () => { const directory = await mkdtemp( join(tmpdir(), 'librechat-code-workspace-home-'), @@ -100,3 +140,76 @@ test('paired identity is persisted atomically with owner-only permissions', asyn await rm(directory, { recursive: true, force: true }); } }); + +test('workspace mutation quarantine persists until explicitly cleared', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-quarantine-')); + const path = join(directory, 'state', 'quarantine.json'); + const record = { + version: 1 as const, + workerId: 'vm-1', + workspaceId: 'primary', + quarantinedAt: new Date().toISOString(), + reason: 'ambiguous settlement delivery', + }; + try { + const probe = await open(directory, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + sync(): Promise; + }; + await probe.close(); + const originalSync = fileHandlePrototype.sync; + let syncCalls = 0; + t.mock.method(fileHandlePrototype, 'sync', async function (this: FileHandle) { + await originalSync.call(this); + syncCalls += 1; + }); + await saveWorkspaceMutationQuarantine(path, record); + assert.deepEqual(await loadWorkspaceMutationQuarantine(path), record); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.equal(syncCalls, process.platform === 'win32' ? 1 : 3); + await clearWorkspaceMutationQuarantine(path); + assert.equal(syncCalls, process.platform === 'win32' ? 1 : 4); + assert.equal(await loadWorkspaceMutationQuarantine(path), undefined); + await writeFile(path, '{bad json', 'utf8'); + await assert.rejects( + loadWorkspaceMutationQuarantine(path), + /invalid workspace quarantine file/i, + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('workspace mutation quarantine cannot be replaced or cleared by another owner', async () => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-quarantine-')); + const path = join(directory, 'quarantine.json'); + const first = { + version: 1 as const, + workerId: 'vm-1', + workspaceId: 'primary', + ownerId: 'incarnation-1', + quarantinedAt: new Date().toISOString(), + reason: 'mutation pending settlement', + }; + try { + await saveWorkspaceMutationQuarantine(path, first); + await assert.rejects( + saveWorkspaceMutationQuarantine(path, { + ...first, + ownerId: 'incarnation-2', + }), + (error: unknown) => + (error as NodeJS.ErrnoException).code === 'EEXIST', + ); + assert.deepEqual(await loadWorkspaceMutationQuarantine(path), first); + await assert.rejects( + clearWorkspaceMutationQuarantine(path, 'incarnation-2'), + /owned by another worker incarnation/i, + ); + assert.deepEqual(await loadWorkspaceMutationQuarantine(path), first); + await clearWorkspaceMutationQuarantine(path, 'incarnation-1'); + assert.equal(await loadWorkspaceMutationQuarantine(path), undefined); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index adeab1bd..2a7eef5c 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,7 +1,7 @@ import { createHash, randomBytes } from 'node:crypto'; import { chmod, lstat, mkdir, open, readFile, rename, rm } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; @@ -41,6 +41,53 @@ function workspaceStorageName(value: string): string { return `id-${createHash('sha256').update(value).digest('hex')}`; } +function canonicalDeploymentUrl(value: string): string { + const url = new URL(value); + url.hash = ''; + url.pathname = url.pathname.replace(/\/+$/, '') || '/'; + return url.toString(); +} + +async function syncParentDirectory(path: string): Promise { + if (process.platform === 'win32') return; + const directory = await open(dirname(path), 'r'); + try { + await directory.sync(); + } finally { + await directory.close(); + } +} + +function isMissingPathError(error: unknown): boolean { + return isRecord(error) && 'code' in error && error.code === 'ENOENT'; +} + +async function ensureDurableDirectory(path: string): Promise { + const missing: string[] = []; + let current = path; + while (true) { + try { + const metadata = await lstat(current); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new BridgeProtocolError( + `Workspace quarantine parent must be a directory: ${current}`, + ); + } + break; + } catch (error) { + if (!isMissingPathError(error)) throw error; + missing.push(current); + const parent = dirname(current); + if (parent === current) throw error; + current = parent; + } + } + await mkdir(path, { recursive: true, mode: 0o700 }); + for (const created of missing.reverse()) { + await syncParentDirectory(created); + } +} + export interface DefaultWorkspacePathOptions { codeApiUrl: string; securityIdentity: string; @@ -49,6 +96,22 @@ export interface DefaultWorkspacePathOptions { homeDirectory?: string; } +export interface WorkspaceMutationQuarantineRecord { + version: 1; + workerId: string; + workspaceId: string; + ownerId?: string; + quarantinedAt: string; + reason: string; +} + +export interface DefaultWorkspaceQuarantinePathOptions { + codeApiUrl: string; + workerId: string; + workspaceRoot: string; + homeDirectory?: string; +} + export function defaultWorkspacePath({ codeApiUrl, securityIdentity, @@ -70,6 +133,22 @@ export function defaultWorkspacePath({ ); } +export function defaultWorkspaceQuarantinePath( + options: DefaultWorkspaceQuarantinePathOptions, +): string { + return join( + options.homeDirectory ?? homedir(), + '.local', + 'state', + 'librechat', + 'code', + 'quarantines', + workspaceStorageName(canonicalDeploymentUrl(options.codeApiUrl)), + workspaceStorageName(options.workerId), + `${workspaceStorageName(resolve(options.workspaceRoot))}.json`, + ); +} + export async function ensurePrivateWorkspaceDirectory( path: string, ): Promise { @@ -103,6 +182,99 @@ export async function saveBridgeIdentity( } } +function isWorkspaceMutationQuarantineRecord( + value: unknown, +): value is WorkspaceMutationQuarantineRecord { + return ( + isRecord(value) && + value.version === 1 && + typeof value.workerId === 'string' && + typeof value.workspaceId === 'string' && + (value.ownerId == null || typeof value.ownerId === 'string') && + typeof value.quarantinedAt === 'string' && + Number.isFinite(Date.parse(value.quarantinedAt)) && + typeof value.reason === 'string' && + value.reason.length > 0 + ); +} + +export async function saveWorkspaceMutationQuarantine( + path: string, + record: WorkspaceMutationQuarantineRecord, +): Promise { + await ensureDurableDirectory(dirname(path)); + const file = await open(path, 'wx', 0o600); + try { + await file.writeFile(`${JSON.stringify(record, null, 2)}\n`, 'utf8'); + await file.sync(); + } finally { + await file.close(); + } + await syncParentDirectory(path); +} + +export async function loadWorkspaceMutationQuarantine( + path: string, +): Promise { + let content: string; + try { + content = await readFile(path, 'utf8'); + } catch (error) { + if ( + isRecord(error) && + 'code' in error && + error.code === 'ENOENT' + ) { + return undefined; + } + throw error; + } + let record: unknown; + try { + record = JSON.parse(content) as unknown; + } catch { + throw new BridgeProtocolError(`Invalid workspace quarantine file: ${path}`); + } + if (!isWorkspaceMutationQuarantineRecord(record)) { + throw new BridgeProtocolError(`Invalid workspace quarantine file: ${path}`); + } + return record; +} + +export async function clearWorkspaceMutationQuarantine( + path: string, + ownerId?: string, +): Promise { + if (ownerId != null) { + const record = await loadWorkspaceMutationQuarantine(path); + if (record == null || record.ownerId !== ownerId) { + throw new BridgeProtocolError( + 'Workspace quarantine is owned by another worker incarnation', + ); + } + } + try { + await lstat(path); + } catch (error) { + if (isMissingPathError(error)) return; + throw error; + } + await rm(path, { force: true }); + await syncParentDirectory(path); +} + +export async function assertWorkspaceMutationQuarantineOwner( + path: string, + ownerId: string, +): Promise { + const record = await loadWorkspaceMutationQuarantine(path); + if (record == null || record.ownerId !== ownerId) { + throw new BridgeProtocolError( + 'Workspace quarantine is owned by another worker incarnation', + ); + } +} + export async function loadBridgeIdentity( path: string, ): Promise { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 98eb9976..b71cfa3a 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -4,6 +4,7 @@ import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, + isWorkspaceToolResult, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -32,6 +33,7 @@ export interface BridgeWorkerOptions { runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; + workspaceMutationQuarantine?: WorkspaceMutationQuarantine; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -54,6 +56,13 @@ export interface BridgeWorkerOptions { incarnationId?: string; } +export interface WorkspaceMutationQuarantine { + assertAvailable(): Promise; + arm(reason: string): Promise; + clear(): Promise; + quarantine(reason: string, cause?: unknown): Promise; +} + export interface BridgeWorkerIdentity { privateKey: string; credential: string; @@ -132,7 +141,14 @@ function workspaceCapabilitiesMatch( advertised.workspaces.every( (workspace, index) => workspace.id === executor.workspaces[index]?.id && - workspace.name === executor.workspaces[index]?.name, + workspace.name === executor.workspaces[index]?.name && + workspace.operations?.length === + executor.workspaces[index]?.operations?.length && + (workspace.operations?.every( + (operation, operationIndex) => + operation === + executor.workspaces[index]?.operations?.[operationIndex], + ) ?? executor.workspaces[index]?.operations == null), ) ); } @@ -143,34 +159,77 @@ function registrationCompatibleCapabilities( const workspaceTools = capabilities.workspaceTools; if ( workspaceTools == null || - !workspaceTools.operations.includes('list_files') + (workspaceTools.operations.every( + (operation) => + operation === 'read_file' || operation === 'search_text', + ) && + workspaceTools.workspaces.every( + (workspace) => workspace.operations == null, + )) ) { return capabilities; } const operations = workspaceTools.operations.filter( - (operation) => operation !== 'list_files', + (operation) => + operation === 'read_file' || operation === 'search_text', ); if (operations.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; return compatible; } + const workspaces = workspaceTools.workspaces.flatMap((workspace) => { + if ( + workspace.operations != null && + !operations.every((operation) => workspace.operations?.includes(operation)) + ) { + return []; + } + const { operations: _operations, ...compatibleWorkspace } = workspace; + return [compatibleWorkspace]; + }); + if (workspaces.length === 0) { + const { workspaceTools: _workspaceTools, ...compatible } = capabilities; + return compatible; + } return { ...capabilities, - workspaceTools: { ...workspaceTools, operations }, + workspaceTools: { + ...workspaceTools, + operations, + workspaces, + }, }; } -function supportsDesiredWorkspaceTools( +function supportedWorkspaceCapabilities( registration: BridgeWorkerRegistrationResponse, capabilities: BridgeWorkerCapabilities, -): boolean { - const desired = capabilities.workspaceTools?.operations; +): BridgeWorkerCapabilities | undefined { + const desired = capabilities.workspaceTools; const supported = registration.supportedWorkspaceToolOperations; - return ( - desired != null && - Array.isArray(supported) && - desired.every((operation) => supported.includes(operation)) + if (desired == null || !Array.isArray(supported)) return undefined; + const operations = desired.operations.filter((operation) => + supported.includes(operation), ); + if (operations.length === 0) return undefined; + const workspaces = desired.workspaces.flatMap((workspace) => { + if (workspace.operations == null) return [workspace]; + const workspaceOperations = workspace.operations.filter((operation) => + operations.includes(operation), + ); + return workspaceOperations.length === 0 + ? [] + : [{ ...workspace, operations: workspaceOperations }]; + }); + if (workspaces.length === 0) return undefined; + return { + ...capabilities, + workspaceTools: { + ...desired, + operations, + workspaces, + }, + }; } export class BridgeWorkspaceQuarantinedError extends Error { @@ -190,8 +249,10 @@ export class BridgeWorker { private readonly incarnationId: string; private readonly compatibleCapabilities: BridgeWorkerCapabilities; private registrationCapabilities: BridgeWorkerCapabilities; + private activeCapabilities: BridgeWorkerCapabilities; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; + private mutationGuardArmed = false; private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; constructor(private readonly options: BridgeWorkerOptions) { @@ -222,6 +283,16 @@ export class BridgeWorker { 'Workspace tool capabilities require a matching executor', ); } + if ( + options.capabilities.workspaceTools?.operations.some( + (operation) => operation === 'write_file' || operation === 'edit_file', + ) === true && + options.workspaceMutationQuarantine == null + ) { + throw new BridgeProtocolError( + 'Workspace mutation capabilities require durable quarantine storage', + ); + } this.fetchImpl = options.fetchImpl ?? fetch; this.codeApiUrl = normalizedBaseUrl(options.codeApiUrl); this.runtimeSupervisor = @@ -236,11 +307,36 @@ export class BridgeWorker { options.capabilities, ); this.registrationCapabilities = this.compatibleCapabilities; + this.activeCapabilities = options.capabilities; } async register( signal?: AbortSignal, ): Promise { + return await this.registerWithPolicy(signal, false); + } + + private async registerWithPolicy( + signal: AbortSignal | undefined, + allowActiveMutation: boolean, + ): Promise { + if (!allowActiveMutation) { + try { + await this.options.workspaceMutationQuarantine?.assertAvailable(); + } catch (error) { + if ( + error instanceof BridgeProtocolError && + error.code === 'WORKER_QUARANTINED' + ) { + throw error; + } + throw new BridgeProtocolError( + 'Workspace mutation quarantine state could not be verified', + undefined, + 'WORKER_QUARANTINED', + ); + } + } const registrationController = new AbortController(); const abortRegistration = (): void => registrationController.abort(); if (signal?.aborted) { @@ -284,11 +380,19 @@ export class BridgeWorker { this.registrationCapabilities = this.compatibleCapabilities; registration = await register(this.registrationCapabilities); } + const supportedCapabilities = supportedWorkspaceCapabilities( + registration, + this.options.capabilities, + ); if ( - this.registrationCapabilities !== this.options.capabilities && - supportsDesiredWorkspaceTools(registration, this.options.capabilities) + supportedCapabilities?.workspaceTools != null && + (this.registrationCapabilities.workspaceTools == null || + !workspaceCapabilitiesMatch( + this.registrationCapabilities.workspaceTools, + supportedCapabilities.workspaceTools, + )) ) { - this.registrationCapabilities = this.options.capabilities; + this.registrationCapabilities = supportedCapabilities; try { registration = await register(this.registrationCapabilities); } catch (error) { @@ -310,6 +414,7 @@ export class BridgeWorker { this.serverClockOffsetMs = registeredAtMs - registrationStartedAtMs; } this.registrationTtlMs = registration.leaseTtlMs; + this.activeCapabilities = this.registrationCapabilities; await this.options.onRegistered?.(registration); if (this.options.capabilities.requiresReadyConfirmation === true) { await this.confirmReady(registration, signal); @@ -714,8 +819,12 @@ export class BridgeWorker { let credentialMaintenance: Promise | undefined; let settlement: BridgeSettlement; let ambiguousSandboxError: unknown; + let ambiguousWorkspaceMutationError: unknown; + let workspaceMutationGuardError: BridgeWorkspaceQuarantinedError | undefined; let sandboxRejectedExecution = false; let sandboxStarted = false; + let workspaceMutationArmed = false; + let workspaceMutationApplied = false; let runtimeLease: RuntimeLease | undefined; try { credentialMaintenance = this.maintainCredential( @@ -738,23 +847,59 @@ export class BridgeWorker { throw new BridgeProtocolError('Invalid workspace tool request'); } const workspaceRequest = assignment.request; - const advertised = this.options.workspaceTools.capabilities; + const advertised = this.activeCapabilities.workspaceTools; + if (advertised == null) { + throw new BridgeProtocolError( + 'Workspace tools are not advertised to this Code API', + ); + } if (!advertised.operations.includes(workspaceRequest.operation)) { throw new BridgeProtocolError( 'Workspace tool operation is not advertised', ); } + const workspace = advertised.workspaces.find( + (candidate) => candidate.id === workspaceRequest.workspaceId, + ); + if (workspace == null) { + throw new BridgeProtocolError('Workspace is not advertised'); + } if ( - !advertised.workspaces.some( - (workspace) => workspace.id === workspaceRequest.workspaceId, - ) + workspace.operations != null && + !workspace.operations.includes(workspaceRequest.operation) ) { - throw new BridgeProtocolError('Workspace is not advertised'); + throw new BridgeProtocolError( + 'Workspace tool operation is not advertised for workspace', + ); + } + const isMutation = + workspaceRequest.operation === 'write_file' || + workspaceRequest.operation === 'edit_file'; + if (isMutation) { + this.mutationGuardArmed = true; + try { + await this.options.workspaceMutationQuarantine!.arm( + `Workspace mutation ${workspaceRequest.operation} is pending settlement`, + ); + workspaceMutationArmed = true; + } catch (error) { + this.mutationGuardArmed = false; + throw new BridgeWorkspaceQuarantinedError( + 'Workspace mutation quarantine could not be armed before execution', + error, + ); + } } payload = await this.options.workspaceTools.execute( workspaceRequest, executionController.signal, ); + workspaceMutationApplied = isMutation; + if (isMutation && !isWorkspaceToolResult(workspaceRequest, payload)) { + throw new BridgeProtocolError( + 'Workspace mutation executor returned an invalid result', + ); + } if (executionController.signal.aborted) { throw ( executionController.signal.reason ?? @@ -845,6 +990,23 @@ export class BridgeWorker { result: payload, }; } catch (error) { + if ( + error instanceof BridgeWorkspaceQuarantinedError && + !workspaceMutationArmed + ) { + workspaceMutationGuardError = error; + } + if ( + workspaceMutationApplied || + (workspaceMutationArmed && + !( + error instanceof WorkspaceToolError && + this.options.workspaceTools?.mutationFailuresAreAtomic === true && + !error.mutationMayHaveCommitted + )) + ) { + ambiguousWorkspaceMutationError = error; + } if ( assignment.runtimeSessionId != null && sandboxStarted && @@ -876,6 +1038,14 @@ export class BridgeWorker { credentialController.abort(); await credentialMaintenance; try { + if (workspaceMutationGuardError != null) throw workspaceMutationGuardError; + if (ambiguousWorkspaceMutationError != null) { + throw await this.quarantineWorkspace( + undefined, + 'Worker stopped after a workspace mutation completed without a fulfilled settlement', + ambiguousWorkspaceMutationError, + ); + } if (ambiguousSandboxError != null) { throw await this.quarantineWorkspace( assignment.runtimeSessionId, @@ -915,8 +1085,20 @@ export class BridgeWorker { settlement, localDeadlineAtMs, signal, + workspaceMutationApplied, ); } + if (workspaceMutationArmed) { + try { + await this.options.workspaceMutationQuarantine!.clear(); + this.mutationGuardArmed = false; + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Workspace mutation settled, but durable quarantine could not be cleared', + error, + ); + } + } } finally { heartbeatController.abort(); try { @@ -983,7 +1165,18 @@ export class BridgeWorker { cause?: unknown, ): Promise { if (runtimeSessionId == null) { - return new BridgeWorkspaceQuarantinedError(message, cause); + try { + await this.options.workspaceMutationQuarantine?.quarantine( + message, + cause, + ); + return new BridgeWorkspaceQuarantinedError(message, cause); + } catch (error) { + return new BridgeWorkspaceQuarantinedError( + `${message}; durable workspace mutation quarantine could not be confirmed`, + error, + ); + } } try { await this.runtimeSupervisor.quarantine(runtimeSessionId, message, cause); @@ -1018,7 +1211,7 @@ export class BridgeWorker { ); if (signal.aborted) return; try { - await this.register(signal); + await this.registerWithPolicy(signal, this.mutationGuardArmed); } catch (error) { const terminal = error instanceof BridgeProtocolError && @@ -1076,12 +1269,22 @@ export class BridgeWorker { settlement: BridgeSettlement, deadlineAtMs: number, signal?: AbortSignal, + workspaceMutationApplied = false, ): Promise { + const fulfilledWorkspaceMutation = + workspaceMutationApplied && + settlement.status === 'fulfilled' && + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + (assignment.request.operation === 'write_file' || + assignment.request.operation === 'edit_file'); if (signal?.aborted === true) { - if (assignment.runtimeSessionId != null) { + if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( assignment.runtimeSessionId, - `Stateful workspace ${assignment.runtimeSessionId} was quarantined before settlement during shutdown`, + assignment.runtimeSessionId != null + ? `Stateful workspace ${assignment.runtimeSessionId} was quarantined before settlement during shutdown` + : 'Worker stopped after a workspace mutation could not be settled during shutdown', signal.reason, ); } @@ -1117,12 +1320,15 @@ export class BridgeWorker { error.status !== 429 ) { if ( - assignment.runtimeSessionId != null && + (assignment.runtimeSessionId != null || + fulfilledWorkspaceMutation) && settlement.status === 'fulfilled' ) { throw await this.quarantineWorkspace( assignment.runtimeSessionId, - `Stateful workspace ${assignment.runtimeSessionId} was quarantined after Code API rejected its fulfilled settlement`, + assignment.runtimeSessionId != null + ? `Stateful workspace ${assignment.runtimeSessionId} was quarantined after Code API rejected its fulfilled settlement` + : 'Worker stopped after Code API rejected a fulfilled workspace mutation settlement', error, ); } @@ -1141,12 +1347,14 @@ export class BridgeWorker { signal?.removeEventListener('abort', abortSettlement); } if ( - assignment.runtimeSessionId != null && + (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) && settlement.status === 'fulfilled' ) { throw await this.quarantineWorkspace( assignment.runtimeSessionId, - `Stateful workspace ${assignment.runtimeSessionId} was quarantined after ambiguous settlement delivery`, + assignment.runtimeSessionId != null + ? `Stateful workspace ${assignment.runtimeSessionId} was quarantined after ambiguous settlement delivery` + : 'Worker stopped after ambiguous workspace mutation settlement delivery', lastError, ); } diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 67692cab..ebea82ab 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -1,14 +1,18 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { once } from 'node:events'; -import { mkdtemp, mkdir, rm, stat } from 'node:fs/promises'; +import { access, mkdtemp, mkdir, realpath, rm, stat } from 'node:fs/promises'; import { createServer } from 'node:http'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; -import { defaultWorkspacePath } from './storage.js'; +import { + defaultWorkspaceQuarantinePath, + defaultWorkspacePath, + saveWorkspaceMutationQuarantine, +} from './storage.js'; test('CLI validates a configured worker directory before registration', () => { const result = spawnSync( @@ -58,7 +62,7 @@ test('CLI trims an environment-configured worker directory', async (t) => { assert.doesNotMatch(result.stderr, /invalid workspace registration/i); }); -test('CLI falls back to the workspace ID when the directory basename is invalid', async (t) => { +test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); await mkdir(workspaceRoot); @@ -76,7 +80,11 @@ test('CLI falls back to the workspace ID when the directory basename is invalid' unknown >; if (request.url?.endsWith('/bridge/workers/register')) { - resolveRegistration?.(body); + const operations = ( + (body.capabilities as Record) + .workspaceTools as { operations: string[] } + ).operations; + if (operations.includes('write_file')) resolveRegistration?.(body); response.setHeader('Content-Type', 'application/json'); response.end( JSON.stringify({ @@ -85,6 +93,13 @@ test('CLI falls back to the workspace ID when the directory basename is invalid' incarnationId: body.incarnationId, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + 'edit_file', + ], }), ); return; @@ -109,6 +124,7 @@ test('CLI falls back to the workspace ID when the directory basename is invalid' workspaceRoot, '--workspace-id', 'root-workspace', + '--allow-workspace-writes', ], { env: { @@ -137,10 +153,29 @@ test('CLI falls back to the workspace ID when the directory basename is invalid' (body.capabilities as Record).workspaceTools, { protocolVersion: 1, - operations: ['read_file', 'search_text'], - workspaces: [{ id: 'root-workspace', name: 'root-workspace' }], + operations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + 'edit_file', + ], + workspaces: [ + { + id: 'root-workspace', + name: 'root-workspace', + operations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + 'edit_file', + ], + }, + ], }, ); + assert.equal(JSON.stringify(body).includes(workspaceRoot), false); }); test('CLI explicitly creates and registers an application-owned default workspace', async () => { @@ -182,3 +217,114 @@ test('CLI explicitly creates and registers an application-owned default workspac await rm(testHome, { recursive: true, force: true }); } }); + +test('CLI refuses quarantined mutation registration until an operator clears it', async () => { + const directory = await mkdtemp( + join(tmpdir(), 'librechat-code-cli-quarantine-'), + ); + const workspace = join(directory, 'workspace'); + const quarantine = join(directory, 'quarantine.json'); + await mkdir(workspace); + try { + await saveWorkspaceMutationQuarantine(quarantine, { + version: 1, + workerId: 'engineering-vm', + workspaceId: 'primary', + quarantinedAt: new Date().toISOString(), + reason: 'ambiguous settlement delivery', + }); + const env = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: quarantine, + }; + const blocked = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspace, + '--allow-workspace-writes', + ], + { encoding: 'utf8', env, timeout: 2_000 }, + ); + assert.notEqual(blocked.status, 0); + assert.match(blocked.stderr, /workspace mutations are quarantined/i); + + const cleared = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'clear-workspace-quarantine', + ], + { encoding: 'utf8', env }, + ); + assert.equal(cleared.status, 0, cleared.stderr); + await assert.rejects(access(quarantine)); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('CLI cannot bypass quarantine by renaming the same physical workspace', async () => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-cli-root-')); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + const quarantine = defaultWorkspaceQuarantinePath({ + codeApiUrl: 'http://127.0.0.1:1/v1', + workerId: 'engineering-vm', + workspaceRoot: await realpath(workspace), + homeDirectory: directory, + }); + try { + await saveWorkspaceMutationQuarantine(quarantine, { + version: 1, + workerId: 'engineering-vm', + workspaceId: 'original-name', + quarantinedAt: new Date().toISOString(), + reason: 'ambiguous settlement delivery', + }); + const env = { + ...process.env, + HOME: directory, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }; + const blocked = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspace, + '--workspace-id', + 'renamed-workspace', + '--allow-workspace-writes', + ], + { encoding: 'utf8', env, timeout: 2_000 }, + ); + assert.notEqual(blocked.status, 0); + assert.match(blocked.stderr, /workspace mutations are quarantined/i); + + const cleared = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'clear-workspace-quarantine', + '--worker-dir', + workspace, + '--workspace-id', + 'renamed-workspace', + ], + { encoding: 'utf8', env }, + ); + assert.equal(cleared.status, 0, cleared.stderr); + await assert.rejects(access(quarantine)); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 742808ea..d6cdcd0a 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1,7 +1,8 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { BridgeWorker } from './worker.js'; +import { BridgeProtocolError } from './protocol.js'; +import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; import { WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; @@ -50,6 +51,25 @@ function listWorkspaceExecutor() { }; } +function mutationQuarantine( + onQuarantine?: (reason: string) => void, + onArm?: (reason: string) => void, + onClear?: () => void, +) { + return { + async assertAvailable() {}, + async arm(reason: string) { + onArm?.(reason); + }, + async clear() { + onClear?.(); + }, + async quarantine(reason: string) { + onQuarantine?.(reason); + }, + }; +} + test('worker keeps v1 registration compatible until list_files support is advertised', async () => { const registrations: string[][] = []; const worker = new BridgeWorker({ @@ -74,13 +94,974 @@ test('worker keeps v1 registration compatible until list_files support is advert }, }); - await worker.register(); - - assert.deepEqual(registrations, [['read_file', 'search_text']]); + await worker.register(); + + assert.deepEqual(registrations, [['read_file', 'search_text']]); +}); + +test('worker re-registers list_files after the Code API advertises support', async () => { + const registrations: string[][] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: listWorkspaceCapabilities, + }, + workspaceTools: listWorkspaceExecutor(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { workspaceTools?: { operations: string[] } }; + }; + registrations.push(body.capabilities.workspaceTools?.operations ?? []); + return registrationResponse(true); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + ['read_file', 'search_text'], + ['read_file', 'search_text', 'list_files'], + ]); +}); + +test('worker omits restricted workspaces that legacy registration would widen', async () => { + const registrations: Array<{ + operations: string[]; + workspaces: Array>; + }> = []; + let executed = false; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'search_text' as const], + workspaces: [ + { id: 'read-only', operations: ['read_file' as const] }, + { + id: 'searchable', + operations: ['read_file' as const, 'search_text' as const], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + executed = true; + throw new Error('must not execute an omitted workspace'); + }, + }, + fetchImpl: async (_input, init) => { + if (String(_input).endsWith('/settle')) { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + } + const body = JSON.parse(String(init?.body)) as { + capabilities: { + workspaceTools: { + operations: string[]; + workspaces: Array>; + }; + }; + }; + registrations.push(body.capabilities.workspaceTools); + return registrationResponse(false); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'searchable' }], + }, + ]); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-legacy-omitted-workspace', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'read-only', + query: 'needle', + }, + }); + assert.equal(executed, false); + assert.equal(settlement?.status, 'rejected'); + assert.match(String(settlement?.error), /workspace is not advertised/i); +}); + +test('worker promotes only operations understood by an older Code API', async () => { + const registrations: Array<{ + operations: string[]; + workspaces: Array>; + }> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: [ + 'read_file' as const, + 'search_text' as const, + 'list_files' as const, + 'write_file' as const, + 'edit_file' as const, + ], + workspaces: [ + { + id: 'primary', + operations: [ + 'read_file' as const, + 'search_text' as const, + 'list_files' as const, + 'write_file' as const, + 'edit_file' as const, + ], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { + workspaceTools: { + operations: string[]; + workspaces: Array>; + }; + }; + }; + registrations.push(body.capabilities.workspaceTools); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + ], + }); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary' }], + }, + { + protocolVersion: 1, + operations: ['read_file', 'search_text', 'list_files'], + workspaces: [ + { + id: 'primary', + operations: ['read_file', 'search_text', 'list_files'], + }, + ], + }, + ]); +}); + +test('worker retains per-workspace restrictions during partial mutation promotion', async () => { + const registrations: Array<{ + operations: string[]; + workspaces: Array>; + }> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: [ + 'read_file' as const, + 'search_text' as const, + 'list_files' as const, + 'write_file' as const, + 'edit_file' as const, + ], + workspaces: [ + { + id: 'readonly', + operations: [ + 'read_file' as const, + 'search_text' as const, + 'list_files' as const, + ], + }, + { + id: 'writable', + operations: [ + 'read_file' as const, + 'search_text' as const, + 'list_files' as const, + 'write_file' as const, + 'edit_file' as const, + ], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { + workspaceTools: { + operations: string[]; + workspaces: Array>; + }; + }; + }; + registrations.push(body.capabilities.workspaceTools); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + ], + }); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations[1], { + protocolVersion: 1, + operations: ['read_file', 'search_text', 'list_files', 'write_file'], + workspaces: [ + { + id: 'readonly', + operations: ['read_file', 'search_text', 'list_files'], + }, + { + id: 'writable', + operations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + ], + }, + ], + }); +}); + +test('worker retains per-workspace restrictions during read-only promotion', async () => { + const registrations: Array<{ + operations: string[]; + workspaces: Array>; + }> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'list_files' as const], + workspaces: [ + { id: 'read-only', operations: ['read_file' as const] }, + { + id: 'listable', + operations: ['read_file' as const, 'list_files' as const], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { + workspaceTools: { + operations: string[]; + workspaces: Array>; + }; + }; + }; + registrations.push(body.capabilities.workspaceTools); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'list_files'], + }); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations[1]?.workspaces, [ + { id: 'read-only', operations: ['read_file'] }, + { id: 'listable', operations: ['read_file', 'list_files'] }, + ]); +}); + +test('worker retains a compatible registration when list_files promotion times out', async () => { + let registrationRequests = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + registrationTransportTimeoutMs: 20, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: listWorkspaceCapabilities, + }, + workspaceTools: listWorkspaceExecutor(), + fetchImpl: async (_input, init) => { + registrationRequests += 1; + if (registrationRequests === 1) return registrationResponse(true); + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(init.signal?.reason ?? new Error('aborted')), + { once: true }, + ); + }); + }, + }); + + const registration = await worker.register(); + + assert.equal(registration.workerId, 'vm-1'); + assert.equal(registrationRequests, 2); +}); + +test('worker preserves bounded workspace rejection codes', async () => { + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['search_text' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new WorkspaceToolError( + 'Workspace search timed out', + 'SEARCH_TIMEOUT', + ); + }, + }, + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-timeout', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + query: 'needle', + }, + }); + + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'SEARCH_TIMEOUT'); +}); + +test('worker executes a workspace tool assignment locally without acquiring a sandbox', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const workspaceRequests: object[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + runtimeSupervisor: { + async acquire() { + throw new Error('workspace tools must not acquire a sandbox'); + }, + async reset() {}, + async quarantine() {}, + }, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + }, + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file', 'search_text'], + workspaces: [{ id: 'primary', name: 'LibreChat' }], + }, + async execute(request) { + workspaceRequests.push(request); + return { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }; + }, + }, + fetchImpl: async (input, init) => { + requests.push({ url: String(input), init }); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-1', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + }); + + assert.deepEqual(workspaceRequests, [ + { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }, + ]); + assert.equal(requests.length, 1); + assert.deepEqual(JSON.parse(String(requests[0].init?.body)), { + protocolVersion: 1, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + incarnationId, + status: 'fulfilled', + result: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + content: '# LibreChat', + startLine: 1, + endLine: 1, + truncated: false, + }, + }); +}); + +test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { + let quarantinedReason: string | undefined; + let armed = 0; + let cleared = 0; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + return { + protocolVersion: 1, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'notes.txt', + bytesWritten: 7, + created: true, + }; + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + (reason) => { + quarantinedReason = reason; + }, + () => { + armed += 1; + }, + () => { + cleared += 1; + }, + ), + fetchImpl: async () => + Response.json({ error: 'assignment was fenced' }, { status: 409 }), + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-rejected', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.match( + quarantinedReason ?? '', + /rejected a fulfilled workspace mutation/i, + ); + assert.equal(armed, 1); + assert.equal(cleared, 0); +}); + +test('worker stops after a fulfilled workspace mutation settlement remains ambiguous', async () => { + let quarantinedReason: string | undefined; + let cleared = 0; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['edit_file' as const], + workspaces: [{ id: 'primary', operations: ['edit_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + return { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: request.workspaceId, + path: 'notes.txt', + bytesWritten: 6, + replacements: 1, + }; + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + (reason) => { + quarantinedReason = reason; + }, + undefined, + () => { + cleared += 1; + }, + ), + fetchImpl: async () => { + throw new TypeError('connection reset'); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-edit-ambiguous', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 50).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.match(quarantinedReason ?? '', /ambiguous workspace mutation/i); + assert.equal(cleared, 0); +}); + +test('worker clears its pre-armed quarantine only after mutation settlement is accepted', async () => { + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute(request) { + lifecycle.push('execute'); + return { + protocolVersion: 1, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'notes.txt', + bytesWritten: 7, + created: true, + }; + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + undefined, + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-success', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); +}); + +test('worker retains quarantine when a mutation executor fails ambiguously', async () => { + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + lifecycle.push('execute'); + throw new Error('unknown executor state'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-unknown', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.deepEqual(lifecycle, ['arm', 'execute', 'quarantine']); +}); + +test('worker retains quarantine for typed errors from untrusted mutation executors', async () => { + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + lifecycle.push('execute'); + throw new WorkspaceToolError( + 'post-commit durability failed', + 'WRITE_UNAVAILABLE', + ); + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-typed-error', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.deepEqual(lifecycle, ['arm', 'execute', 'quarantine']); +}); + +test('worker clears quarantine after an atomic executor rejection is settled', async () => { + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-clean-rejection', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'missing/outside.txt', + content: 'blocked', + }, + }); + assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); }); -test('worker re-registers list_files after the Code API advertises support', async () => { - const registrations: string[][] = []; +test('worker retains quarantine when an atomic executor cannot confirm durability', async () => { + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', @@ -91,67 +1072,121 @@ test('worker re-registers list_files after the Code API advertises support', asy statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], - workspaceTools: listWorkspaceCapabilities, + workspaceTools: workspaceCapabilities, }, - workspaceTools: listWorkspaceExecutor(), - fetchImpl: async (_input, init) => { - const body = JSON.parse(String(init?.body)) as { - capabilities: { workspaceTools?: { operations: string[] } }; - }; - registrations.push(body.capabilities.workspaceTools?.operations ?? []); - return registrationResponse(true); + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + throw new WorkspaceToolError( + 'Workspace mutation durability could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); }, }); - await worker.register(); - - assert.deepEqual(registrations, [ - ['read_file', 'search_text'], - ['read_file', 'search_text', 'list_files'], - ]); + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-uncertain-durability', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.deepEqual(lifecycle, ['arm', 'execute', 'quarantine']); }); -test('worker retains a compatible registration when list_files promotion times out', async () => { - let registrationRequests = 0; +test('worker retains quarantine when a mutation executor returns an invalid result', async () => { + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', - registrationTransportTimeoutMs: 20, capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], - workspaceTools: listWorkspaceCapabilities, + workspaceTools: workspaceCapabilities, }, - workspaceTools: listWorkspaceExecutor(), - fetchImpl: async (_input, init) => { - registrationRequests += 1; - if (registrationRequests === 1) return registrationResponse(true); - return new Promise((_resolve, reject) => { - init?.signal?.addEventListener( - 'abort', - () => reject(init.signal?.reason ?? new Error('aborted')), - { once: true }, - ); - }); + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + lifecycle.push('execute'); + return { malformed: true } as never; + }, + }, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); }, }); - const registration = await worker.register(); - - assert.equal(registration.workerId, 'vm-1'); - assert.equal(registrationRequests, 2); + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-invalid-result', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.deepEqual(lifecycle, ['arm', 'execute', 'quarantine']); }); -test('worker preserves bounded workspace rejection codes', async () => { - let settlement: Record | undefined; +test('worker heartbeats through the marker armed by its active mutation', async () => { + let availabilityChecks = 0; + let registrations = 0; const workspaceCapabilities = { protocolVersion: 1 as const, - operations: ['search_text' as const], - workspaces: [{ id: 'primary' }], + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', @@ -167,134 +1202,247 @@ test('worker preserves bounded workspace rejection codes', async () => { }, workspaceTools: { capabilities: workspaceCapabilities, - async execute() { - throw new WorkspaceToolError( - 'Workspace search timed out', - 'SEARCH_TIMEOUT', - ); + async execute(request) { + await new Promise((resolve) => setTimeout(resolve, 20)); + return { + protocolVersion: 1, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'notes.txt', + bytesWritten: 7, + created: true, + }; }, }, - fetchImpl: async (_input, init) => { - settlement = JSON.parse(String(init?.body)) as Record; + workspaceMutationQuarantine: { + async assertAvailable() { + availabilityChecks += 1; + }, + async arm() { + await new Promise((resolve) => setTimeout(resolve, 30)); + }, + async clear() {}, + async quarantine() {}, + }, + fetchImpl: async (input) => { + if (String(input).endsWith('/register')) { + registrations += 1; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 50, + supportedWorkspaceToolOperations: ['write_file'], + }); + } return Response.json({ protocolVersion: 1, accepted: true }); }, }); + await worker.register(); await worker.executeAndSettle({ protocolVersion: 1, - assignmentId: 'assignment-workspace-timeout', + assignmentId: 'assignment-workspace-write-heartbeat', workerId: 'vm-1', incarnationId, - generation: 1, + generation: 4, leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(Date.now() + 5_000).toISOString(), executionKind: 'workspace_tool', request: { protocolVersion: 1, - operation: 'search_text', + operation: 'write_file', workspaceId: 'primary', - query: 'needle', + path: 'notes.txt', + content: 'written', }, }); - assert.equal(settlement?.status, 'rejected'); - assert.equal(settlement?.errorCode, 'SEARCH_TIMEOUT'); + assert.ok(registrations >= 2); + assert.equal(availabilityChecks, 1); }); -test('worker executes a workspace tool assignment locally without acquiring a sandbox', async () => { - const requests: Array<{ url: string; init?: RequestInit }> = []; - const workspaceRequests: object[] = []; +test('worker stops when cancellation races a completed workspace mutation', async () => { + const controller = new AbortController(); + let settlementAttempts = 0; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, - runtimeSupervisor: { - async acquire() { - throw new Error('workspace tools must not acquire a sandbox'); - }, - async reset() {}, - async quarantine() {}, - }, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], - workspaceTools: { - protocolVersion: 1, - operations: ['read_file', 'search_text'], - workspaces: [{ id: 'primary', name: 'LibreChat' }], - }, + workspaceTools: workspaceCapabilities, }, workspaceTools: { - capabilities: { - protocolVersion: 1, - operations: ['read_file', 'search_text'], - workspaces: [{ id: 'primary', name: 'LibreChat' }], - }, + capabilities: workspaceCapabilities, async execute(request) { - workspaceRequests.push(request); + controller.abort(new Error('shutdown')); return { protocolVersion: 1, - operation: 'read_file', - workspaceId: 'primary', - path: 'README.md', - content: '# LibreChat', - startLine: 1, - endLine: 1, - truncated: false, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'notes.txt', + bytesWritten: 7, + created: true, }; }, }, - fetchImpl: async (input, init) => { - requests.push({ url: String(input), init }); + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async () => { + settlementAttempts += 1; return Response.json({ protocolVersion: 1, accepted: true }); }, }); - await worker.executeAndSettle({ - protocolVersion: 1, - assignmentId: 'assignment-workspace-1', + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-cancelled', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }, + controller.signal, + ), + BridgeWorkspaceQuarantinedError, + ); + assert.equal(settlementAttempts, 0); +}); + +test('worker refuses registration while durable mutation quarantine is active', async () => { + let registrations = 0; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', workerId: 'vm-1', incarnationId, - generation: 4, - leaseToken: 'lease-token-that-is-long-enough-for-testing', - expiresAt: new Date(Date.now() + 5_000).toISOString(), - executionKind: 'workspace_tool', - request: { - protocolVersion: 1, - operation: 'read_file', - workspaceId: 'primary', - path: 'README.md', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: { + async assertAvailable() { + throw new BridgeProtocolError( + 'workspace quarantined', + undefined, + 'WORKER_QUARANTINED', + ); + }, + async arm() {}, + async clear() {}, + async quarantine() {}, + }, + fetchImpl: async () => { + registrations += 1; + return registrationResponse(true); }, }); - assert.deepEqual(workspaceRequests, [ - { - protocolVersion: 1, - operation: 'read_file', - workspaceId: 'primary', - path: 'README.md', - }, - ]); - assert.equal(requests.length, 1); - assert.deepEqual(JSON.parse(String(requests[0].init?.body)), { - protocolVersion: 1, - generation: 4, - leaseToken: 'lease-token-that-is-long-enough-for-testing', + await assert.rejects(worker.register(), (error: unknown) => { + assert.equal((error as BridgeProtocolError).code, 'WORKER_QUARANTINED'); + return true; + }); + assert.equal(registrations, 0); +}); + +test('worker never executes a mutation when durable quarantine cannot be armed', async () => { + let executions = 0; + let requests = 0; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [{ id: 'primary', operations: ['write_file' as const] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', incarnationId, - status: 'fulfilled', - result: { - protocolVersion: 1, - operation: 'read_file', - workspaceId: 'primary', - path: 'README.md', - content: '# LibreChat', - startLine: 1, - endLine: 1, - truncated: false, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + executions += 1; + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: { + async assertAvailable() {}, + async arm() { + throw new Error('disk unavailable'); + }, + async clear() {}, + async quarantine() {}, + }, + fetchImpl: async () => { + requests += 1; + return Response.json({ protocolVersion: 1, accepted: true }); }, }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-write-unarmed', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'written', + }, + }), + BridgeWorkspaceQuarantinedError, + ); + assert.equal(executions, 0); + assert.equal(requests, 0); }); test('worker refuses to advertise workspace tools without a matching executor', () => { @@ -829,8 +1977,10 @@ test('worker rejects workspace operations outside its advertised capability', as let settlement: Record | undefined; const workspaceCapabilities = { protocolVersion: 1 as const, - operations: ['read_file' as const], - workspaces: [{ id: 'primary' }], + operations: ['read_file' as const, 'search_text' as const], + workspaces: [ + { id: 'primary', operations: ['read_file' as const] }, + ], }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index b0335507..5fd8de8c 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -1,11 +1,27 @@ import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { mkdtemp, mkdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { + chmod, + chown, + mkdtemp, + mkdir, + open, + readFile, + readdir, + realpath, + rm, + stat, + symlink, + unlink, + writeFile, +} from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, sep } from 'node:path'; import test from 'node:test'; import { promisify } from 'node:util'; +import type { FileHandle } from 'node:fs/promises'; + import { isWorkspaceToolResult, LocalWorkspaceTools, @@ -754,6 +770,531 @@ test('advertises workspace IDs and names without exposing host roots', async (t) assert.equal(JSON.stringify(tools.capabilities).includes(root), false); }); +test('workspace mutations are disabled by default', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'WRITE_DISABLED', + ); +}); + +test('writable workspaces create, replace, and exactly edit files', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', name: 'Writable', root, writable: true }], + }); + + assert.deepEqual(tools.capabilities, { + protocolVersion: 1, + operations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + 'edit_file', + ], + workspaces: [ + { + id: 'primary', + name: 'Writable', + operations: [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + 'edit_file', + ], + }, + ], + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'hello world', + }); + const edit = await tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'world', + newText: 'BYOM', + }); + assert.deepEqual(edit, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + replacements: 1, + bytesWritten: 10, + }); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'hello BYOM'); +}); + +test('workspace mutations sync the containing directory after replacement', async (t) => { + if (process.platform === 'win32') { + t.skip('Directory fsync is unavailable on Windows'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const probe = await open(root, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + sync(): Promise; + }; + await probe.close(); + const originalSync = fileHandlePrototype.sync; + let syncCalls = 0; + t.mock.method(fileHandlePrototype, 'sync', async function (this: FileHandle) { + await originalSync.call(this); + syncCalls += 1; + }); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'before', + }); + assert.equal(syncCalls, 2); + + await tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', + }); + assert.equal(syncCalls, 5); +}); + +test('workspace mutations report uncertain commit when directory sync fails', async (t) => { + if (process.platform === 'win32') { + t.skip('Directory fsync is unavailable on Windows'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const probe = await open(root, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + sync(): Promise; + }; + await probe.close(); + const originalSync = fileHandlePrototype.sync; + let syncCalls = 0; + t.mock.method(fileHandlePrototype, 'sync', async function (this: FileHandle) { + syncCalls += 1; + if (syncCalls === 2) throw new Error('directory sync failed'); + await originalSync.call(this); + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'possibly committed', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'WRITE_UNAVAILABLE' && + error.mutationMayHaveCommitted, + ); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'possibly committed'); +}); + +test('workspace mutations reject a replaced staging inode after installation', async (t) => { + if (process.platform === 'win32') { + t.skip('Open-file replacement semantics differ on Windows'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const probe = await open(root, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + stat(): ReturnType; + }; + await probe.close(); + const originalStat = fileHandlePrototype.stat; + let replaced = false; + t.mock.method(fileHandlePrototype, 'stat', async function (this: FileHandle) { + const metadata = await originalStat.call(this); + if (!replaced && metadata.isFile()) { + const [temporary] = (await readdir(root)).filter((entry) => + entry.startsWith('.librechat-code-'), + ); + if (temporary != null) { + replaced = true; + await unlink(join(root, temporary)); + await writeFile(join(root, temporary), 'attacker-controlled'); + } + } + return metadata; + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'requested', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'WRITE_UNAVAILABLE' && + error.mutationMayHaveCommitted, + ); + assert.equal(replaced, true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'attacker-controlled'); +}); + +test('exact edits reject missing or repeated text without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'notes.txt'), 'same same'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + for (const oldText of ['missing', 'same']) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText, + newText: 'changed', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT', + ); + } + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'same same'); + + await writeFile(join(root, 'notes.txt'), 'aaa'); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'aa', + newText: 'changed', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT', + ); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'aaa'); +}); + +test('writes reject symlink targets and missing parent directories', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + await mkdir(root); + await writeFile(join(parent, 'outside.txt'), 'outside'); + await symlink(join(parent, 'outside.txt'), join(root, 'link.txt')); + await mkdir(join(root, 'real-directory')); + await symlink(join(root, 'real-directory'), join(root, 'directory-link')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + for (const path of [ + 'link.txt', + 'missing/notes.txt', + 'directory-link/notes.txt', + ]) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_PATH', + ); + } + assert.equal(await readFile(join(parent, 'outside.txt'), 'utf8'), 'outside'); +}); + +test('workspace mutations preserve existing file permissions', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const target = join(root, 'notes.txt'); + await writeFile(target, 'before'); + await chmod(target, 0o664); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'after', + }); + + assert.equal((await stat(target)).mode & 0o777, 0o664); +}); + +test('workspace mutations preserve existing file ownership', async (t) => { + if ( + process.platform === 'win32' || + process.getuid == null || + process.getgid == null || + process.getgroups == null + ) { + t.skip('POSIX ownership is unavailable'); + return; + } + const alternateGroup = process + .getgroups() + .find((group) => group !== process.getgid?.()); + if (alternateGroup == null) { + t.skip('No alternate group is available'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const target = join(root, 'notes.txt'); + await writeFile(target, 'before'); + try { + await chown(target, process.getuid(), alternateGroup); + } catch { + t.skip('The current user cannot assign an alternate group'); + return; + } + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'after', + }); + + const metadata = await stat(target); + assert.equal(metadata.uid, process.getuid()); + assert.equal(metadata.gid, alternateGroup); +}); + +test('workspace edits revalidate source after restoring temporary metadata', async (t) => { + if (process.platform === 'win32') { + t.skip('POSIX temporary-file mode observation is unavailable'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const target = join(root, 'notes.txt'); + const original = `before-${'x'.repeat(512 * 1024)}`; + await writeFile(target, original); + await chmod(target, 0o664); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const probe = await open(target, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = fileHandlePrototype.chmod; + t.mock.method(fileHandlePrototype, 'chmod', async function ( + this: FileHandle, + mode: number, + ) { + await originalChmod.call(this, mode); + await writeFile(target, 'concurrent update'); + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT', + ); + assert.equal(await readFile(target, 'utf8'), 'concurrent update'); +}); + +test('workspace edits honor cancellation immediately before atomic replacement', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const target = join(root, 'notes.txt'); + await writeFile(target, 'before'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const controller = new AbortController(); + const probe = await open(target, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + sync(): Promise; + }; + await probe.close(); + const originalSync = fileHandlePrototype.sync; + let syncCalls = 0; + t.mock.method(fileHandlePrototype, 'sync', async function (this: FileHandle) { + await originalSync.call(this); + syncCalls += 1; + if (syncCalls === 1) controller.abort(); + }); + + await assert.rejects( + tools.execute( + { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', + }, + controller.signal, + ), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED', + ); + assert.equal(await readFile(target, 'utf8'), 'before'); +}); + +test('workspace mutations accept filesystem-equivalent directory casing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'MixedCase')); + try { + await realpath(join(root, 'mixedcase')); + } catch { + t.skip('The test filesystem is case-sensitive'); + return; + } + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'mixedcase/notes.txt', + content: 'written', + }); + + assert.equal( + await readFile(join(root, 'MixedCase', 'notes.txt'), 'utf8'), + 'written', + ); +}); + +test('workspace mutations classify operational write failures as unavailable', async (t) => { + if (process.platform === 'win32') { + t.skip('POSIX directory permissions are unavailable'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const locked = join(root, 'locked'); + await mkdir(locked); + await writeFile(join(locked, 'notes.txt'), 'before'); + await chmod(locked, 0o000); + try { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'locked/notes.txt', + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'WRITE_UNAVAILABLE', + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'locked/notes.txt', + oldText: 'before', + newText: 'after', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'WRITE_UNAVAILABLE', + ); + } finally { + await chmod(locked, 0o700); + } +}); + +test('workspace edits bound descriptor reads to the write limit', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'large.txt'), 'x'.repeat(1024 * 1024 + 1)); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'large.txt', + oldText: 'x', + newText: 'y', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'WRITE_LIMIT_EXCEEDED', + ); +}); + test('rejects unbounded file read parameters before reading the file', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 43114516..0f452d94 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,7 +1,8 @@ import { spawn } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { lstat, open, realpath, stat } from 'node:fs/promises'; -import { isAbsolute, relative, resolve, sep } from 'node:path'; +import { lstat, open, realpath, rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -9,6 +10,7 @@ import { BRIDGE_PROTOCOL_VERSION, BRIDGE_WORKSPACE_READ_MAX_BYTES, BRIDGE_WORKSPACE_READ_MAX_LINES, + BRIDGE_WORKSPACE_WRITE_MAX_BYTES, BRIDGE_WORKSPACE_LIST_MAX_RESULTS, BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS, BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, @@ -23,11 +25,15 @@ import type { BridgeWorkspaceToolCapabilities, WorkspaceReadFileRequest, WorkspaceReadFileResult, + WorkspaceEditFileRequest, + WorkspaceEditFileResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, WorkspaceSearchTextRequest, WorkspaceSearchTextResult, + WorkspaceWriteFileRequest, + WorkspaceWriteFileResult, WorkspaceToolRequest, WorkspaceToolErrorCode, WorkspaceToolResult, @@ -37,11 +43,15 @@ export { isWorkspaceToolRequest, isWorkspaceToolResult }; export type { WorkspaceReadFileRequest, WorkspaceReadFileResult, + WorkspaceEditFileRequest, + WorkspaceEditFileResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, WorkspaceSearchTextRequest, WorkspaceSearchTextResult, + WorkspaceWriteFileRequest, + WorkspaceWriteFileResult, WorkspaceToolRequest, WorkspaceToolResult, }; @@ -50,6 +60,8 @@ export interface LocalWorkspaceConfig { id: string; name?: string; root: string; + /** Mutating operations are never advertised unless explicitly enabled. */ + writable?: boolean; } export interface LocalWorkspaceToolsOptions { @@ -58,6 +70,11 @@ export interface LocalWorkspaceToolsOptions { export interface WorkspaceToolExecutor { capabilities: BridgeWorkspaceToolCapabilities; + /** + * True only when every thrown mutation error proves no mutation committed, + * unless the error explicitly reports mutationMayHaveCommitted. + */ + mutationFailuresAreAtomic?: true; execute( request: WorkspaceToolRequest, signal?: AbortSignal, @@ -69,6 +86,13 @@ const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; const LIST_TIMEOUT_MS = 10_000; +const READ_OPERATIONS = [ + 'read_file', + 'search_text', + 'list_files', +] as const; +const WRITE_OPERATIONS = ['write_file', 'edit_file'] as const; + function isUtf8ScalarString(value: string): boolean { return Buffer.from(value).toString('utf8') === value; } @@ -118,12 +142,62 @@ export class WorkspaceToolError extends Error { constructor( message: string, public readonly code: WorkspaceToolErrorCode, + public readonly mutationMayHaveCommitted = false, ) { super(message); this.name = 'WorkspaceToolError'; } } +async function syncWorkspaceDirectory(path: string): Promise { + if (process.platform === 'win32') return; + const directory = await open(path, 'r'); + try { + await directory.sync(); + } finally { + await directory.close(); + } +} + +async function confirmInstalledMutation( + root: string, + installTarget: string, + expected: { dev: bigint | number; ino: bigint | number; content: Buffer }, +): Promise { + let handle: FileHandle | undefined; + try { + await syncWorkspaceDirectory(dirname(installTarget)); + handle = await open( + installTarget, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const [openedStat, canonicalPath] = await Promise.all([ + handle.stat(), + realpath(installTarget), + ]); + const canonicalStat = await stat(canonicalPath); + if ( + !openedStat.isFile() || + !isWithinRoot(root, canonicalPath) || + openedStat.dev !== canonicalStat.dev || + openedStat.ino !== canonicalStat.ino || + openedStat.dev !== expected.dev || + openedStat.ino !== expected.ino || + !(await readBoundedEditFile(handle)).equals(expected.content) + ) { + throw new Error('installed workspace mutation changed'); + } + } catch { + throw new WorkspaceToolError( + 'Workspace mutation durability could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + } finally { + await handle?.close().catch(() => undefined); + } +} + function isWithinRoot(root: string, candidate: string): boolean { const relativePath = relative(root, candidate); return !( @@ -222,6 +296,397 @@ async function readConfinedFile( return decoded; } +interface WorkspaceRoot { + root: string; + writable: boolean; +} + +async function verifyDirectoryPathHasNoSymlinks( + root: string, + directory: string, +): Promise>> { + const relativeDirectory = relative(root, directory); + let current = root; + let currentIdentity = await lstat(root); + for (const segment of relativeDirectory.split(sep).filter(Boolean)) { + current = resolve(current, segment); + currentIdentity = await lstat(current); + if (currentIdentity.isSymbolicLink() || !currentIdentity.isDirectory()) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } + return currentIdentity; +} + +function classifyWritePathValidationError(error: unknown): WorkspaceToolError { + if (error instanceof WorkspaceToolError) return error; + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT' || code === 'ENOTDIR' || code === 'ELOOP') { + return new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + return new WorkspaceToolError( + 'Workspace storage is unavailable', + 'WRITE_UNAVAILABLE', + ); +} + +function throwIfAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); +} + +async function readBoundedEditFile(handle: FileHandle): Promise { + const content = Buffer.allocUnsafe(BRIDGE_WORKSPACE_WRITE_MAX_BYTES + 1); + let bytesRead = 0; + while (bytesRead < content.byteLength) { + const result = await handle.read( + content, + bytesRead, + content.byteLength - bytesRead, + bytesRead, + ); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + if (bytesRead > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + return content.subarray(0, bytesRead); +} + +async function commitVerifiedEdit( + root: string, + candidate: string, + expected: { dev: bigint | number; ino: bigint | number; content: Buffer }, + temporary: string, + installTarget: string, + staged: { dev: bigint | number; ino: bigint | number; content: Buffer }, + signal?: AbortSignal, +): Promise { + let handle: FileHandle | undefined; + try { + handle = await open( + candidate, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const [openedStat, canonicalPath] = await Promise.all([ + handle.stat(), + realpath(candidate), + ]); + const canonicalStat = await stat(canonicalPath); + if ( + !openedStat.isFile() || + !isWithinRoot(root, canonicalPath) || + openedStat.dev !== canonicalStat.dev || + openedStat.ino !== canonicalStat.ino || + openedStat.dev !== expected.dev || + openedStat.ino !== expected.ino + ) { + throw new WorkspaceToolError( + 'Workspace file changed before edit could be committed', + 'EDIT_CONFLICT', + ); + } + const current = await readBoundedEditFile(handle); + if (!current.equals(expected.content)) { + throw new WorkspaceToolError( + 'Workspace file changed before edit could be committed', + 'EDIT_CONFLICT', + ); + } + throwIfAborted(signal); + await rename(temporary, installTarget); + await confirmInstalledMutation(root, installTarget, staged); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT' && code !== 'ENOTDIR' && code !== 'ELOOP') { + throw new WorkspaceToolError( + 'Workspace storage is unavailable', + 'WRITE_UNAVAILABLE', + ); + } + throw new WorkspaceToolError( + 'Workspace file changed before edit could be committed', + 'EDIT_CONFLICT', + ); + } finally { + await handle?.close().catch(() => undefined); + } +} + +async function atomicWriteConfinedFile( + root: string, + requestedPath: string, + content: Buffer, + signal?: AbortSignal, + expected?: { dev: bigint | number; ino: bigint | number; content: Buffer }, +): Promise<{ created: boolean }> { + throwIfAborted(signal); + if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const candidate = resolveWorkspacePath(root, requestedPath); + const parent = dirname(candidate); + let canonicalParent: string; + let parentIdentity: Awaited>; + try { + canonicalParent = await realpath(parent); + parentIdentity = await verifyDirectoryPathHasNoSymlinks(root, parent); + if (!isWithinRoot(root, canonicalParent) || !parentIdentity.isDirectory()) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } catch (error) { + throw classifyWritePathValidationError(error); + } + + let existing: Awaited> | undefined; + try { + existing = await lstat(candidate); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw classifyWritePathValidationError(error); + } + } + if (existing?.isSymbolicLink() || (existing != null && !existing.isFile())) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + if ( + expected != null && + (existing == null || + existing.dev !== expected.dev || + existing.ino !== expected.ino) + ) { + throw new WorkspaceToolError( + 'Workspace file changed before edit could be committed', + 'EDIT_CONFLICT', + ); + } + + const temporary = resolve( + canonicalParent, + `.librechat-code-${randomBytes(18).toString('hex')}.tmp`, + ); + const installTarget = resolve(canonicalParent, basename(candidate)); + let handle: FileHandle | undefined; + let staged: { dev: bigint | number; ino: bigint | number; content: Buffer }; + try { + handle = await open( + temporary, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); + let offset = 0; + while (offset < content.byteLength) { + const { bytesWritten } = await handle.write( + content, + offset, + content.byteLength - offset, + offset, + ); + if (bytesWritten < 1) throw new Error('short workspace write'); + offset += bytesWritten; + } + await handle.sync(); + if (existing != null) { + if (process.platform !== 'win32') { + await handle.chown(Number(existing.uid), Number(existing.gid)); + } + await handle.chmod(Number(existing.mode) & 0o777); + await handle.sync(); + } + const stagedStat = await handle.stat(); + staged = { dev: stagedStat.dev, ino: stagedStat.ino, content }; + if (process.platform === 'win32') { + await handle.close(); + handle = undefined; + } + + let current: Awaited> | undefined; + try { + current = await lstat(candidate); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + if ( + current?.isSymbolicLink() || + (expected == null + ? existing == null + ? current != null + : current == null || + current.dev !== existing.dev || + current.ino !== existing.ino + : current == null || + current.dev !== expected.dev || + current.ino !== expected.ino) + ) { + throw new WorkspaceToolError( + 'Workspace file changed before write could be committed', + 'EDIT_CONFLICT', + ); + } + const [currentParent, currentParentIdentity] = await Promise.all([ + realpath(parent), + verifyDirectoryPathHasNoSymlinks(root, parent), + ]); + if ( + currentParent !== canonicalParent || + currentParentIdentity.isSymbolicLink() || + !currentParentIdentity.isDirectory() || + currentParentIdentity.dev !== parentIdentity.dev || + currentParentIdentity.ino !== parentIdentity.ino + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + if (expected != null) { + await commitVerifiedEdit( + root, + candidate, + expected, + temporary, + installTarget, + staged, + signal, + ); + return { created: false }; + } + throwIfAborted(signal); + await rename(temporary, installTarget); + await confirmInstalledMutation(root, installTarget, staged); + return { created: existing == null }; + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError( + 'Workspace storage is unavailable', + 'WRITE_UNAVAILABLE', + ); + } finally { + await handle?.close().catch(() => undefined); + await unlink(temporary).catch(() => undefined); + } +} + +async function writeWorkspaceFile( + root: string, + request: WorkspaceWriteFileRequest, + signal?: AbortSignal, +): Promise { + const content = Buffer.from(request.content, 'utf8'); + const { created } = await atomicWriteConfinedFile( + root, + request.path, + content, + signal, + ); + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'write_file', + workspaceId: request.workspaceId, + path: request.path, + created, + bytesWritten: content.byteLength, + }; +} + +async function editWorkspaceFile( + root: string, + request: WorkspaceEditFileRequest, + signal?: AbortSignal, +): Promise { + const candidate = resolveWorkspacePath(root, request.path); + let opened: FileHandle | undefined; + try { + opened = await open( + candidate, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const [openedStat, canonicalPath] = await Promise.all([ + opened.stat(), + realpath(candidate), + ]); + const canonicalStat = await stat(canonicalPath); + if ( + !openedStat.isFile() || + !isWithinRoot(root, canonicalPath) || + openedStat.dev !== canonicalStat.dev || + openedStat.ino !== canonicalStat.ino + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + if (openedStat.size > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const original = await readBoundedEditFile(opened); + const hasBom = + original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf; + const body = hasBom ? original.subarray(3) : original; + const text = body.toString('utf8'); + if (!Buffer.from(text, 'utf8').equals(body)) { + throw new WorkspaceToolError( + 'Workspace file is not UTF-8 text', + 'INVALID_REQUEST', + ); + } + const first = text.indexOf(request.oldText); + if ( + first < 0 || + text.indexOf(request.oldText, first + 1) >= 0 + ) { + throw new WorkspaceToolError( + 'Workspace edit must match exactly once', + 'EDIT_CONFLICT', + ); + } + const updatedText = + text.slice(0, first) + + request.newText + + text.slice(first + request.oldText.length); + const updatedBody = Buffer.from(updatedText, 'utf8'); + const updated = hasBom + ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), updatedBody]) + : updatedBody; + await atomicWriteConfinedFile( + root, + request.path, + updated, + signal, + { + dev: openedStat.dev, + ino: openedStat.ino, + content: original, + }, + ); + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: request.workspaceId, + path: request.path, + replacements: 1, + bytesWritten: updated.byteLength, + }; + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw classifyWritePathValidationError(error); + } finally { + await opened?.close().catch(() => undefined); + } +} + interface SearchCandidates { paths: string[]; truncated: boolean; @@ -799,14 +1264,16 @@ async function withinListDeadline( export class LocalWorkspaceTools implements WorkspaceToolExecutor { readonly capabilities: BridgeWorkspaceToolCapabilities; + readonly mutationFailuresAreAtomic = true as const; private constructor( - private readonly roots: ReadonlyMap, + private readonly roots: ReadonlyMap, + operations: BridgeWorkspaceToolCapabilities['operations'], workspaces: BridgeWorkspaceDescriptor[], ) { this.capabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, - operations: ['read_file', 'search_text', 'list_files'], + operations, workspaces, }; } @@ -814,16 +1281,31 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { static async create( options: LocalWorkspaceToolsOptions, ): Promise { - const roots = new Map(); + const roots = new Map(); + const anyWritable = options.workspaces.some( + (workspace) => workspace.writable === true, + ); + const operations: BridgeWorkspaceToolCapabilities['operations'] = [ + ...READ_OPERATIONS, + ...(anyWritable ? WRITE_OPERATIONS : []), + ]; const workspaces: BridgeWorkspaceDescriptor[] = options.workspaces.map( (workspace) => ({ id: workspace.id, ...(workspace.name !== undefined ? { name: workspace.name } : {}), + ...(anyWritable + ? { + operations: [ + ...READ_OPERATIONS, + ...(workspace.writable === true ? WRITE_OPERATIONS : []), + ], + } + : {}), }), ); const capabilities: BridgeWorkspaceToolCapabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, - operations: ['read_file', 'search_text', 'list_files'], + operations, workspaces, }; if (!isValidBridgeWorkspaceToolCapabilities(capabilities)) { @@ -843,9 +1325,12 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { 'REGISTRATION_INVALID', ); } - roots.set(workspace.id, canonicalRoot); + roots.set(workspace.id, { + root: canonicalRoot, + writable: workspace.writable === true, + }); } - return new LocalWorkspaceTools(roots, workspaces); + return new LocalWorkspaceTools(roots, operations, workspaces); } async execute( @@ -864,10 +1349,29 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { 'INVALID_REQUEST', ); } - const root = this.roots.get(request.workspaceId); - if (!root) { + const workspace = this.roots.get(request.workspaceId); + if (!workspace) { throw new WorkspaceToolError('Unknown workspace', 'INVALID_REQUEST'); } + const { root } = workspace; + + if (request.operation === 'write_file' || request.operation === 'edit_file') { + if (!workspace.writable) { + throw new WorkspaceToolError( + 'Workspace mutations are disabled by the worker', + 'WRITE_DISABLED', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + return request.operation === 'write_file' + ? writeWorkspaceFile(root, request, signal) + : editWorkspaceFile(root, request, signal); + } if (request.operation === 'search_text') { return searchWorkspace(root, request, signal); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 2e5343f7..23a38a76 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -283,6 +283,8 @@ describe('paired bridge HTTP API', () => { 'read_file', 'search_text', 'list_files', + 'write_file', + 'edit_file', ], }); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index f4c2b693..122f80c4 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -383,6 +383,8 @@ router.post( 'read_file', 'search_text', 'list_files', + 'write_file', + 'edit_file', ], }); } catch (error) { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 649224cf..b0a5c2fa 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -73,12 +73,15 @@ function supportsWorkspaceTool( request: WorkspaceToolRequest, ): boolean { const capabilities = registration.capabilities.workspaceTools; + const workspace = capabilities?.workspaces.find( + (candidate) => candidate.id === request.workspaceId, + ); return ( capabilities != null && capabilities.operations.includes(request.operation) && - capabilities.workspaces.some( - (workspace) => workspace.id === request.workspaceId, - ) + workspace != null && + (workspace.operations == null || + workspace.operations.includes(request.operation)) ); } diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index 182a2da7..44ab66fe 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -106,6 +106,43 @@ test('rejects a workspace tool that the selected worker did not advertise', asyn expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); }); +test('rejects an operation omitted from the selected workspace capability', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'write_file'], + workspaces: [ + { id: 'readonly', operations: ['read_file'] }, + { id: 'writable', operations: ['read_file', 'write_file'] }, + ], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'write_file', + workspaceId: 'readonly', + path: 'notes.txt', + content: 'blocked', + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + test('rejects a fulfilled workspace settlement that violates the result contract', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 28893910..6ac3b3f3 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -73,6 +73,10 @@ test.each([ ['SEARCH_UNAVAILABLE', 503], ['LIST_TIMEOUT', 504], ['LIST_UNAVAILABLE', 503], + ['WRITE_DISABLED', 403], + ['WRITE_LIMIT_EXCEEDED', 413], + ['WRITE_UNAVAILABLE', 503], + ['EDIT_CONFLICT', 409], ] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { const app = express(); app.use(json()); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 5daf83ea..eb9a5eb4 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -113,6 +113,10 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) ) { status = 503; } + if (settlement.errorCode === 'WRITE_DISABLED') status = 403; + if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; + if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; + if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; res.status(status).json({ error: settlement.error, code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', From 82ca4f60b480352aebc3c5d1d0a5a462526de4e7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 3 Sep 2026 12:52:21 -0400 Subject: [PATCH 030/116] =?UTF-8?q?=F0=9F=96=A5=EF=B8=8F=20feat:=20Define?= =?UTF-8?q?=20BYOM=20Command=20Protocol=20(#97)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/remote-bridge/README.md | 7 +- packages/code/README.md | 6 + packages/code/src/protocol.test.ts | 71 ++++++++++++ packages/code/src/protocol.ts | 125 ++++++++++++++++++++- packages/code/src/worker.ts | 11 +- packages/code/src/workspace-worker.test.ts | 33 ++++++ packages/code/src/workspace.ts | 10 ++ service/src/bridge/router.test.ts | 1 + service/src/bridge/router.ts | 1 + service/src/workspace-tools/router.test.ts | 3 + service/src/workspace-tools/router.ts | 7 +- 11 files changed, 261 insertions(+), 14 deletions(-) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index d8725c28..aed51fc1 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -137,9 +137,10 @@ listings, and later tool results necessarily cross the outbound bridge to Code API and the model. Treat them as explicit tool outputs, apply the same retention and audit policy as chat content, and do not register a directory containing secrets. The -default operations are read-only. Shell execution remains a separate future -capability because it requires a sandboxed process boundary in addition to -LibreChat's tool-approval hooks and worker capability checks. +default operations are read-only. The bridge protocol reserves a bounded +`execute_command` operation, but the local filesystem executor and CLI do not +advertise it. A later layer must bind it to a sandboxed process boundary and +LibreChat's tool-approval hooks before it becomes dispatchable. Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, diff --git a/packages/code/README.md b/packages/code/README.md index db223994..f10a233f 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -216,6 +216,12 @@ with `--allow-workspace-writes` or Only IDs, names, protocol version, and supported operations appear in worker capabilities; absolute host paths remain local to the worker process. +The protocol also defines a bounded `execute_command` request and result for a +sandbox-backed executor. Commands are treated as workspace mutations and cannot +be advertised without durable quarantine storage. `LocalWorkspaceTools` never +runs them in the worker host process; the CLI does not advertise command support +until a sandbox runtime executor is configured. + Reads reject absolute paths, traversal, escaping symlinks, non-regular files, and files larger than 1 MiB. The opened file is checked against its canonical in-workspace inode before it is read. Text search uses `rg` only to enumerate a diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 10e59105..26a9928f 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -200,6 +200,77 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' ); }); +test('workspace commands require bounded sandbox inputs and outputs', () => { + const request = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'npm test', + cwd: 'packages/code', + timeoutMs: 60_000, + maxOutputBytes: 1024, + }; + assert.equal(isWorkspaceToolRequest(request), true); + assert.equal(isWorkspaceToolRequest({ ...request, command: ' ' }), false); + assert.equal( + isWorkspaceToolRequest({ ...request, command: `echo\0secret` }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ ...request, cwd: '../outside' }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ ...request, timeoutMs: 300_001 }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ ...request, maxOutputBytes: 1024 * 1024 + 1 }), + false, + ); + + const result = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'ok\n', + stderr: '', + truncated: false, + timedOut: false, + }; + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + stdout: 'x'.repeat(1025), + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + exitCode: null, + }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + exitCode: null, + timedOut: true, + }), + true, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + hostCwd: '/Users/operator/project', + }), + false, + ); +}); + test('workspace capabilities allow per-workspace operation restrictions', () => { const capabilities = { statefulWorkspace: true, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index cae6e59e..28e99f90 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -13,6 +13,12 @@ export const BRIDGE_WORKSPACE_READ_MAX_LINES = 500; export const BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS = 200; export const BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH = 2000; export const BRIDGE_WORKSPACE_LIST_MAX_RESULTS = 500; +export const BRIDGE_WORKSPACE_COMMAND_MAX_BYTES = 32 * 1024; +export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS = 30_000; +export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; +export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; +export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; +export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; @@ -21,7 +27,8 @@ export type BridgeWorkspaceToolOperation = | 'search_text' | 'list_files' | 'write_file' - | 'edit_file'; + | 'edit_file' + | 'execute_command'; export interface BridgeWorkspaceDescriptor { id: string; @@ -132,18 +139,45 @@ export interface WorkspaceEditFileResult { bytesWritten: number; } +export interface WorkspaceExecuteCommandRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'execute_command'; + workspaceId: string; + /** Shell source evaluated only inside the selected sandbox runtime. */ + command: string; + /** Portable path relative to the workspace root; defaults to '.'. */ + cwd?: string; + timeoutMs?: number; + /** Aggregate UTF-8 stdout and stderr budget. */ + maxOutputBytes?: number; +} + +export interface WorkspaceExecuteCommandResult { + protocolVersion: BridgeProtocolVersion; + operation: 'execute_command'; + workspaceId: string; + exitCode: number | null; + signal?: string; + stdout: string; + stderr: string; + truncated: boolean; + timedOut: boolean; +} + export type WorkspaceToolRequest = | WorkspaceReadFileRequest | WorkspaceSearchTextRequest | WorkspaceListFilesRequest | WorkspaceWriteFileRequest - | WorkspaceEditFileRequest; + | WorkspaceEditFileRequest + | WorkspaceExecuteCommandRequest; export type WorkspaceToolResult = | WorkspaceReadFileResult | WorkspaceSearchTextResult | WorkspaceListFilesResult | WorkspaceWriteFileResult - | WorkspaceEditFileResult; + | WorkspaceEditFileResult + | WorkspaceExecuteCommandResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -183,6 +217,15 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'oldText', 'newText', ]); +const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'command', + 'cwd', + 'timeoutMs', + 'maxOutputBytes', +]); const WORKSPACE_READ_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -224,6 +267,17 @@ const WORKSPACE_EDIT_RESULT_KEYS = new Set([ 'replacements', 'bytesWritten', ]); +const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'exitCode', + 'signal', + 'stdout', + 'stderr', + 'truncated', + 'timedOut', +]); const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ 'path', 'line', @@ -332,7 +386,10 @@ export type WorkspaceToolErrorCode = | 'LIST_TIMEOUT' | 'LIST_UNAVAILABLE' | 'SEARCH_TIMEOUT' - | 'SEARCH_UNAVAILABLE'; + | 'SEARCH_UNAVAILABLE' + | 'COMMAND_TIMEOUT' + | 'COMMAND_UNAVAILABLE' + | 'COMMAND_DISABLED'; const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'INVALID_PATH', @@ -348,6 +405,9 @@ const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'LIST_UNAVAILABLE', 'SEARCH_TIMEOUT', 'SEARCH_UNAVAILABLE', + 'COMMAND_TIMEOUT', + 'COMMAND_UNAVAILABLE', + 'COMMAND_DISABLED', ]); export function isWorkspaceToolErrorCode( @@ -515,6 +575,28 @@ export function isWorkspaceToolRequest( BRIDGE_WORKSPACE_WRITE_MAX_BYTES ); } + if (request.operation === 'execute_command') { + return ( + hasOnlyKeys(request, WORKSPACE_COMMAND_REQUEST_KEYS) && + typeof request.command === 'string' && + request.command.trim().length > 0 && + Buffer.from(request.command).toString('utf8') === request.command && + !request.command.includes('\0') && + new TextEncoder().encode(request.command).byteLength <= + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES && + (request.cwd === undefined || isSafePortableRelativePath(request.cwd)) && + (request.timeoutMs === undefined || + (Number.isSafeInteger(request.timeoutMs) && + Number(request.timeoutMs) >= 1 && + Number(request.timeoutMs) <= + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS)) && + (request.maxOutputBytes === undefined || + (Number.isSafeInteger(request.maxOutputBytes) && + Number(request.maxOutputBytes) >= 1 && + Number(request.maxOutputBytes) <= + BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES)) + ); + } return false; } @@ -616,6 +698,36 @@ export function isWorkspaceToolResult( ); } + if (request.operation === 'execute_command') { + const stdout = typeof result.stdout === 'string' ? result.stdout : null; + const stderr = typeof result.stderr === 'string' ? result.stderr : null; + const outputLimit = + request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + return ( + hasOnlyKeys(result, WORKSPACE_COMMAND_RESULT_KEYS) && + stdout !== null && + stderr !== null && + Buffer.from(stdout).toString('utf8') === stdout && + Buffer.from(stderr).toString('utf8') === stderr && + new TextEncoder().encode(stdout).byteLength + + new TextEncoder().encode(stderr).byteLength <= + outputLimit && + (result.exitCode === null || + (Number.isSafeInteger(result.exitCode) && + Number(result.exitCode) >= 0 && + Number(result.exitCode) <= 255)) && + (result.signal === undefined || + (typeof result.signal === 'string' && + result.signal.length <= BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && + /^SIG[A-Z0-9]+$/.test(result.signal))) && + typeof result.truncated === 'boolean' && + typeof result.timedOut === 'boolean' && + (result.exitCode === null + ? result.timedOut === true || result.signal !== undefined + : result.timedOut === false && result.signal === undefined) + ); + } + if (!Array.isArray(result.matches)) return false; const maxResults = request.maxResults ?? 50; return ( @@ -649,14 +761,15 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !Array.isArray(capabilities.operations) || capabilities.operations.length < 1 || - capabilities.operations.length > 5 || + capabilities.operations.length > 6 || !capabilities.operations.every( (operation) => operation === 'read_file' || operation === 'search_text' || operation === 'list_files' || operation === 'write_file' || - operation === 'edit_file', + operation === 'edit_file' || + operation === 'execute_command', ) || new Set(capabilities.operations).size !== capabilities.operations.length || !Array.isArray(capabilities.workspaces) || diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index b71cfa3a..0b6f5c13 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -285,7 +285,10 @@ export class BridgeWorker { } if ( options.capabilities.workspaceTools?.operations.some( - (operation) => operation === 'write_file' || operation === 'edit_file', + (operation) => + operation === 'write_file' || + operation === 'edit_file' || + operation === 'execute_command', ) === true && options.workspaceMutationQuarantine == null ) { @@ -874,7 +877,8 @@ export class BridgeWorker { } const isMutation = workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file'; + workspaceRequest.operation === 'edit_file' || + workspaceRequest.operation === 'execute_command'; if (isMutation) { this.mutationGuardArmed = true; try { @@ -1277,7 +1281,8 @@ export class BridgeWorker { assignment.executionKind === 'workspace_tool' && isWorkspaceToolRequest(assignment.request) && (assignment.request.operation === 'write_file' || - assignment.request.operation === 'edit_file'); + assignment.request.operation === 'edit_file' || + assignment.request.operation === 'execute_command'); if (signal?.aborted === true) { if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index d6cdcd0a..b1be1cbb 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1469,6 +1469,39 @@ test('worker refuses to advertise workspace tools without a matching executor', ); }); +test('worker requires durable quarantine before advertising command execution', () => { + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + workspaces: [ + { id: 'primary', operations: ['execute_command' as const] }, + ], + }; + assert.throws( + () => + new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + }), + /durable quarantine storage/i, + ); +}); + test('worker compares workspace capabilities structurally', () => { assert.doesNotThrow( () => diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 0f452d94..842a4e65 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -27,6 +27,8 @@ import type { WorkspaceReadFileResult, WorkspaceEditFileRequest, WorkspaceEditFileResult, + WorkspaceExecuteCommandRequest, + WorkspaceExecuteCommandResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, @@ -45,6 +47,8 @@ export type { WorkspaceReadFileResult, WorkspaceEditFileRequest, WorkspaceEditFileResult, + WorkspaceExecuteCommandRequest, + WorkspaceExecuteCommandResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, @@ -1379,6 +1383,12 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { if (request.operation === 'list_files') { return listWorkspaceFiles(root, request, signal); } + if (request.operation === 'execute_command') { + throw new WorkspaceToolError( + 'Command execution requires a sandbox runtime executor', + 'COMMAND_DISABLED', + ); + } const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 23a38a76..2c0d8998 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -285,6 +285,7 @@ describe('paired bridge HTTP API', () => { 'list_files', 'write_file', 'edit_file', + 'execute_command', ], }); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 122f80c4..b764bb39 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -385,6 +385,7 @@ router.post( 'list_files', 'write_file', 'edit_file', + 'execute_command', ], }); } catch (error) { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 6ac3b3f3..5688d739 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -77,6 +77,9 @@ test.each([ ['WRITE_LIMIT_EXCEEDED', 413], ['WRITE_UNAVAILABLE', 503], ['EDIT_CONFLICT', 409], + ['COMMAND_TIMEOUT', 504], + ['COMMAND_UNAVAILABLE', 503], + ['COMMAND_DISABLED', 403], ] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { const app = express(); app.use(json()); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index eb9a5eb4..7f563891 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -103,17 +103,20 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) let status = 422; if ( settlement.errorCode === 'SEARCH_TIMEOUT' || - settlement.errorCode === 'LIST_TIMEOUT' + settlement.errorCode === 'LIST_TIMEOUT' || + settlement.errorCode === 'COMMAND_TIMEOUT' ) { status = 504; } if ( settlement.errorCode === 'SEARCH_UNAVAILABLE' || - settlement.errorCode === 'LIST_UNAVAILABLE' + settlement.errorCode === 'LIST_UNAVAILABLE' || + settlement.errorCode === 'COMMAND_UNAVAILABLE' ) { status = 503; } if (settlement.errorCode === 'WRITE_DISABLED') status = 403; + if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; From 2b1a929acd55448c63801044ebb1316883e5cf68 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 3 Sep 2026 13:23:09 -0400 Subject: [PATCH 031/116] =?UTF-8?q?=F0=9F=A7=B1=20feat:=20Compose=20Sandbo?= =?UTF-8?q?xed=20Workspace=20Commands=20(#98)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/remote-bridge/README.md | 7 ++ packages/code/README.md | 9 ++ packages/code/src/workspace.test.ts | 136 ++++++++++++++++++++++++++++ packages/code/src/workspace.ts | 105 +++++++++++++++++++++ 4 files changed, 257 insertions(+) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index aed51fc1..2f9120eb 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -142,6 +142,13 @@ default operations are read-only. The bridge protocol reserves a bounded advertise it. A later layer must bind it to a sandboxed process boundary and LibreChat's tool-approval hooks before it becomes dispatchable. +The package exposes `SandboxWorkspaceTools` for composing that boundary without +ever invoking a host shell. It requires an explicit sandbox implementation and +an allowlist of workspace IDs, preserves per-workspace operation restrictions, +validates bounded results, and treats an unknown command failure as an uncertain +mutation. The built-in CLI remains command-disabled until its supported NsJail +adapter can safely map a registered directory without changing host ownership. + Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, persistent local runner per session. A single sandbox endpoint is stateless and diff --git a/packages/code/README.md b/packages/code/README.md index f10a233f..0fa1edf7 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -222,6 +222,15 @@ be advertised without durable quarantine storage. `LocalWorkspaceTools` never runs them in the worker host process; the CLI does not advertise command support until a sandbox runtime executor is configured. +`SandboxWorkspaceTools` is the composition boundary for that runtime. It adds +`execute_command` only to workspace IDs explicitly backed by a +`WorkspaceCommandSandbox`, delegates every file operation to the confined local +executor, and validates the sandbox's complete result before returning it. It +does not include a shell fallback. Invalid responses and unknown sandbox errors +are reported as potentially committed mutations so the worker's durable +quarantine remains armed. A concrete runtime adapter must prove its mount and +identity behavior before the CLI can enable this composition. + Reads reject absolute paths, traversal, escaping symlinks, non-regular files, and files larger than 1 MiB. The opened file is checked against its canonical in-workspace inode before it is read. Text search uses `rg` only to enumerate a diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 5fd8de8c..ab09ac28 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -25,6 +25,7 @@ import type { FileHandle } from 'node:fs/promises'; import { isWorkspaceToolResult, LocalWorkspaceTools, + SandboxWorkspaceTools, WorkspaceToolError, } from './workspace.js'; @@ -1544,3 +1545,138 @@ test('validates search result paths against the requested scope', () => { false, ); }); + +test('composes sandboxed commands without exposing them on unconfigured workspaces', async (t) => { + const first = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + const second = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => Promise.all([ + rm(first, { recursive: true, force: true }), + rm(second, { recursive: true, force: true }), + ])); + await writeFile(join(first, 'README.md'), 'first'); + const local = await LocalWorkspaceTools.create({ + workspaces: [ + { id: 'sandboxed', root: first, writable: true }, + { id: 'read-only', root: second }, + ], + }); + const requests: object[] = []; + const tools = new SandboxWorkspaceTools({ + workspaceTools: local, + commandWorkspaces: ['sandboxed'], + commandSandbox: { + async execute(request) { + requests.push(request); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: 'ok\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + + assert.deepEqual(tools.capabilities.operations, [ + 'read_file', + 'search_text', + 'list_files', + 'write_file', + 'edit_file', + 'execute_command', + ]); + assert.deepEqual( + tools.capabilities.workspaces.find(({ id }) => id === 'sandboxed')?.operations, + tools.capabilities.operations, + ); + assert.deepEqual( + tools.capabilities.workspaces.find(({ id }) => id === 'read-only')?.operations, + ['read_file', 'search_text', 'list_files'], + ); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'sandboxed', + command: 'pwd', + }; + assert.deepEqual(await tools.execute(command), { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'sandboxed', + exitCode: 0, + stdout: 'ok\n', + stderr: '', + truncated: false, + timedOut: false, + }); + assert.deepEqual(requests, [command]); + assert.equal( + (await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'sandboxed', + path: 'README.md', + })).operation, + 'read_file', + ); + await assert.rejects( + tools.execute({ ...command, workspaceId: 'read-only' }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_DISABLED', + ); +}); + +test('fails closed on invalid or failed sandbox command results', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const local = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + const request = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'pwd', + }; + for (const execute of [ + async () => ({ ...request, exitCode: 0, stdout: 'ok', stderr: '' }), + async () => { throw new Error('container details'); }, + ]) { + const tools = new SandboxWorkspaceTools({ + workspaceTools: local, + commandWorkspaces: ['primary'], + commandSandbox: { execute }, + }); + await assert.rejects( + tools.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + error.mutationMayHaveCommitted === true && + !error.message.includes('container details'), + ); + } +}); + +test('rejects empty, duplicate, and unknown sandbox workspace registration', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const local = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + for (const commandWorkspaces of [[], ['primary', 'primary'], ['unknown']]) { + assert.throws( + () => new SandboxWorkspaceTools({ + workspaceTools: local, + commandWorkspaces, + commandSandbox: { async execute() { return {}; } }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); + } +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 842a4e65..94366dff 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -85,6 +85,25 @@ export interface WorkspaceToolExecutor { ): Promise; } +/** + * Sandboxed command boundary used by {@link SandboxWorkspaceTools}. The + * implementation is responsible for process, filesystem, and network + * confinement; this package deliberately never falls back to a host shell. + */ +export interface WorkspaceCommandSandbox { + execute( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + ): Promise; +} + +export interface SandboxWorkspaceToolsOptions { + workspaceTools: WorkspaceToolExecutor; + commandSandbox: WorkspaceCommandSandbox; + /** Workspace IDs whose sandbox is configured and may run commands. */ + commandWorkspaces: string[]; +} + const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; @@ -1428,3 +1447,89 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { }; } } + +/** + * Composes ordinary workspace tools with an explicitly supplied sandboxed + * command boundary. Command failures are intentionally not declared atomic: + * callers must retain their durable mutation quarantine until settlement. + */ +export class SandboxWorkspaceTools implements WorkspaceToolExecutor { + readonly capabilities: BridgeWorkspaceToolCapabilities; + private readonly commandWorkspaces: ReadonlySet; + + constructor(private readonly options: SandboxWorkspaceToolsOptions) { + const base = options.workspaceTools.capabilities; + const registeredIds = new Set(base.workspaces.map(({ id }) => id)); + const commandWorkspaces = new Set(options.commandWorkspaces); + if ( + commandWorkspaces.size === 0 || + commandWorkspaces.size !== options.commandWorkspaces.length || + [...commandWorkspaces].some((id) => !registeredIds.has(id)) + ) { + throw new WorkspaceToolError( + 'Invalid sandbox workspace registration', + 'REGISTRATION_INVALID', + ); + } + this.commandWorkspaces = commandWorkspaces; + this.capabilities = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: [...new Set([...base.operations, 'execute_command' as const])], + workspaces: base.workspaces.map((workspace) => ({ + ...workspace, + operations: [ + ...(workspace.operations ?? base.operations), + ...(commandWorkspaces.has(workspace.id) + ? (['execute_command'] as const) + : []), + ], + })), + }; + if (!isValidBridgeWorkspaceToolCapabilities(this.capabilities)) { + throw new WorkspaceToolError( + 'Invalid sandbox workspace registration', + 'REGISTRATION_INVALID', + ); + } + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if (request.operation !== 'execute_command') { + return this.options.workspaceTools.execute(request, signal); + } + if (!this.commandWorkspaces.has(request.workspaceId)) { + throw new WorkspaceToolError( + 'Command execution is disabled for this workspace', + 'COMMAND_DISABLED', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + ); + } + let result: unknown; + try { + result = await this.options.commandSandbox.execute(request, signal); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError( + 'Sandboxed command execution unavailable', + 'COMMAND_UNAVAILABLE', + true, + ); + } + if (!isWorkspaceToolResult(request, result)) { + throw new WorkspaceToolError( + 'Sandboxed command returned an invalid result', + 'COMMAND_UNAVAILABLE', + true, + ); + } + return result; + } +} From 1f6da22ea315f8370dd5fc72fc9a106d2ec9aa3f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 3 Sep 2026 21:18:11 -0400 Subject: [PATCH 032/116] =?UTF-8?q?=F0=9F=A7=B0=20feat:=20Run=20BYOM=20Com?= =?UTF-8?q?mands=20in=20NsJail=20(#100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/src/api/v2-session-binding.test.ts | 60 ++++++ api/src/api/v2.ts | 53 ++++++ api/src/config.ts | 4 + api/src/workspace-command.test.ts | 145 ++++++++++++++ api/src/workspace-command.ts | 200 ++++++++++++++++++++ docs/remote-bridge/README.md | 10 + packages/code/README.md | 22 ++- packages/code/package.json | 4 + packages/code/src/cli.test.ts | 4 +- packages/code/src/cli.ts | 193 ++++++++++++------- packages/code/src/index.ts | 1 + packages/code/src/runtime.test.ts | 42 ++++ packages/code/src/runtime.ts | 11 +- packages/code/src/workspace-cli.test.ts | 49 +++++ packages/code/src/workspace-runtime.test.ts | 153 +++++++++++++++ packages/code/src/workspace-runtime.ts | 143 ++++++++++++++ packages/code/src/workspace-worker.test.ts | 67 ++++++- packages/code/src/workspace.test.ts | 4 + packages/code/src/workspace.ts | 27 ++- 19 files changed, 1109 insertions(+), 83 deletions(-) create mode 100644 api/src/workspace-command.test.ts create mode 100644 api/src/workspace-command.ts create mode 100644 packages/code/src/workspace-runtime.test.ts create mode 100644 packages/code/src/workspace-runtime.ts diff --git a/api/src/api/v2-session-binding.test.ts b/api/src/api/v2-session-binding.test.ts index 0e530734..32a245f6 100644 --- a/api/src/api/v2-session-binding.test.ts +++ b/api/src/api/v2-session-binding.test.ts @@ -20,6 +20,8 @@ let baseUrl: string; let packageDir: string; const savedSessionWorkspaceEnabled = config.session_workspace_enabled; const savedRequireExecutionManifest = config.require_execution_manifest; +const savedExternalWorkspaceEnabled = config.external_workspace_enabled; +const savedExternalWorkspaceToken = config.external_workspace_token; const testLanguage = 'headerless-session-regression'; const testVersion = '1.0.0'; @@ -49,6 +51,8 @@ afterAll(async () => { afterEach(() => { config.session_workspace_enabled = savedSessionWorkspaceEnabled; config.require_execution_manifest = savedRequireExecutionManifest; + config.external_workspace_enabled = savedExternalWorkspaceEnabled; + config.external_workspace_token = savedExternalWorkspaceToken; resetSessionWorkspaceStateForTests(); }); @@ -66,6 +70,62 @@ const execute = (runtimeSessionId?: string) => }), }); +describe('workspace command route boundary', () => { + test('stays hidden unless the external workspace profile is enabled', async () => { + config.external_workspace_enabled = false; + const response = await fetch(`${baseUrl}/api/v2/workspace/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{', + }); + expect(response.status).toBe(404); + }); + + test('authenticates before parsing or binding a command request', async () => { + config.external_workspace_enabled = true; + config.external_workspace_token = 'a'.repeat(32); + config.session_workspace_enabled = true; + const unauthorized = await fetch(`${baseUrl}/api/v2/workspace/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Runtime-Session-Id': 'rt_must_not_bind', + }, + body: '{', + }); + expect(unauthorized.status).toBe(401); + expect(getBoundSessionWorkspace()).toBeUndefined(); + + const authenticated = await fetch(`${baseUrl}/api/v2/workspace/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-LibreChat-Workspace-Token': 'a'.repeat(32), + }, + body: JSON.stringify({ command: 'pwd' }), + }); + expect(authenticated.status).toBe(400); + expect(await authenticated.json()).toEqual({ + message: 'X-Runtime-Session-Id is required', + }); + }); + + test('parses the worst-case escaped valid command envelope', async () => { + config.external_workspace_enabled = true; + config.external_workspace_token = 'a'.repeat(32); + const response = await fetch(`${baseUrl}/api/v2/workspace/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-LibreChat-Workspace-Token': 'a'.repeat(32), + 'X-Runtime-Session-Id': 'rt_maximum_escaped_command', + }, + body: JSON.stringify({ command: '\u0001'.repeat(32 * 1024) }), + }); + expect(response.status).not.toBe(413); + }); +}); + describe('per-request session binding', () => { test('a headerless execute does not inherit the runner session bound by a prior request', async () => { config.session_workspace_enabled = true; diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 9656c257..40ba2517 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -36,6 +36,11 @@ import { HostedAppError, hostedAppSupervisor, } from '../hosted-app'; +import { + executeWorkspaceCommand, + hasWorkspaceCommandToken, + WorkspaceCommandRequestError, +} from '../workspace-command'; const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; @@ -382,6 +387,20 @@ function manifestErrorStatus(error: ExecutionManifestError): number { return 403; } +router.use('/workspace/execute', (req: Request, res: Response, next: NextFunction) => { + if (req.method !== 'POST') return next(); + if (!config.external_workspace_enabled) { + return res.status(404).json({ message: 'Not Found' }); + } + if (!hasWorkspaceCommandToken( + config.external_workspace_token, + req.header('X-LibreChat-Workspace-Token'), + )) { + return res.status(401).json({ message: 'Unauthorized' }); + } + next(); +}); + router.use((req: Request, res: Response, next: NextFunction) => { if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) return next(); /* Checkpoint restore and additive file delivery stream tar.gz bodies, not JSON. */ @@ -614,6 +633,40 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } }); +router.post('/workspace/execute', express.json({ limit: '256kb' }), async (req: Request, res: Response) => { + let binding; + try { + binding = parseSessionBindingFromHeader(req.headers[RUNTIME_SESSION_ID_HEADER]); + } catch (error) { + return res.status(400).json({ + message: error instanceof Error ? error.message : 'Invalid runtime session', + }); + } + if (!binding) { + return res.status(400).json({ message: 'X-Runtime-Session-Id is required' }); + } + if (!bindSessionWorkspace(binding)) { + return res.status(409).json({ + error: 'session_workspace_dirty', + message: 'Runner is bound to a different runtime session', + }); + } + try { + return res.status(200).json(await executeWorkspaceCommand(req.body, { + workspaceRoot: config.external_workspace_root, + })); + } catch (error) { + if (error instanceof WorkspaceCommandRequestError) { + return res.status(400).json({ message: error.message }); + } + logger.error({ err: error }, 'Sandboxed workspace command failed'); + return res.status(500).json({ + error: 'sandbox_execution_failed', + message: 'Sandboxed workspace command failed', + }); + } +}); + router.get('/health', async (_req: Request, res: Response) => { try { return res.status(200).json(await checkSandboxWorkspaceHealth()); diff --git a/api/src/config.ts b/api/src/config.ts index 79c0e078..bbc2c184 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -71,6 +71,10 @@ export const config = { * session mode. An enabled runner additionally binds each request to a * workspace through the authenticated X-Runtime-Session-Id header. */ session_workspace_enabled: (process.env.SANDBOX_SESSION_WORKSPACE_ENABLED ?? 'false') === 'true', + external_workspace_enabled: (process.env.SANDBOX_EXTERNAL_WORKSPACE_ENABLED ?? 'false') === 'true', + external_workspace_root: cleanDirectory(process.env.SANDBOX_EXTERNAL_WORKSPACE_ROOT) + ?? '/mnt/workspace', + external_workspace_token: process.env.SANDBOX_EXTERNAL_WORKSPACE_TOKEN ?? '', /** * Enables the Lambda-only hosted-app runner surface. This must only be set * on a dedicated app-host MicroVM image: user application processes share diff --git a/api/src/workspace-command.test.ts b/api/src/workspace-command.test.ts new file mode 100644 index 00000000..38936a3c --- /dev/null +++ b/api/src/workspace-command.test.ts @@ -0,0 +1,145 @@ +import { mkdtemp, mkdir, realpath, rm, symlink } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, test } from 'bun:test'; +import semver from 'semver'; + +import type { Runtime } from './runtime'; +import { + executeWorkspaceCommand, + hasWorkspaceCommandToken, + WorkspaceCommandRequestError, +} from './workspace-command'; +import type { WorkspaceCommandExecutionOptions } from './workspace-command'; + +const runtime: Runtime = { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir: '/pkgs/bash/5.2.0', + compiled: false, + env_vars: { PATH: '/usr/bin:/bin' }, + timeouts: { compile: 30_000, run: 30_000 }, + cpu_times: { compile: 30_000, run: 30_000 }, + memory_limits: { compile: 256_000_000, run: 256_000_000 }, + max_process_count: 64, + max_open_files: 2048, + max_file_size: 10_000_000, + output_max_size: 1024, +}; + +describe('sandboxed workspace commands', () => { + test('requires an exact high-entropy runner capability', () => { + const token = 'a'.repeat(32); + expect(hasWorkspaceCommandToken(token, token)).toBe(true); + expect(hasWorkspaceCommandToken(token, `${token}x`)).toBe(false); + expect(hasWorkspaceCommandToken(token, undefined)).toBe(false); + expect(hasWorkspaceCommandToken('short', 'short')).toBe(false); + }); + + test('passes command and canonical cwd as arguments to NsJail', async () => { + const root = await mkdtemp(join(tmpdir(), 'workspace-command-')); + try { + await mkdir(join(root, 'src')); + let captured: Parameters>[0] | undefined; + const response = await executeWorkspaceCommand({ + command: 'printf ok', + cwd: 'src', + timeoutMs: 1234, + maxOutputBytes: 64, + }, { + workspaceRoot: root, + runtime, + executeNsJail: async (options) => { + captured = options; + return { + stdout: 'ok', stderr: '', code: 0, signal: null, output: 'ok', + memory: null, message: null, status: null, cpu_time: null, wall_time: 1, + }; + }, + }); + + expect(captured?.command).toEqual([ + '/bin/bash', '--noprofile', '--norc', '-c', + 'cd -- "$1" && exec /bin/bash --noprofile --norc -c "$2"', + 'librechat-code', '/mnt/data/src', 'printf ok', + ]); + expect(captured?.submissionDir).toBe(await realpath(root)); + expect(captured?.timeout).toBe(1234); + expect(captured?.outputMaxSize).toBe(64); + expect(response).toEqual({ + exitCode: 0, + stdout: 'ok', + stderr: '', + truncated: false, + timedOut: false, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + test('rejects a cwd symlink that escapes the mounted workspace', async () => { + const parent = await mkdtemp(join(tmpdir(), 'workspace-command-')); + try { + const root = join(parent, 'root'); + await mkdir(root); + await symlink(parent, join(root, 'escape')); + await expect(executeWorkspaceCommand({ + command: 'pwd', + cwd: 'escape', + }, { workspaceRoot: root, runtime, executeNsJail: async () => { throw new Error('must not run'); } })) + .rejects.toBeInstanceOf(WorkspaceCommandRequestError); + } finally { + await rm(parent, { recursive: true, force: true }); + } + }); + + test('rejects malformed requests before starting NsJail', async () => { + const root = await mkdtemp(join(tmpdir(), 'workspace-command-')); + try { + for (const body of [ + { command: '' }, + { command: 'pwd', cwd: '../outside' }, + { command: 'pwd', timeoutMs: 300_001 }, + { command: 'pwd', unexpected: true }, + ]) { + await expect(executeWorkspaceCommand(body, { + workspaceRoot: root, + runtime, + executeNsJail: async () => { throw new Error('must not run'); }, + })).rejects.toBeInstanceOf(WorkspaceCommandRequestError); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + test('bounds aggregate UTF-8 output and maps timeout state', async () => { + const root = await mkdtemp(join(tmpdir(), 'workspace-command-')); + try { + const response = await executeWorkspaceCommand({ + command: 'run', + maxOutputBytes: 5, + }, { + workspaceRoot: root, + runtime, + executeNsJail: async () => ({ + stdout: '€€', stderr: 'tail', code: null, signal: 'SIGKILL', output: '', + memory: null, message: 'Time limit exceeded', status: 'TO', cpu_time: null, wall_time: 1, + }), + }); + expect(Buffer.byteLength(response.stdout) + Buffer.byteLength(response.stderr)).toBeLessThanOrEqual(5); + expect(response).toEqual({ + exitCode: null, + signal: 'SIGKILL', + stdout: '€', + stderr: 'ta', + truncated: true, + timedOut: true, + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/api/src/workspace-command.ts b/api/src/workspace-command.ts new file mode 100644 index 00000000..a2cd5339 --- /dev/null +++ b/api/src/workspace-command.ts @@ -0,0 +1,200 @@ +import * as path from 'node:path'; +import * as fsp from 'node:fs/promises'; +import { timingSafeEqual } from 'node:crypto'; + +import { aggregateBashExtras } from './job'; +import { execute, type NsJailResult } from './nsjail'; +import { getLatestRuntimeMatchingLanguageVersion } from './runtime'; +import type { Runtime } from './runtime'; + +const COMMAND_MAX_BYTES = 32 * 1024; +const COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; +const COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; +const REQUEST_KEYS = new Set([ + 'command', + 'cwd', + 'timeoutMs', + 'maxOutputBytes', +]); + +export interface WorkspaceCommandBody { + command: string; + cwd?: string; + timeoutMs?: number; + maxOutputBytes?: number; +} + +export interface WorkspaceCommandResponse { + exitCode: number | null; + signal?: string; + stdout: string; + stderr: string; + truncated: boolean; + timedOut: boolean; +} + +export class WorkspaceCommandRequestError extends Error { + constructor(message: string) { + super(message); + this.name = 'WorkspaceCommandRequestError'; + } +} + +export interface WorkspaceCommandExecutionOptions { + workspaceRoot: string; + executeNsJail?: typeof execute; + runtime?: Runtime; +} + +function safePortablePath(value: string): boolean { + return ( + value.length > 0 && + value.length <= 4096 && + value !== '..' && + !value.startsWith('/') && + !value.includes('\\') && + !value.includes('\0') && + !value.split('/').some((segment) => segment === '..') && + Buffer.from(value).toString('utf8') === value + ); +} + +function isWithin(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +} + +function integerInRange(value: unknown, minimum: number, maximum: number): value is number { + return Number.isSafeInteger(value) && Number(value) >= minimum && Number(value) <= maximum; +} + +export function hasWorkspaceCommandToken( + expected: string, + supplied: string | undefined, +): boolean { + const expectedBytes = Buffer.from(expected); + const suppliedBytes = Buffer.from(supplied ?? ''); + return ( + expectedBytes.length >= 32 && + suppliedBytes.length === expectedBytes.length && + timingSafeEqual(suppliedBytes, expectedBytes) + ); +} + +function validateBody(value: unknown): WorkspaceCommandBody { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new WorkspaceCommandRequestError('Command request must be an object'); + } + const body = value as Record; + if (Object.keys(body).some((key) => !REQUEST_KEYS.has(key))) { + throw new WorkspaceCommandRequestError('Command request contains unexpected fields'); + } + if ( + typeof body.command !== 'string' || + body.command.trim().length === 0 || + body.command.includes('\0') || + Buffer.from(body.command).toString('utf8') !== body.command || + Buffer.byteLength(body.command) > COMMAND_MAX_BYTES + ) { + throw new WorkspaceCommandRequestError('Command is invalid or exceeds its size limit'); + } + if (body.cwd !== undefined && (typeof body.cwd !== 'string' || !safePortablePath(body.cwd))) { + throw new WorkspaceCommandRequestError('Command working directory is invalid'); + } + if ( + body.timeoutMs !== undefined && + !integerInRange(body.timeoutMs, 1, COMMAND_MAX_TIMEOUT_MS) + ) { + throw new WorkspaceCommandRequestError('Command timeout is invalid'); + } + if ( + body.maxOutputBytes !== undefined && + !integerInRange(body.maxOutputBytes, 1, COMMAND_MAX_OUTPUT_BYTES) + ) { + throw new WorkspaceCommandRequestError('Command output limit is invalid'); + } + return body as unknown as WorkspaceCommandBody; +} + +function takeUtf8(value: string, budget: number): { value: string; bytes: number; truncated: boolean } { + const encoded = Buffer.from(value); + if (encoded.byteLength <= budget) { + return { value, bytes: encoded.byteLength, truncated: false }; + } + let end = budget; + while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1; + const bounded = encoded.subarray(0, end).toString('utf8'); + return { value: bounded, bytes: Buffer.byteLength(bounded), truncated: true }; +} + +function boundedResult(result: NsJailResult, maxOutputBytes: number): WorkspaceCommandResponse { + const stdout = takeUtf8(result.stdout, maxOutputBytes); + const stderr = takeUtf8(result.stderr, maxOutputBytes - stdout.bytes); + const timedOut = result.status === 'TO' || /time limit/i.test(result.message ?? ''); + return { + exitCode: result.signal || timedOut ? null : (result.code ?? 1), + ...(result.signal ? { signal: result.signal } : {}), + stdout: stdout.value, + stderr: stderr.value, + truncated: stdout.truncated || stderr.truncated || result.status === 'OL', + timedOut, + }; +} + +export async function executeWorkspaceCommand( + rawBody: unknown, + options: WorkspaceCommandExecutionOptions, +): Promise { + const body = validateBody(rawBody); + if ( + !path.isAbsolute(options.workspaceRoot) || + path.parse(options.workspaceRoot).root === options.workspaceRoot + ) { + throw new WorkspaceCommandRequestError('Configured workspace is unavailable'); + } + const workspaceRoot = await fsp.realpath(options.workspaceRoot); + const rootStat = await fsp.stat(workspaceRoot); + if (!rootStat.isDirectory()) { + throw new WorkspaceCommandRequestError('Configured workspace is unavailable'); + } + const cwd = await fsp.realpath(path.resolve(workspaceRoot, body.cwd ?? '.')) + .catch(() => { throw new WorkspaceCommandRequestError('Command working directory is unavailable'); }); + if (!isWithin(workspaceRoot, cwd) || !(await fsp.stat(cwd)).isDirectory()) { + throw new WorkspaceCommandRequestError('Command working directory is unavailable'); + } + const runtime = options.runtime ?? getLatestRuntimeMatchingLanguageVersion('bash', '*'); + if (!runtime) throw new Error('Bash runtime is unavailable'); + const envVars = { + ...runtime.env_vars, + HOME: '/mnt/data', + SANDBOX_LANGUAGE: 'bash', + }; + const extraPkgdirs = aggregateBashExtras(runtime.pkgdir, envVars); + const relativeCwd = path.relative(workspaceRoot, cwd).split(path.sep).join('/'); + const result = await (options.executeNsJail ?? execute)({ + command: [ + '/bin/bash', + '--noprofile', + '--norc', + '-c', + 'cd -- "$1" && exec /bin/bash --noprofile --norc -c "$2"', + 'librechat-code', + relativeCwd ? `/mnt/data/${relativeCwd}` : '/mnt/data', + body.command, + ], + envVars, + submissionDir: workspaceRoot, + pkgdir: runtime.pkgdir, + timeout: body.timeoutMs ?? 30_000, + memoryLimit: runtime.memory_limits.run, + outputMaxSize: body.maxOutputBytes ?? 256 * 1024, + extraPkgdirs, + identity: { + slot: 0, + uid: rootStat.uid, + gid: rootStat.gid, + perJobUid: true, + }, + }); + return boundedResult(result, body.maxOutputBytes ?? 256 * 1024); +} diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 2f9120eb..fc0d86c3 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -149,6 +149,16 @@ validates bounded results, and treats an unknown command failure as an uncertain mutation. The built-in CLI remains command-disabled until its supported NsJail adapter can safely map a registered directory without changing host ownership. +The supported `docker-nsjail` adapter enables that mapping only with +`--allow-workspace-commands` (or +`LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS=true`) and a registered worker/default +directory. It mounts only the canonical workspace, keeps the runner port +unpublished, authenticates its dedicated command route with an ephemeral +container capability, and runs Bash inside the existing NsJail profile. Direct +endpoint mode is rejected. This deployment permission does not replace the +per-call approval decision: LibreChat must apply its configurable tool-approval +hooks before dispatching `execute_command`. + Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, persistent local runner per session. A single sandbox endpoint is stateless and diff --git a/packages/code/README.md b/packages/code/README.md index 0fa1edf7..2d6e9fb8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -77,7 +77,7 @@ the same capability and seccomp policy as `docker-compose.mac.yml`: docker build --target local-oci-runtime \ -t librechat-code-runtime:local -f api/Dockerfile . -LIBRECHAT_CODE_RUNTIME_SUPERVISOR=docker-macos-nsjail \ +LIBRECHAT_CODE_RUNTIME_SUPERVISOR=docker-nsjail \ LIBRECHAT_CODE_RUNTIME_IMAGE=librechat-code-runtime:local \ LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE=./seccomp/nsjail.json \ LIBRECHAT_CODE_DOCKER_PACKAGES_PATH=./data/pkgs \ @@ -231,6 +231,26 @@ are reported as potentially committed mutations so the worker's durable quarantine remains armed. A concrete runtime adapter must prove its mount and identity behavior before the CLI can enable this composition. +The built-in Docker/NsJail adapter can be enabled explicitly for one registered +directory: + +```bash +LIBRECHAT_CODE_RUNTIME_SUPERVISOR=docker-nsjail \ +LIBRECHAT_CODE_RUNTIME_IMAGE=librechat-code-runtime:local \ +LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE=./seccomp/nsjail.json \ +LIBRECHAT_CODE_DOCKER_PACKAGES_PATH=./data/pkgs \ +librechat-code run --worker-dir /path/to/workspace --allow-workspace-commands +``` + +`docker-macos-nsjail` remains accepted as a compatibility alias. The worker +bind-mounts only that canonical directory into an unexposed runtime +container and submits commands to a private, capability-authenticated runner +route. The runner maps the mounted directory owner into NsJail without chowning +the directory, disables network access by default, rejects an escaping `cwd`, +and bounds command, time, stdout, and stderr. The endpoint supervisor cannot +enable this feature. This operator switch controls availability; LibreChat tool +approval hooks remain the user-facing allow/deny boundary for each invocation. + Reads reject absolute paths, traversal, escaping symlinks, non-regular files, and files larger than 1 MiB. The opened file is checked against its canonical in-workspace inode before it is read. Text search uses `rg` only to enumerate a diff --git a/packages/code/package.json b/packages/code/package.json index 2bce9bc6..5c1d3aef 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -26,6 +26,10 @@ "./workspace": { "types": "./dist/workspace.d.ts", "import": "./dist/workspace.js" + }, + "./workspace-runtime": { + "types": "./dist/workspace-runtime.d.ts", + "import": "./dist/workspace-runtime.js" } }, "bin": { diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 00ae78c8..1f081f86 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -67,7 +67,7 @@ test('CLI rejects an unknown runtime supervisor before entering the run loop', ( assert.notEqual(result.status, 0); assert.match( result.stderr, - /LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be endpoint, docker, or docker-macos-nsjail/, + /LIBRECHAT_CODE_RUNTIME_SUPERVISOR must be endpoint, docker, docker-nsjail, or docker-macos-nsjail/, ); }); @@ -102,7 +102,7 @@ test('CLI requires the macOS NsJail seccomp profile', () => { LIBRECHAT_CODE_URL: 'https://code.example/v1', LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', - LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-macos-nsjail', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker-nsjail', LIBRECHAT_CODE_RUNTIME_IMAGE: 'example/runtime:latest', }, }, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index d52ed957..fcbece02 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -21,7 +21,13 @@ import { } from './storage.js'; import { BridgeWorker } from './worker.js'; import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; -import { LocalWorkspaceTools } from './workspace.js'; +import { + LocalWorkspaceTools, + SandboxWorkspaceTools, +} from './workspace.js'; +import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; +import type { RuntimeSupervisor } from './runtime.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; import { BRIDGE_WORKSPACE_NAME_MAX_LENGTH, BridgeProtocolError, @@ -201,12 +207,15 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise 0; const workspaceId = @@ -253,6 +262,11 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise { const seccompProfile = resolve( required('LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE'), @@ -392,6 +396,105 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise { + const { seccompProfile, packagesPath, profileRevision } = + nsjailLaunchProfile!; + return { + capabilities: MACOS_NSJAIL_CAPABILITIES, + securityOptions: [`seccomp=${seccompProfile}`], + profileRevision, + restartStoppedContainers: false, + ...(fileRelayProfile ? { network: fileRelayProfile.network } : {}), + bindMounts: [ + { source: packagesPath, target: '/pkgs', readOnly: true }, + ...(workspaceMount ? [workspaceMount] : []), + ], + httpClient: 'bun' as const, + environment: { + SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', + ...(workspaceMount + ? { + SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', + SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, + SANDBOX_EXTERNAL_WORKSPACE_TOKEN: workspaceCommandToken!, + } + : {}), + ...(fileRelayProfile + ? { + EGRESS_GATEWAY_URL: fileRelayProfile.url, + SANDBOX_PRIME_CONCURRENCY: String(fileRelayLimits!.maxConcurrentRequests), + SANDBOX_UPLOAD_CONCURRENCY: String(fileRelayLimits!.maxConcurrentRequests), + SANDBOX_FILE_RELAY_TOKEN: fileRelayProfile.token, + SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: executionManifestPublicKey!, + } + : {}), + }, + }; + })() + : workspaceMount + ? { + bindMounts: [workspaceMount], + environment: { + SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', + SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, + SANDBOX_EXTERNAL_WORKSPACE_TOKEN: workspaceCommandToken!, + }, + } + : {}), + }) + : new EndpointRuntimeSupervisor({ + endpoint: sandboxEndpoint, + statefulWorkspace, + }); + if (allowWorkspaceCommands && workspaceTools) { + workspaceTools = new SandboxWorkspaceTools({ + workspaceTools, + commandWorkspaces: [workspaceId], + commandSandbox: new RuntimeWorkspaceCommandSandbox({ + runtimeSupervisor, + workerId, + incarnationId, + }), + }); + } + const capabilities = { + statefulWorkspace, + sandboxProfile: + process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? + (runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), + runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), + policyDigest: createHash('sha256').update(policy).digest('hex'), + ...(fileRelayEnabled ? { requiresReadyConfirmation: true } : {}), + ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), + }; + if (!isValidBridgeWorkerCapabilities(capabilities)) { + throw new Error( + 'LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid', + ); + } try { const worker = new BridgeWorker({ codeApiUrl, @@ -399,57 +502,7 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise { - const { seccompProfile, packagesPath, profileRevision } = - macLaunchProfile!; - return { - capabilities: MACOS_NSJAIL_CAPABILITIES, - securityOptions: [`seccomp=${seccompProfile}`], - profileRevision, - restartStoppedContainers: false, - ...(fileRelayProfile - ? { network: fileRelayProfile.network } - : {}), - bindMounts: [ - { - source: packagesPath, - target: '/pkgs', - readOnly: true, - }, - ], - httpClient: 'bun', - environment: { - SANDBOX_USE_CGROUPV2: 'false', - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', - ...(fileRelayProfile - ? { - EGRESS_GATEWAY_URL: fileRelayProfile.url, - SANDBOX_PRIME_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, - ), - SANDBOX_UPLOAD_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, - ), - SANDBOX_FILE_RELAY_TOKEN: fileRelayProfile.token, - SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', - SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: - executionManifestPublicKey!, - } - : {}), - }, - }; - })() - : {}), - }) - : new EndpointRuntimeSupervisor({ - endpoint: sandboxEndpoint, - statefulWorkspace, - }), + runtimeSupervisor, capabilities, workspaceTools, workspaceMutationQuarantine: mutationQuarantinePath diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index a7c0a830..0bee6941 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -4,4 +4,5 @@ export * from './pairing.js'; export * from './storage.js'; export * from './runtime.js'; export * from './workspace.js'; +export * from './workspace-runtime.js'; export * from './worker.js'; diff --git a/packages/code/src/runtime.test.ts b/packages/code/src/runtime.test.ts index 9ee67fae..ba40161f 100644 --- a/packages/code/src/runtime.test.ts +++ b/packages/code/src/runtime.test.ts @@ -110,6 +110,48 @@ test('docker runtime supervisor creates a networkless stateful runtime and execu assert.ok(health?.includes('--max-time')); }); +test('docker runtime supervisor permits only its fixed workspace command route', async () => { + const calls: string[][] = []; + const supervisor = new DockerRuntimeSupervisor({ + image: 'runner:latest', + environment: { + SANDBOX_EXTERNAL_WORKSPACE_TOKEN: 'private-workspace-capability', + }, + client: { + async run(args) { + calls.push(args); + if (args[0] === 'container') throw new Error('No such container'); + if (args[0] === 'run') return 'container'; + if (args.some((value) => value.includes('/api/v2/health'))) return '200'; + if (args.some((value) => value.includes('/api/v2/workspace/execute'))) { + const script = args.find((value) => value.includes("const b=await Bun.stdin.text")); + const marker = script?.match(/\\n([0-9a-f]{64})/)?.[1]; + return `{"exitCode":0}\n${marker}200`; + } + return ''; + }, + }, + httpClient: 'bun', + }); + const lease = await supervisor.acquire(assignment('workspace-route')); + const response = await lease.execute?.({ + body: '{"command":"pwd"}', + headers: { 'Content-Type': 'application/json' }, + path: '/api/v2/workspace/execute', + }); + assert.equal(response?.status, 200); + assert.ok(calls.some((args) => args.includes('http://127.0.0.1:2000/api/v2/workspace/execute'))); + const execution = calls.find((args) => + args.includes('http://127.0.0.1:2000/api/v2/workspace/execute'), + ); + assert.equal(execution?.includes('curl'), false); + assert.equal(JSON.stringify(execution).includes('private-workspace-capability'), false); + assert.equal( + execution?.some((value) => value.includes('X-LibreChat-Workspace-Token')), + true, + ); +}); + test('docker runtime supervisor preserves the legacy profile digest for the default network', async () => { const image = 'example/code-runtime:latest'; const legacyDigest = createHash('sha256') diff --git a/packages/code/src/runtime.ts b/packages/code/src/runtime.ts index cf6eea48..a71cdfc1 100644 --- a/packages/code/src/runtime.ts +++ b/packages/code/src/runtime.ts @@ -15,6 +15,8 @@ export interface RuntimeLease { export interface RuntimeExecutionRequest { body: string; headers: Record; + /** Fixed runner route; omitted for ordinary code execution. */ + path?: '/api/v2/execute' | '/api/v2/workspace/execute'; signal?: AbortSignal; } @@ -434,16 +436,19 @@ export class DockerRuntimeSupervisor implements RuntimeSupervisor { throw new Error('Runtime request headers cannot contain line breaks'); } const marker = randomBytes(32).toString('hex'); - const executeUrl = `http://127.0.0.1:${this.runnerPort}/api/v2/execute`; + const executeUrl = `http://127.0.0.1:${this.runnerPort}${request.path ?? '/api/v2/execute'}`; + const httpClient = request.path === '/api/v2/workspace/execute' + ? 'bun' + : this.httpClient; const output = await this.client.run( - this.httpClient === 'bun' + httpClient === 'bun' ? [ 'exec', '--interactive', name, 'bun', '-e', - `const b=await Bun.stdin.text();const r=await fetch(process.argv.at(-2),{method:'POST',headers:JSON.parse(process.argv.at(-1)),body:b});process.stdout.write(await r.text());process.stdout.write('\\n${marker}'+r.status);`, + `const b=await Bun.stdin.text();const h=JSON.parse(process.argv.at(-1));if(process.argv.at(-2).endsWith('/workspace/execute')){const t=process.env.SANDBOX_EXTERNAL_WORKSPACE_TOKEN;if(!t)throw new Error('workspace capability unavailable');h['X-LibreChat-Workspace-Token']=t;}const r=await fetch(process.argv.at(-2),{method:'POST',headers:h,body:b});process.stdout.write(await r.text());process.stdout.write('\\n${marker}'+r.status);`, executeUrl, JSON.stringify(request.headers), ] diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index ebea82ab..7fe28016 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -62,6 +62,55 @@ test('CLI trims an environment-configured worker directory', async (t) => { assert.doesNotMatch(result.stderr, /invalid workspace registration/i); }); +test('CLI refuses workspace commands without its Docker sandbox profile', async (t) => { + const workspaceRoot = await mkdtemp( + join(tmpdir(), 'librechat-code-command-workspace-'), + ); + t.after(() => rm(workspaceRoot, { recursive: true, force: true })); + const endpoint = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-commands', + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + assert.notEqual(endpoint.status, 0); + assert.match(endpoint.stderr, /require.*docker-nsjail runtime supervisor/i); + + const noWorkspace = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--allow-workspace-commands', + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_RUNTIME_SUPERVISOR: 'docker', + }, + }, + ); + assert.notEqual(noWorkspace.status, 0); + assert.match(noWorkspace.stderr, /require.*registered directory/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/packages/code/src/workspace-runtime.test.ts b/packages/code/src/workspace-runtime.test.ts new file mode 100644 index 00000000..c9d4cb14 --- /dev/null +++ b/packages/code/src/workspace-runtime.test.ts @@ -0,0 +1,153 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { RuntimeSupervisor } from './runtime.js'; + +const request = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'pwd', + cwd: 'src', + timeoutMs: 1234, + maxOutputBytes: 64, +}; + +test('executes a command through a stable stateful runtime session', async () => { + const requests: object[] = []; + const assignments: object[] = []; + const supervisor: RuntimeSupervisor = { + async acquire(assignment) { + assignments.push(assignment); + return { + sessionId: assignment.runtimeSessionId, + async execute(runtimeRequest) { + requests.push(runtimeRequest); + return { + status: 200, + body: JSON.stringify({ + exitCode: 0, + stdout: '/mnt/data/src\n', + stderr: '', + truncated: false, + timedOut: false, + }), + }; + }, + }; + }, + async reset() {}, + async quarantine() { throw new Error('must not quarantine'); }, + }; + const sandbox = new RuntimeWorkspaceCommandSandbox({ + runtimeSupervisor: supervisor, + workerId: 'worker-1', + incarnationId: 'incarnation-1', + }); + + assert.equal(sandbox.mutationFailuresAreAtomic, true); + assert.deepEqual(await sandbox.execute(request), { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + exitCode: 0, + stdout: '/mnt/data/src\n', + stderr: '', + truncated: false, + timedOut: false, + }); + const assignment = assignments[0] as { runtimeSessionId?: string }; + assert.match(assignment.runtimeSessionId ?? '', /^workspace-[0-9a-f]{40}$/); + assert.deepEqual(requests, [{ + path: '/api/v2/workspace/execute', + body: JSON.stringify({ + command: 'pwd', cwd: 'src', timeoutMs: 1234, maxOutputBytes: 64, + }), + headers: { + 'Content-Type': 'application/json', + 'X-Runtime-Session-Id': assignment.runtimeSessionId, + }, + signal: undefined, + }]); +}); + +test('quarantines the runtime and hides malformed sandbox responses', async () => { + const quarantined: string[] = []; + const supervisor: RuntimeSupervisor = { + async acquire(assignment) { + return { + sessionId: assignment.runtimeSessionId, + async execute() { + return { status: 200, body: '{"stdout":"host secret"}' }; + }, + }; + }, + async reset() {}, + async quarantine(sessionId) { quarantined.push(sessionId); }, + }; + const sandbox = new RuntimeWorkspaceCommandSandbox({ + runtimeSupervisor: supervisor, + workerId: 'worker-1', + incarnationId: 'incarnation-1', + }); + + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + error.mutationMayHaveCommitted === true && + !error.message.includes('host secret'), + ); + assert.equal(quarantined.length, 1); +}); + +test('keeps definite pre-execution request rejections clean', async () => { + let quarantined = false; + const supervisor: RuntimeSupervisor = { + async acquire(assignment) { + return { + sessionId: assignment.runtimeSessionId, + async execute() { + return { status: 400, body: '{"message":"Command working directory is unavailable"}' }; + }, + }; + }, + async reset() {}, + async quarantine() { quarantined = true; }, + }; + const sandbox = new RuntimeWorkspaceCommandSandbox({ + runtimeSupervisor: supervisor, + workerId: 'worker-1', + incarnationId: 'incarnation-1', + }); + + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'INVALID_REQUEST' && + error.mutationMayHaveCommitted === false && + !error.message.includes('working directory'), + ); + assert.equal(quarantined, false); +}); + +test('refuses a runtime lease that is not bound to the requested workspace', async () => { + let quarantined = false; + const supervisor: RuntimeSupervisor = { + async acquire() { return { sessionId: 'wrong', async execute() { throw new Error(); } }; }, + async reset() {}, + async quarantine() { quarantined = true; }, + }; + const sandbox = new RuntimeWorkspaceCommandSandbox({ + runtimeSupervisor: supervisor, + workerId: 'worker-1', + incarnationId: 'incarnation-1', + }); + await assert.rejects(sandbox.execute(request), /unavailable/i); + assert.equal(quarantined, true); +}); diff --git a/packages/code/src/workspace-runtime.ts b/packages/code/src/workspace-runtime.ts new file mode 100644 index 00000000..a5504bac --- /dev/null +++ b/packages/code/src/workspace-runtime.ts @@ -0,0 +1,143 @@ +import { createHash } from 'node:crypto'; + +import { BRIDGE_PROTOCOL_VERSION, isWorkspaceToolResult } from './protocol.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { + BridgeAssignment, + WorkspaceExecuteCommandRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; +import type { RuntimeSupervisor } from './runtime.js'; +import type { WorkspaceCommandSandbox } from './workspace.js'; + +const RESPONSE_KEYS = new Set([ + 'exitCode', + 'signal', + 'stdout', + 'stderr', + 'truncated', + 'timedOut', +]); + +export interface RuntimeWorkspaceCommandSandboxOptions { + runtimeSupervisor: RuntimeSupervisor; + workerId: string; + incarnationId: string; +} + +function runtimeSessionId(workerId: string, workspaceId: string): string { + return `workspace-${createHash('sha256') + .update(`${workerId}\0${workspaceId}`) + .digest('hex') + .slice(0, 40)}`; +} + +function commandAssignment( + options: RuntimeWorkspaceCommandSandboxOptions, + request: WorkspaceExecuteCommandRequest, + sessionId: string, +): BridgeAssignment { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + assignmentId: `workspace-command-${createHash('sha256') + .update(request.command) + .digest('hex') + .slice(0, 24)}`, + workerId: options.workerId, + incarnationId: options.incarnationId, + generation: 1, + leaseToken: 'local-workspace-command', + expiresAt: new Date(Date.now() + (request.timeoutMs ?? 30_000)).toISOString(), + runtimeSessionId: sessionId, + executionKind: 'workspace_tool', + request, + }; +} + +function parseResponse( + request: WorkspaceExecuteCommandRequest, + value: unknown, +): WorkspaceExecuteCommandResult { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('invalid sandbox response'); + } + const result = value as Record; + if (Object.keys(result).some((key) => !RESPONSE_KEYS.has(key))) { + throw new Error('invalid sandbox response'); + } + const candidate = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: result.exitCode as number | null, + ...(result.signal !== undefined ? { signal: result.signal as string } : {}), + stdout: result.stdout as string, + stderr: result.stderr as string, + truncated: result.truncated as boolean, + timedOut: result.timedOut as boolean, + }; + if (!isWorkspaceToolResult(request, candidate)) { + throw new Error('invalid sandbox response'); + } + return candidate; +} + +/** Runs workspace commands only through a stateful sandbox runner route. */ +export class RuntimeWorkspaceCommandSandbox implements WorkspaceCommandSandbox { + readonly mutationFailuresAreAtomic = true as const; + + constructor(private readonly options: RuntimeWorkspaceCommandSandboxOptions) {} + + async execute( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + ): Promise { + const sessionId = runtimeSessionId(this.options.workerId, request.workspaceId); + try { + const lease = await this.options.runtimeSupervisor.acquire( + commandAssignment(this.options, request, sessionId), + signal, + ); + if (!lease.execute || lease.sessionId !== sessionId) { + throw new Error('runtime does not provide isolated command execution'); + } + const response = await lease.execute({ + path: '/api/v2/workspace/execute', + body: JSON.stringify({ + command: request.command, + ...(request.cwd !== undefined ? { cwd: request.cwd } : {}), + ...(request.timeoutMs !== undefined ? { timeoutMs: request.timeoutMs } : {}), + ...(request.maxOutputBytes !== undefined + ? { maxOutputBytes: request.maxOutputBytes } + : {}), + }), + headers: { + 'Content-Type': 'application/json', + 'X-Runtime-Session-Id': sessionId, + }, + signal, + }); + if (response.status < 200 || response.status >= 300) { + if (response.status >= 400 && response.status < 500) { + throw new WorkspaceToolError( + 'Sandboxed command request was rejected', + response.status === 400 ? 'INVALID_REQUEST' : 'COMMAND_UNAVAILABLE', + ); + } + throw new Error(`sandbox rejected command with HTTP ${response.status}`); + } + return parseResponse(request, JSON.parse(response.body) as unknown); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + await this.options.runtimeSupervisor + .quarantine(sessionId, 'Sandboxed workspace command failed', error) + .catch(() => undefined); + throw new WorkspaceToolError( + 'Sandboxed command execution unavailable', + 'COMMAND_UNAVAILABLE', + true, + ); + } + } +} diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index b1be1cbb..c98182b0 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -3,7 +3,7 @@ import test from 'node:test'; import { BridgeProtocolError } from './protocol.js'; import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; -import { WorkspaceToolError } from './workspace.js'; +import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; @@ -1055,6 +1055,71 @@ test('worker clears quarantine after an atomic executor rejection is settled', a assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); }); +test('worker clears quarantine after a composed command is cleanly rejected', async () => { + const lifecycle: string[] = []; + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + throw new WorkspaceToolError('Sandboxed command request was rejected', 'INVALID_REQUEST'); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-command-clean-rejection', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'pwd', + }, + }); + assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); +}); + test('worker retains quarantine when an atomic executor cannot confirm durability', async () => { const lifecycle: string[] = []; const workspaceCapabilities = { diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index ab09ac28..e0cacb48 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -1565,6 +1565,7 @@ test('composes sandboxed commands without exposing them on unconfigured workspac workspaceTools: local, commandWorkspaces: ['sandboxed'], commandSandbox: { + mutationFailuresAreAtomic: true, async execute(request) { requests.push(request); return { @@ -1581,6 +1582,7 @@ test('composes sandboxed commands without exposing them on unconfigured workspac }, }); + assert.equal(tools.mutationFailuresAreAtomic, true); assert.deepEqual(tools.capabilities.operations, [ 'read_file', 'search_text', @@ -1645,6 +1647,7 @@ test('fails closed on invalid or failed sandbox command results', async (t) => { for (const execute of [ async () => ({ ...request, exitCode: 0, stdout: 'ok', stderr: '' }), async () => { throw new Error('container details'); }, + async () => { throw new WorkspaceToolError('untrusted clean claim', 'COMMAND_UNAVAILABLE'); }, ]) { const tools = new SandboxWorkspaceTools({ workspaceTools: local, @@ -1659,6 +1662,7 @@ test('fails closed on invalid or failed sandbox command results', async (t) => { error.mutationMayHaveCommitted === true && !error.message.includes('container details'), ); + assert.equal(tools.mutationFailuresAreAtomic, undefined); } }); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 94366dff..3f8ed7b6 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -91,6 +91,11 @@ export interface WorkspaceToolExecutor { * confinement; this package deliberately never falls back to a host shell. */ export interface WorkspaceCommandSandbox { + /** + * True only when every thrown WorkspaceToolError proves no command-side + * mutation committed unless mutationMayHaveCommitted is explicitly set. + */ + mutationFailuresAreAtomic?: true; execute( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -1448,16 +1453,18 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { } } -/** - * Composes ordinary workspace tools with an explicitly supplied sandboxed - * command boundary. Command failures are intentionally not declared atomic: - * callers must retain their durable mutation quarantine until settlement. - */ +/** Composes ordinary workspace tools with an explicitly supplied sandboxed command boundary. */ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { readonly capabilities: BridgeWorkspaceToolCapabilities; + readonly mutationFailuresAreAtomic?: true; private readonly commandWorkspaces: ReadonlySet; constructor(private readonly options: SandboxWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = + options.workspaceTools.mutationFailuresAreAtomic === true && + options.commandSandbox.mutationFailuresAreAtomic === true + ? true + : undefined; const base = options.workspaceTools.capabilities; const registeredIds = new Set(base.workspaces.map(({ id }) => id)); const commandWorkspaces = new Set(options.commandWorkspaces); @@ -1516,7 +1523,15 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { try { result = await this.options.commandSandbox.execute(request, signal); } catch (error) { - if (error instanceof WorkspaceToolError) throw error; + if (error instanceof WorkspaceToolError) { + if ( + this.options.commandSandbox.mutationFailuresAreAtomic === true || + error.mutationMayHaveCommitted + ) { + throw error; + } + throw new WorkspaceToolError(error.message, error.code, true); + } throw new WorkspaceToolError( 'Sandboxed command execution unavailable', 'COMMAND_UNAVAILABLE', From 94c21a28e55e8f6b3df4c2a88657cf612103032e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 3 Sep 2026 21:32:53 -0400 Subject: [PATCH 033/116] =?UTF-8?q?=F0=9F=9B=96=20feat:=20Run=20BYOM=20Com?= =?UTF-8?q?mands=20in=20Native=20Sandboxes=20(#101)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): add native SRT command sandbox * fix(code): harden native SRT command lifecycle * fix(code): clarify native SRT containment * fix(code): close native sandbox review gaps --- docs/remote-bridge/README.md | 43 +- packages/code/Dockerfile | 4 +- packages/code/README.md | 86 +++- packages/code/package-lock.json | 56 ++- packages/code/package.json | 9 +- packages/code/src/cli.test.ts | 23 ++ packages/code/src/cli.ts | 101 ++++- packages/code/src/index.ts | 1 + packages/code/src/native-sandbox.test.ts | 320 +++++++++++++++ packages/code/src/native-sandbox.ts | 481 +++++++++++++++++++++++ packages/code/src/workspace-cli.test.ts | 12 +- 11 files changed, 1084 insertions(+), 52 deletions(-) create mode 100644 packages/code/src/native-sandbox.test.ts create mode 100644 packages/code/src/native-sandbox.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index fc0d86c3..ab7e2f64 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -137,27 +137,42 @@ listings, and later tool results necessarily cross the outbound bridge to Code API and the model. Treat them as explicit tool outputs, apply the same retention and audit policy as chat content, and do not register a directory containing secrets. The -default operations are read-only. The bridge protocol reserves a bounded -`execute_command` operation, but the local filesystem executor and CLI do not -advertise it. A later layer must bind it to a sandboxed process boundary and -LibreChat's tool-approval hooks before it becomes dispatchable. +default operations are read-only. Operators can explicitly add bounded +`execute_command` support with `--allow-workspace-commands` (or +`LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS=true`). The CLI prepares its sandbox +before registration, so Code API cannot dispatch commands to an unavailable +boundary. LibreChat's tool-approval hooks remain the per-call decision point. The package exposes `SandboxWorkspaceTools` for composing that boundary without -ever invoking a host shell. It requires an explicit sandbox implementation and -an allowlist of workspace IDs, preserves per-workspace operation restrictions, -validates bounded results, and treats an unknown command failure as an uncertain -mutation. The built-in CLI remains command-disabled until its supported NsJail -adapter can safely map a registered directory without changing host ownership. - -The supported `docker-nsjail` adapter enables that mapping only with +ever invoking an unsandboxed host shell. It requires an explicit sandbox +implementation and an allowlist of workspace IDs, preserves per-workspace +operation restrictions, validates bounded results, and treats an unknown +command failure as an uncertain mutation. + +Native SRT is the MVP and default command backend on a user's chosen laptop or +VM. It uses Seatbelt on macOS, bubblewrap/seccomp on Linux, and the SRT +restricted-account helper on Windows. It confines writes to the registered +workspace, denies reads of the worker home and control files, strips worker +credentials, and denies network egress by default. Startup fails closed when +the platform dependencies are unavailable; there is no unsandboxed fallback. +Use `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` for an explicit comma-separated +egress allowlist. +Linux hosts must provide Bash at `/bin/bash`, `bubblewrap`, `socat`, and +`ripgrep`; macOS uses system facilities. Windows requires SRT's one-time +restricted-account setup. + +The optional `docker-nsjail` adapter enables a stronger container boundary with `--allow-workspace-commands` (or `LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS=true`) and a registered worker/default directory. It mounts only the canonical workspace, keeps the runner port unpublished, authenticates its dedicated command route with an ephemeral container capability, and runs Bash inside the existing NsJail profile. Direct -endpoint mode is rejected. This deployment permission does not replace the -per-call approval decision: LibreChat must apply its configurable tool-approval -hooks before dispatching `execute_command`. +endpoint mode cannot be used as the `runtime` command backend, but endpoint +runtime supervision can coexist with native SRT commands. Set +`LIBRECHAT_CODE_COMMAND_SANDBOX=runtime` to select Docker/NsJail explicitly. +This deployment permission does not replace the per-call approval decision: +LibreChat must apply its configurable tool-approval hooks before dispatching +`execute_command`. Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true` and route the CLI's `{runtimeSessionId}` endpoint template to an isolated, diff --git a/packages/code/Dockerfile b/packages/code/Dockerfile index 015ca2d0..93ed9c57 100644 --- a/packages/code/Dockerfile +++ b/packages/code/Dockerfile @@ -7,11 +7,13 @@ RUN npm run build FROM node:24-alpine ENV NODE_ENV=production -RUN apk add --no-cache ripgrep \ +RUN apk add --no-cache bash bubblewrap ripgrep socat \ && addgroup -S librechat-code \ && adduser -S librechat-code -G librechat-code WORKDIR /app COPY --from=build /app/package.json ./package.json +COPY --from=build /app/package-lock.json ./package-lock.json +RUN npm ci --omit=dev COPY --from=build /app/dist ./dist USER librechat-code ENTRYPOINT ["node", "dist/cli.js"] diff --git a/packages/code/README.md b/packages/code/README.md index 2d6e9fb8..3e8bf72d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,9 +3,11 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. -The CLI owns the runtime-supervisor seam. The bundled endpoint adapter connects -to an already-running loopback Code Interpreter sandbox; future adapters create -and isolate the runtime themselves. It connects outbound to Code API, +The CLI owns the runtime-supervisor seam. Native workspace commands use +Anthropic's open-source Sandbox Runtime (SRT) on the worker machine. The +bundled endpoint adapter can also connect to an already-running loopback Code +Interpreter sandbox, while the optional Docker adapter provides a stronger +container/NsJail profile. The worker connects outbound to Code API, long-polls for assignments, sends them to the local runtime, and returns fenced results. The VM does not need an inbound public port. @@ -37,7 +39,61 @@ Use `--identity ` while pairing and `LIBRECHAT_CODE_IDENTITY_FILE=` while running to override the identity file location. -## Docker runtime supervisor (programmatic adapter) +## Native BYOM sandbox (default) + +The MVP command sandbox runs directly on the user's chosen laptop or VM. It +does not require Docker. Enable commands for an existing project or for a new +application-owned directory: + +```bash +librechat-code run --worker-dir /path/to/project --allow-workspace-commands + +# Git is optional; this creates and reuses an empty workspace. +librechat-code run --default-workspace --allow-workspace-commands +``` + +`native-srt` is the default command sandbox unless a Docker/NsJail runtime was +selected. It uses `@anthropic-ai/sandbox-runtime`: Seatbelt on macOS, +bubblewrap plus seccomp on Linux, and the SRT restricted-account helper on +Windows. Startup fails before worker registration when the platform or its +dependencies are unavailable. There is no unsandboxed command fallback. + +The bridge worker remains outside the sandbox so it can maintain its outbound +Code API connection. Each command and its descendants run inside SRT with: + +- write access restricted to the one canonical registered workspace; +- read access denied to the worker's home directory except for that workspace; +- paired identity and mutation-quarantine files explicitly denied; +- `LIBRECHAT_CODE_*` and nonessential inherited environment variables removed; +- network egress denied by default, local binding denied, and Unix sockets + denied; and +- bounded time and aggregate output, with best-effort process-group termination + on cancellation, timeout, and completion. + +SRT restrictions remain inherited by descendants. Windows additionally uses a +kill-on-close Job Object. Native macOS does not provide an equivalent hard +process-lifetime boundary: a deliberately daemonized descendant can outlive +the command while remaining confined to the approved workspace and network +policy. This matches the personal-machine SRT trust model; use the Docker/NsJail +backend or a dedicated VM boundary when hard teardown of adversarial process +trees is required. + +Linux hosts need Bash at `/bin/bash`, `bubblewrap`, `socat`, and `ripgrep`; macOS uses system +facilities. Follow SRT's one-time restricted-account setup when using Windows. +An operator may allow explicit egress destinations with the comma-separated +`LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` setting. Treat that as a security +policy: an allowed destination can receive workspace data. The normalized +allowlist is included in the worker policy digest. Tool approval hooks remain +the user-facing allow/deny boundary for each invocation. + +Select the backend explicitly when desired: + +```bash +LIBRECHAT_CODE_COMMAND_SANDBOX=native-srt librechat-code run \ + --worker-dir /path/to/project --allow-workspace-commands +``` + +## Docker runtime supervisor (optional hardened adapter) `DockerRuntimeSupervisor` is the first self-contained local OCI adapter. It owns one named container per runtime session, does not publish the runner port, @@ -158,7 +214,9 @@ librechat-code run Optional environment variables: -- `LIBRECHAT_CODE_SANDBOX_PROFILE`: capability label; defaults to `nsjail`. +- `LIBRECHAT_CODE_SANDBOX_PROFILE`: capability label; defaults to + `anthropic-srt` for native workspace commands, `oci-docker` for Docker, and + the existing `nsjail` label otherwise. - `LIBRECHAT_CODE_RUNTIMES`: comma-separated capability labels. - `LIBRECHAT_CODE_POLICY`: local policy description hashed into the worker's registration; defaults to `default-deny`. @@ -219,8 +277,9 @@ capabilities; absolute host paths remain local to the worker process. The protocol also defines a bounded `execute_command` request and result for a sandbox-backed executor. Commands are treated as workspace mutations and cannot be advertised without durable quarantine storage. `LocalWorkspaceTools` never -runs them in the worker host process; the CLI does not advertise command support -until a sandbox runtime executor is configured. +runs them directly in the trusted worker process; the CLI does not advertise +command support until its selected SRT or Docker/NsJail sandbox has passed +startup checks. `SandboxWorkspaceTools` is the composition boundary for that runtime. It adds `execute_command` only to workspace IDs explicitly backed by a @@ -228,8 +287,8 @@ until a sandbox runtime executor is configured. executor, and validates the sandbox's complete result before returning it. It does not include a shell fallback. Invalid responses and unknown sandbox errors are reported as potentially committed mutations so the worker's durable -quarantine remains armed. A concrete runtime adapter must prove its mount and -identity behavior before the CLI can enable this composition. +quarantine remains armed. The concrete adapter must pass its platform and +identity checks before the CLI can enable this composition. The built-in Docker/NsJail adapter can be enabled explicitly for one registered directory: @@ -239,6 +298,7 @@ LIBRECHAT_CODE_RUNTIME_SUPERVISOR=docker-nsjail \ LIBRECHAT_CODE_RUNTIME_IMAGE=librechat-code-runtime:local \ LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE=./seccomp/nsjail.json \ LIBRECHAT_CODE_DOCKER_PACKAGES_PATH=./data/pkgs \ +LIBRECHAT_CODE_COMMAND_SANDBOX=runtime \ librechat-code run --worker-dir /path/to/workspace --allow-workspace-commands ``` @@ -247,9 +307,11 @@ bind-mounts only that canonical directory into an unexposed runtime container and submits commands to a private, capability-authenticated runner route. The runner maps the mounted directory owner into NsJail without chowning the directory, disables network access by default, rejects an escaping `cwd`, -and bounds command, time, stdout, and stderr. The endpoint supervisor cannot -enable this feature. This operator switch controls availability; LibreChat tool -approval hooks remain the user-facing allow/deny boundary for each invocation. +and bounds command, time, stdout, and stderr. The endpoint supervisor cannot be +used as the `runtime` command backend, but it can coexist with the default +native SRT command backend. This operator switch controls availability; +LibreChat tool approval hooks remain the user-facing allow/deny boundary for +each invocation. Reads reject absolute paths, traversal, escaping symlinks, non-regular files, and files larger than 1 MiB. The opened file is checked against its canonical diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index 15ca1ad6..a2da04a0 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -8,6 +8,9 @@ "name": "@librechat/code", "version": "0.1.0", "license": "Apache-2.0", + "dependencies": { + "@anthropic-ai/sandbox-runtime": "0.0.75" + }, "bin": { "librechat-code": "dist/cli.js" }, @@ -16,9 +19,33 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20" + "node": ">=20.11" } }, + "node_modules/@anthropic-ai/sandbox-runtime": { + "version": "0.0.75", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.75.tgz", + "integrity": "sha512-oqAKi6QtkT2DpLwFoDCDD757zw2i6ftpLTyV8rNSV9QWF53q2m1JxEs0RYXv2CIXtCoje4RGYQylagn15RKmww==", + "license": "Apache-2.0", + "dependencies": { + "@pondwader/socks5-server": "^1.0.10", + "commander": "^12.1.0", + "node-forge": "^1.4.0", + "zod": "^3.24.1" + }, + "bin": { + "srt": "dist/cli.js" + }, + "engines": { + "node": ">=20.11.0" + } + }, + "node_modules/@pondwader/socks5-server": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", + "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "22.20.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", @@ -29,6 +56,24 @@ "undici-types": "~6.21.0" } }, + "node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/node-forge": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", + "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==", + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -49,6 +94,15 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true, "license": "MIT" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/packages/code/package.json b/packages/code/package.json index 5c1d3aef..cfb97ddd 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -30,6 +30,10 @@ "./workspace-runtime": { "types": "./dist/workspace-runtime.d.ts", "import": "./dist/workspace-runtime.js" + }, + "./native-sandbox": { + "types": "./dist/native-sandbox.d.ts", + "import": "./dist/native-sandbox.js" } }, "bin": { @@ -49,6 +53,9 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20" + "node": ">=20.11" + }, + "dependencies": { + "@anthropic-ai/sandbox-runtime": "0.0.75" } } diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 1f081f86..7bdb3d30 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -71,6 +71,29 @@ test('CLI rejects an unknown runtime supervisor before entering the run loop', ( ); }); +test('CLI rejects an unknown command sandbox before entering the run loop', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_SANDBOX: 'host-shell', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime/, + ); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index fcbece02..06aec93b 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -20,12 +20,13 @@ import { saveWorkspaceMutationQuarantine, } from './storage.js'; import { BridgeWorker } from './worker.js'; -import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; import { LocalWorkspaceTools, SandboxWorkspaceTools, } from './workspace.js'; +import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; import type { RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; import { @@ -267,6 +268,18 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise path != null, + ), + allowedDomains: commandAllowedDomains, + }) + : undefined; if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ workspaceTools, commandWorkspaces: [workspaceId], - commandSandbox: new RuntimeWorkspaceCommandSandbox({ - runtimeSupervisor, - workerId, - incarnationId, - }), + commandSandbox: + nativeCommandSandbox ?? + new RuntimeWorkspaceCommandSandbox({ + runtimeSupervisor, + workerId, + incarnationId, + }), }); } const capabilities = { statefulWorkspace, sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? - (runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), + (allowWorkspaceCommands && commandSandboxMode === 'native-srt' + ? 'anthropic-srt' + : runtimeMode.startsWith('docker') + ? 'oci-docker' + : 'nsjail'), runtimes: list(process.env.LIBRECHAT_CODE_RUNTIMES), - policyDigest: createHash('sha256').update(policy).digest('hex'), + policyDigest: createHash('sha256') + .update(policy) + .update( + allowWorkspaceCommands && commandSandboxMode === 'native-srt' + ? `\0native-srt\0${commandAllowedDomains.join('\0')}` + : '', + ) + .digest('hex'), ...(fileRelayEnabled ? { requiresReadyConfirmation: true } : {}), ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { + await fileRelaySupervisor?.stop().catch(() => undefined); throw new Error( 'LIBRECHAT_CODE_SANDBOX_PROFILE or LIBRECHAT_CODE_RUNTIMES is invalid', ); } + try { + await nativeCommandSandbox?.prepare(); + } catch (error) { + await fileRelaySupervisor?.stop().catch(() => undefined); + throw error; + } try { const worker = new BridgeWorker({ codeApiUrl, @@ -587,7 +642,11 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise true, + async checkDependenciesAsync() { + return { warnings: [], errors: options.dependencyErrors ?? [] }; + }, + async initialize(value: SandboxRuntimeConfig) { + config = value; + }, + async wrapWithSandboxArgv(command: string) { + return { + argv: ['/bin/bash', '-c', command], + env: { PATH: process.env.PATH }, + }; + }, + annotateStderrWithSandboxFailures(_commandId: string, stderr: string) { + return stderr; + }, + cleanupAfterCommand() {}, + async reset() { + reset = true; + }, + }; + return { + manager, + get config() { + return config; + }, + get reset() { + return reset; + }, + }; +} + +test('initializes SRT with a default-deny network and scrubbed worker credentials', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + const identity = join(tmpdir(), 'librechat-code-identity.json'); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + protectedPaths: [identity], + environment: { + PATH: '/usr/bin', + Path: '/windows/system32', + LANG: 'en_US.UTF-8', + lc_api_token: 'lowercase-secret', + LIBRECHAT_CODE_WORKER_TOKEN: 'secret', + AWS_SECRET_ACCESS_KEY: 'secret', + }, + manager: fake.manager, + }); + + await sandbox.prepare(); + const canonicalRoot = await realpath(root); + const canonicalIdentity = await realpath(identity).catch(async () => + join(await realpath(tmpdir()), 'librechat-code-identity.json'), + ); + const canonicalHome = await realpath(homedir()); + assert.deepEqual(fake.config?.network.allowedDomains, []); + assert.equal(fake.config?.network.strictAllowlist, true); + assert.equal(fake.config?.network.allowAllUnixSockets, false); + assert.deepEqual(fake.config?.filesystem.allowRead, [canonicalRoot]); + assert.deepEqual(fake.config?.filesystem.allowWrite, [canonicalRoot]); + assert.ok(fake.config?.filesystem.denyRead.includes(canonicalHome)); + assert.ok(fake.config?.filesystem.denyWrite.includes(canonicalIdentity)); + const denied = fake.config?.credentials?.envVars?.map(({ name }) => name); + assert.ok(denied?.includes('LIBRECHAT_CODE_WORKER_TOKEN')); + assert.ok(denied?.includes('AWS_SECRET_ACCESS_KEY')); + assert.ok(!denied?.includes('PATH')); + assert.ok(denied?.includes('Path')); + assert.ok(denied?.includes('lc_api_token')); + await sandbox.close(); + assert.equal(fake.reset, true); +}); + +test('filters environment names case-insensitively only on Windows', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + platform: 'win32', + environment: { + PATH: '/usr/bin', + Path: 'C:\\Windows\\System32', + LC_API_TOKEN: 'secret', + librechat_code_worker_token: 'secret', + }, + manager: fake.manager, + }); + + await sandbox.prepare(); + const denied = fake.config?.credentials?.envVars?.map(({ name }) => name); + assert.ok(!denied?.includes('PATH')); + assert.ok(!denied?.includes('Path')); + assert.ok(!denied?.includes('LC_API_TOKEN')); + assert.ok(denied?.includes('librechat_code_worker_token')); +}); + +test('fails closed when the configured POSIX shell is unavailable', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + platform: 'linux', + shellPath: join(root, 'missing-bash'), + manager: fakeManager().manager, + }); + + await assert.rejects( + sandbox.prepare(), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + /shell is unavailable/i.test(error.message), + ); +}); + +test('fails closed when SRT dependencies are unavailable', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager({ dependencyErrors: ['bubblewrap missing'] }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + + await assert.rejects( + sandbox.prepare(), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + /bubblewrap missing/.test(error.message), + ); +}); + +test('refuses workspace roots that expose worker home or control files', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + const controlDirectory = join(root, '.control'); + await mkdir(controlDirectory); + const controlFile = join(controlDirectory, 'identity.json'); + await writeFile(controlFile, '{}'); + t.after(() => rm(root, { recursive: true, force: true })); + + await assert.rejects( + new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: homedir(), + manager: fakeManager().manager, + }).prepare(), + /cannot contain the worker home directory/i, + ); + await assert.rejects( + new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + protectedPaths: [controlFile], + manager: fakeManager().manager, + }).prepare(), + /cannot contain worker control files/i, + ); +}); + +test('executes in the canonical workspace and bounds aggregate output', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + await mkdir(join(root, 'src')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + assert.equal(sandbox.mutationFailuresAreAtomic, true); + assert.deepEqual( + await sandbox.execute({ + ...request, + command: "printf '1234567890'; printf 'abcdefghij' >&2", + cwd: 'src', + maxOutputBytes: 12, + }), + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + exitCode: 0, + stdout: '1234567890', + stderr: 'ab', + truncated: true, + timedOut: false, + }, + ); +}); + +test('rejects an escaping or unavailable command working directory', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + await assert.rejects( + sandbox.execute({ ...request, cwd: '..' }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('terminates detached command descendants before returning', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + const result = await sandbox.execute({ + ...request, + command: '(sleep 0.2; printf late > late.txt) >/dev/null 2>&1 &', + }); + assert.equal(result.exitCode, 0); + await new Promise((resolve) => setTimeout(resolve, 350)); + await assert.rejects(access(join(root, 'late.txt'))); +}); + +test('reports cancellation after command start as a potentially committed mutation', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + const controller = new AbortController(); + const execution = sandbox.execute( + { ...request, command: 'sleep 30' }, + controller.signal, + ); + setTimeout(() => controller.abort(), 25); + + await assert.rejects( + execution, + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + error.mutationMayHaveCommitted === true, + ); +}); + +test('closes stdin immediately when the command protocol provides no input', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + const result = await sandbox.execute({ + ...request, + command: 'cat', + timeoutMs: 250, + }); + assert.equal(result.exitCode, 0); + assert.equal(result.timedOut, false); +}); + +test('maps platform-native exit statuses into the bridge protocol range', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const spawnCommand = () => { + const child = new EventEmitter() as ChildProcessWithoutNullStreams; + Object.assign(child, { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + pid: undefined, + kill: () => true, + }); + queueMicrotask(() => child.emit('close', 300, null)); + return child; + }; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + spawnCommand, + }); + + const result = await sandbox.execute(request); + assert.equal(result.exitCode, 1); +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts new file mode 100644 index 00000000..2136f9b3 --- /dev/null +++ b/packages/code/src/native-sandbox.ts @@ -0,0 +1,481 @@ +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { homedir } from 'node:os'; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from 'node:path'; +import { constants as fsConstants } from 'node:fs'; +import { access, realpath, stat } from 'node:fs/promises'; + +import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; + +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + isWorkspaceToolRequest, +} from './protocol.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { + ChildProcessWithoutNullStreams, + SpawnOptionsWithoutStdio, +} from 'node:child_process'; +import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + WorkspaceExecuteCommandRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; +import type { WorkspaceCommandSandbox } from './workspace.js'; + +const SAFE_CHILD_ENV_NAMES = new Set([ + 'COLORTERM', + 'HOME', + 'LANG', + 'LC_ALL', + 'LOGNAME', + 'NO_COLOR', + 'PATH', + 'SHELL', + 'TERM', + 'TMPDIR', + 'USER', +]); + +interface NativeSandboxManager { + isSupportedPlatform(): boolean; + checkDependenciesAsync(): Promise<{ warnings: string[]; errors: string[] }>; + initialize(config: SandboxRuntimeConfig): Promise; + wrapWithSandboxArgv( + command: string, + binShell?: string, + customConfig?: Partial, + abortSignal?: AbortSignal, + cwd?: string, + options?: { commandId?: string; commandText?: string }, + ): Promise<{ argv: string[]; env: NodeJS.ProcessEnv }>; + annotateStderrWithSandboxFailures(commandId: string, stderr: string): string; + cleanupAfterCommand(): void; + reset(): Promise; +} + +type SpawnCommand = ( + command: string, + args: readonly string[], + options: SpawnOptionsWithoutStdio, +) => ChildProcessWithoutNullStreams; + +export interface NativeSrtWorkspaceCommandSandboxOptions { + workspaceRoot: string; + /** Trusted worker files that must never become workspace-readable or writable. */ + protectedPaths?: string[]; + allowedDomains?: string[]; + environment?: NodeJS.ProcessEnv; + manager?: NativeSandboxManager; + spawnCommand?: SpawnCommand; + homeDirectory?: string; + platform?: NodeJS.Platform; + /** Trusted shell path used by SRT on POSIX hosts. */ + shellPath?: string; +} + +function isWithin(root: string, candidate: string): boolean { + const path = relative(root, candidate); + return ( + path === '' || + (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path)) + ); +} + +async function canonicalPath(path: string): Promise { + const absolute = resolve(path); + let cursor = absolute; + const missingSegments: string[] = []; + for (;;) { + try { + return join(await realpath(cursor), ...missingSegments); + } catch { + const parent = dirname(cursor); + if (parent === cursor) + throw new Error(`Cannot canonicalize protected path: ${path}`); + missingSegments.unshift(basename(cursor)); + cursor = parent; + } + } +} + +function boundedUtf8(buffer: Buffer, budget: number): string { + let end = Math.min(buffer.byteLength, budget); + while (end > 0) { + const value = buffer.subarray(0, end).toString('utf8'); + if (Buffer.byteLength(value) <= budget) return value; + end -= 1; + } + return ''; +} + +function safeEnvironmentNames( + environment: NodeJS.ProcessEnv, + platform: NodeJS.Platform, +): string[] { + return Object.keys(environment) + .filter((name) => { + const normalized = platform === 'win32' ? name.toUpperCase() : name; + return ( + normalized.startsWith('LIBRECHAT_CODE_') || + (!SAFE_CHILD_ENV_NAMES.has(normalized) && !normalized.startsWith('LC_')) + ); + }) + .sort(); +} + +export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { + readonly mutationFailuresAreAtomic = true as const; + private readonly manager: NativeSandboxManager; + private readonly spawnCommand: SpawnCommand; + private readonly environment: NodeJS.ProcessEnv; + private readonly platform: NodeJS.Platform; + private initialized?: Promise; + private canonicalRoot?: string; + + constructor( + private readonly options: NativeSrtWorkspaceCommandSandboxOptions, + ) { + this.manager = options.manager ?? SandboxManager; + this.spawnCommand = options.spawnCommand ?? spawn; + this.environment = { ...(options.environment ?? process.env) }; + this.platform = options.platform ?? process.platform; + } + + /** Fail closed before the worker advertises command execution. */ + async prepare(): Promise { + await this.initialize(); + } + + private async initialize(): Promise { + if (this.initialized) return this.initialized; + this.initialized = this.initializeOnce().catch(async (error) => { + await this.manager.reset().catch(() => undefined); + this.initialized = undefined; + throw error; + }); + return this.initialized; + } + + private async initializeOnce(): Promise { + if (!this.manager.isSupportedPlatform()) { + throw new WorkspaceToolError( + 'Native sandbox is unsupported on this platform', + 'COMMAND_UNAVAILABLE', + ); + } + const root = await realpath(this.options.workspaceRoot); + if (!(await stat(root)).isDirectory()) { + throw new WorkspaceToolError( + 'Native sandbox workspace is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const home = await canonicalPath(this.options.homeDirectory ?? homedir()); + if (isWithin(root, home)) { + throw new WorkspaceToolError( + 'Native sandbox workspace cannot contain the worker home directory', + 'REGISTRATION_INVALID', + ); + } + const protectedPaths = await Promise.all( + (this.options.protectedPaths ?? []).map(canonicalPath), + ); + if (protectedPaths.some((path) => isWithin(root, path))) { + throw new WorkspaceToolError( + 'Native sandbox workspace cannot contain worker control files', + 'REGISTRATION_INVALID', + ); + } + const dependencies = await this.manager.checkDependenciesAsync(); + if (dependencies.errors.length > 0) { + throw new WorkspaceToolError( + `Native sandbox dependencies are unavailable: ${dependencies.errors.join('; ')}`, + 'COMMAND_UNAVAILABLE', + ); + } + if (this.platform !== 'win32') { + try { + await access(this.options.shellPath ?? '/bin/bash', fsConstants.X_OK); + } catch { + throw new WorkspaceToolError( + `Native sandbox shell is unavailable: ${this.options.shellPath ?? '/bin/bash'}`, + 'COMMAND_UNAVAILABLE', + ); + } + } + const config: SandboxRuntimeConfig = { + network: { + allowedDomains: [...(this.options.allowedDomains ?? [])], + deniedDomains: [], + strictAllowlist: true, + allowAllUnixSockets: false, + allowLocalBinding: false, + }, + filesystem: { + denyRead: [home], + allowRead: [root], + allowWrite: [root], + denyWrite: protectedPaths, + allowGitConfig: false, + }, + credentials: { + files: protectedPaths.map((path) => ({ + path, + mode: 'deny' as const, + })), + envVars: safeEnvironmentNames(this.environment, this.platform).map((name) => ({ + name, + mode: 'deny' as const, + })), + }, + allowAppleEvents: false, + enableWeakerNestedSandbox: false, + enableWeakerNetworkIsolation: false, + git: { safeDirectories: [root] }, + }; + await this.manager.initialize(config); + this.canonicalRoot = root; + } + + async execute( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + ): Promise { + if ( + !isWorkspaceToolRequest(request) || + request.operation !== 'execute_command' + ) { + throw new WorkspaceToolError( + 'Invalid native sandbox command', + 'INVALID_REQUEST', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + ); + } + await this.initialize(); + const root = this.canonicalRoot!; + let cwd: string; + try { + cwd = await realpath(resolve(root, request.cwd ?? '.')); + if (!isWithin(root, cwd) || !(await stat(cwd)).isDirectory()) + throw new Error('invalid cwd'); + } catch { + throw new WorkspaceToolError( + 'Command working directory is unavailable', + 'INVALID_PATH', + ); + } + const commandId = `librechat-code-${randomUUID()}`; + let wrapped: Awaited< + ReturnType + >; + try { + wrapped = await this.manager.wrapWithSandboxArgv( + request.command, + this.platform === 'win32' + ? undefined + : this.options.shellPath ?? '/bin/bash', + undefined, + signal, + cwd, + { commandId, commandText: request.command }, + ); + } catch (error) { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + ); + } + throw new WorkspaceToolError( + 'Native sandbox command could not start', + 'COMMAND_UNAVAILABLE', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + ); + } + return await this.runWrapped(request, wrapped, cwd, commandId, signal); + } + + private async runWrapped( + request: WorkspaceExecuteCommandRequest, + wrapped: { argv: string[]; env: NodeJS.ProcessEnv }, + cwd: string, + commandId: string, + signal?: AbortSignal, + ): Promise { + const outputLimit = + request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + const timeoutMs = + request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS; + return await new Promise( + (resolvePromise, reject) => { + let child: ChildProcessWithoutNullStreams; + try { + child = this.spawnCommand(wrapped.argv[0], wrapped.argv.slice(1), { + cwd, + env: wrapped.env, + detached: this.platform !== 'win32', + shell: false, + windowsHide: true, + }); + child.stdin.end(); + } catch { + reject( + new WorkspaceToolError( + 'Native sandbox command could not start', + 'COMMAND_UNAVAILABLE', + ), + ); + return; + } + let settled = false; + let timedOut = false; + let outputBytes = 0; + let truncated = false; + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + const append = (target: Buffer[], chunk: Buffer): void => { + const remaining = outputLimit - outputBytes; + if (remaining <= 0) { + truncated = true; + return; + } + const accepted = chunk.subarray(0, remaining); + target.push(accepted); + outputBytes += accepted.byteLength; + if (accepted.byteLength !== chunk.byteLength) truncated = true; + }; + child.stdout.on('data', (chunk: Buffer) => append(stdout, chunk)); + child.stderr.on('data', (chunk: Buffer) => append(stderr, chunk)); + const abort = (): void => { + if (settled) return; + this.killCommandTree(child); + }; + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + const timer = setTimeout(() => { + if (settled) return; + timedOut = true; + this.killCommandTree(child); + }, timeoutMs); + const cleanup = (): void => { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + try { + this.manager.cleanupAfterCommand(); + } catch { + // Cleanup is retried by close(); command settlement must still finish. + } + }; + child.once('error', () => { + if (settled) return; + settled = true; + const mayHaveStarted = child.pid != null; + this.killCommandTree(child); + cleanup(); + reject( + new WorkspaceToolError( + 'Native sandbox command could not start', + 'COMMAND_UNAVAILABLE', + mayHaveStarted, + ), + ); + }); + child.once('close', (code, childSignal) => { + if (settled) return; + settled = true; + this.killCommandTree(child); + cleanup(); + if (signal?.aborted) { + reject( + new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + ), + ); + return; + } + const stdoutValue = boundedUtf8(Buffer.concat(stdout), outputLimit); + const stderrBudget = Math.max( + 0, + outputLimit - Buffer.byteLength(stdoutValue), + ); + const rawStderr = Buffer.concat(stderr).toString('utf8'); + let annotatedStderr = rawStderr; + try { + annotatedStderr = this.manager.annotateStderrWithSandboxFailures( + commandId, + rawStderr, + ); + } catch { + // Preserve the bounded child error if optional violation annotation fails. + } + const stderrValue = boundedUtf8( + Buffer.from(annotatedStderr), + stderrBudget, + ); + resolvePromise({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: + timedOut || childSignal ? null : this.protocolExitCode(code), + ...(childSignal ? { signal: childSignal } : {}), + stdout: stdoutValue, + stderr: stderrValue, + truncated: + truncated || Buffer.byteLength(annotatedStderr) > stderrBudget, + timedOut, + }); + }); + }, + ); + } + + private killCommandTree(child: ChildProcessWithoutNullStreams): void { + try { + if (this.platform !== 'win32' && child.pid != null) { + process.kill(-child.pid, 'SIGKILL'); + } else { + child.kill('SIGKILL'); + } + } catch { + // The command group has already exited. + } + } + + private protocolExitCode(code: number | null): number { + return Number.isSafeInteger(code) && code != null && code >= 0 && code <= 255 + ? code + : 1; + } + + async close(): Promise { + if (!this.initialized) return; + await this.manager.reset(); + this.initialized = undefined; + this.canonicalRoot = undefined; + } +} diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 7fe28016..169bdf50 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -62,7 +62,7 @@ test('CLI trims an environment-configured worker directory', async (t) => { assert.doesNotMatch(result.stderr, /invalid workspace registration/i); }); -test('CLI refuses workspace commands without its Docker sandbox profile', async (t) => { +test('CLI supports native SRT by default and validates explicit runtime mode', async (t) => { const workspaceRoot = await mkdtemp( join(tmpdir(), 'librechat-code-command-workspace-'), ); @@ -83,11 +83,15 @@ test('CLI refuses workspace commands without its Docker sandbox profile', async LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_SANDBOX: 'runtime', }, }, ); assert.notEqual(endpoint.status, 0); - assert.match(endpoint.stderr, /require.*docker-nsjail runtime supervisor/i); + assert.match( + endpoint.stderr, + /runtime command sandbox requires.*docker-nsjail/i, + ); const noWorkspace = spawnSync( process.execPath, @@ -198,6 +202,10 @@ test('CLI advertises explicitly enabled writes without exposing the workspace ro child.kill(); await once(child, 'exit'); + assert.equal( + (body.capabilities as Record).sandboxProfile, + 'nsjail', + ); assert.deepEqual( (body.capabilities as Record).workspaceTools, { From f51fdc3fd1af56931dcd79bd1d90c0765dcae44e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 4 Sep 2026 10:33:03 -0400 Subject: [PATCH 034/116] =?UTF-8?q?=F0=9F=94=82=20feat:=20Continue=20Bound?= =?UTF-8?q?ed=20Workspace=20Listings=20(#106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(code): add native SRT command sandbox * fix(code): harden native SRT command lifecycle * fix(code): clarify native SRT containment * fix(code): close native sandbox review gaps * 🈳 feat: Create Workspace Files Only When Paths Are Vacant (#102) * 🛖 feat: Run BYOM Commands in Native Sandboxes (#101) * feat(code): add native SRT command sandbox * fix(code): harden native SRT command lifecycle * fix(code): clarify native SRT containment * fix(code): close native sandbox review gaps * feat(code): Add atomic workspace creates * fix(code): negotiate durable create-only writes * 🧺 feat: Commit Ordered Workspace Edit Batches Atomically (#103) * feat(code): Apply workspace edit batches atomically * fix(code): negotiate atomic edit batches * feat(code): Fence inspected workspace edits * fix(code): Normalize edit preview results * fix(code): negotiate preview edit fencing * fix(code): drop incompatible edit capabilities * feat(code): Continue Bounded Workspace Listings * fix(code): harden workspace list pagination * fix(code): preserve negotiated workspace defaults * fix(code): preserve legacy list result ordering --- packages/code/README.md | 26 +- packages/code/src/protocol.test.ts | 299 +++++++++++++++++- packages/code/src/protocol.ts | 328 ++++++++++++++++++- packages/code/src/worker.ts | 154 ++++++++- packages/code/src/workspace-cli.test.ts | 9 + packages/code/src/workspace-worker.test.ts | 350 ++++++++++++++++++++- packages/code/src/workspace.test.ts | 334 ++++++++++++++++++++ packages/code/src/workspace.ts | 219 +++++++++++-- service/src/bridge/router.test.ts | 5 + service/src/bridge/router.ts | 5 + service/src/bridge/store.ts | 65 +++- service/src/bridge/workspace-store.test.ts | 317 ++++++++++++++++++- 12 files changed, 2036 insertions(+), 75 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 3e8bf72d..f53741a8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -271,8 +271,19 @@ names and exposes bounded `read_file`, literal `search_text`, and deterministic can explicitly add confined `write_file` and exact-match `edit_file` operations with `--allow-workspace-writes` or `LIBRECHAT_CODE_ALLOW_WORKSPACE_WRITES=true`. -Only IDs, names, protocol version, and supported operations appear in worker -capabilities; absolute host paths remain local to the worker process. +`write_file` preserves its overwrite behavior by default; callers can set +`overwrite: false` to require an atomic create that returns `EDIT_CONFLICT` if +the target already exists. Code API dispatches that mode only after the worker +and server negotiate `create` in `writeFileModes`. +`edit_file` accepts either the legacy `oldText`/`newText` pair or an ordered +`edits` array; every exact replacement is validated before the updated file is +installed as one atomic mutation. Code API dispatches the batch form only after +the worker and server negotiate `batch` in `editFileModes`. +Revision-fenced edits likewise require the negotiated +`expected_base_sha256` entry in `editFileFeatures`. +Only IDs, names, protocol version, supported operations, and negotiated write +modes appear in worker capabilities; absolute host paths remain local to the +worker process. The protocol also defines a bounded `execute_command` request and result for a sandbox-backed executor. Commands are treated as workspace mutations and cannot @@ -320,9 +331,14 @@ bounded set of ignored-aware candidates with configuration and symlink following disabled. It then opens and verifies each candidate through the same confined 1 MiB read boundary before matching locally. File listing invokes `rg` without a shell, with configuration and symlink following disabled. Both operations -stop after bounded global result counts. The worker process still belongs inside -the trusted BYOM boundary and should receive filesystem access only to roots the -operator intentionally registers. +stop after bounded global result counts. A truncated `list_files` result includes +`nextAfterPath`; pass that value back as `afterPath` with the same workspace and +path to continue deterministically beyond the 500-file protocol ceiling. +Continuation is advertised and negotiated as the `after_path` list-file feature, +so mixed Code API and worker versions keep the legacy bounded response shape +during rolling upgrades. The worker process still belongs inside the trusted +BYOM boundary and should receive filesystem access only to roots the operator +intentionally registers. Writes are limited to 1 MiB of UTF-8 text and require an existing directory inside the registered root. They reject traversal, symlink targets, and diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 26a9928f..fd426783 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -2,11 +2,66 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { bridgeWorkerPath, + comparePortableRelativePaths, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolRequest, isWorkspaceToolResult, } from './protocol.js'; +import type { + WorkspaceEditFileRequest, + WorkspacePreviewEditRequest, +} from './protocol.js'; + +const validSingleEditRequest: WorkspaceEditFileRequest = { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', +}; +const validBatchEditRequest: WorkspaceEditFileRequest = { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'before', newText: 'after' }], +}; +// @ts-expect-error An edit request must choose a complete single or batch form. +const invalidEmptyEditRequest: WorkspaceEditFileRequest = { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', +}; +// @ts-expect-error Single and batch edit forms are mutually exclusive. +const invalidMixedEditRequest: WorkspaceEditFileRequest = { + ...validSingleEditRequest, + edits: validBatchEditRequest.edits, +}; +void invalidEmptyEditRequest; +void invalidMixedEditRequest; + +// @ts-expect-error A preview request must choose a complete single or batch form. +const invalidEmptyPreviewRequest: WorkspacePreviewEditRequest = { + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', +}; +// @ts-expect-error Single and batch preview forms are mutually exclusive. +const invalidMixedPreviewRequest: WorkspacePreviewEditRequest = { + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', + edits: [{ oldText: 'before', newText: 'after' }], +}; +void invalidEmptyPreviewRequest; +void invalidMixedPreviewRequest; test('bridgeWorkerPath encodes worker-controlled path segments', () => { assert.equal( @@ -70,6 +125,91 @@ test('bridge worker capabilities accept only bounded public workspace descriptor }; assert.equal(isValidBridgeWorkerCapabilities(valid), true); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + operations: ['read_file', 'write_file'], + writeFileModes: ['replace', 'create'], + }, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + operations: ['read_file', 'preview_edit'], + editFileModes: ['single', 'batch'], + }, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + operations: ['read_file', 'edit_file'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['expected_base_sha256'], + }, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + editFileFeatures: ['expected_base_sha256'], + }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + operations: ['read_file', 'list_files'], + listFileFeatures: ['after_path'], + }, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + listFileFeatures: ['after_path'], + }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + editFileModes: ['batch'], + }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + ...valid, + workspaceTools: { + ...valid.workspaceTools, + writeFileModes: ['create'], + }, + }), + false, + ); assert.equal( isValidBridgeWorkerCapabilities({ ...valid, @@ -99,6 +239,7 @@ test('workspace file listing accepts only bounded portable requests and results' workspaceId: 'primary', path: 'src', maxResults: 20, + afterPath: 'src/app.ts', }; assert.equal(isWorkspaceToolRequest(request), true); assert.equal( @@ -106,15 +247,65 @@ test('workspace file listing accepts only bounded portable requests and results' false, ); assert.equal(isWorkspaceToolRequest({ ...request, maxResults: 501 }), false); + assert.equal( + isWorkspaceToolRequest({ ...request, afterPath: 'outside/app.ts' }), + false, + ); const result = { protocolVersion: 1 as const, operation: 'list_files' as const, workspaceId: 'primary', - paths: ['src/app.ts', 'src/worker.ts'], - truncated: false, + paths: ['src/worker.ts', 'src/z.ts'], + truncated: true, + nextAfterPath: 'src/z.ts', }; assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal( + isWorkspaceToolResult( + { ...request, afterPath: undefined }, + { + ...result, + paths: ['src//z.ts', 'src/worker.ts'], + nextAfterPath: undefined, + }, + {}, + ), + true, + ); + assert.equal( + isWorkspaceToolResult( + { ...request, afterPath: undefined }, + { + ...result, + paths: ['src//z.ts', 'src/worker.ts'], + nextAfterPath: undefined, + }, + { listFileFeatures: ['after_path'] }, + ), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + paths: ['src/\uE000.ts', 'src/\u{10000}.ts'], + truncated: false, + nextAfterPath: undefined, + }), + true, + ); + assert.equal( + isWorkspaceToolResult(request, { ...result, nextAfterPath: 'src/worker.ts' }), + false, + ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + paths: ['src/z.ts', 'src/worker.ts'], + nextAfterPath: 'src/worker.ts', + }), + false, + ); assert.equal( isWorkspaceToolResult(request, { ...result, @@ -140,6 +331,14 @@ test('workspace file listing accepts only bounded portable requests and results' }), false, ); + assert.equal( + isWorkspaceToolResult(request, { + ...result, + paths: ['src//worker.ts'], + nextAfterPath: 'src//worker.ts', + }), + false, + ); assert.equal( isWorkspaceToolResult(request, { ...result, @@ -149,6 +348,12 @@ test('workspace file listing accepts only bounded portable requests and results' ); }); +test('workspace path ordering matches sorted depth-first traversal', () => { + assert.ok(comparePortableRelativePaths('src/app.ts', 'src.ts') < 0); + assert.ok(comparePortableRelativePaths('src/app.ts', 'src/worker.ts') < 0); + assert.ok(comparePortableRelativePaths('src.ts', 'src/app.ts') > 0); +}); + test('workspace mutations accept bounded UTF-8 requests and exact result shapes', () => { const writeRequest = { protocolVersion: 1 as const, @@ -156,8 +361,13 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' workspaceId: 'primary', path: 'notes.txt', content: 'hello', + overwrite: false, }; assert.equal(isWorkspaceToolRequest(writeRequest), true); + assert.equal( + isWorkspaceToolRequest({ ...writeRequest, overwrite: 'false' }), + false, + ); assert.equal( isWorkspaceToolRequest({ ...writeRequest, @@ -176,6 +386,17 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' }), true, ); + assert.equal( + isWorkspaceToolResult(writeRequest, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + created: false, + bytesWritten: 5, + }), + false, + ); const editRequest = { protocolVersion: 1 as const, @@ -198,6 +419,80 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' }), true, ); + const batchEditRequest = { + protocolVersion: 1 as const, + operation: 'edit_file' as const, + workspaceId: 'primary', + path: 'notes.txt', + edits: [ + { oldText: 'hello', newText: 'goodbye' }, + { oldText: 'world', newText: 'BYOM' }, + ], + }; + assert.equal(isWorkspaceToolRequest(batchEditRequest), true); + assert.equal( + isWorkspaceToolRequest({ ...batchEditRequest, oldText: 'mixed' }), + false, + ); + assert.equal(isWorkspaceToolRequest({ ...batchEditRequest, edits: [] }), false); + assert.equal( + isWorkspaceToolRequest({ + ...batchEditRequest, + edits: Array.from({ length: 101 }, () => ({ oldText: 'a', newText: 'b' })), + }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ + ...batchEditRequest, + edits: [{ oldText: 'a'.repeat(600_000), newText: 'b'.repeat(600_000) }], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(batchEditRequest, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + replacements: 2, + bytesWritten: 12, + }), + true, + ); + const previewRequest = { + ...batchEditRequest, + operation: 'preview_edit' as const, + }; + assert.equal(isWorkspaceToolRequest(previewRequest), true); + assert.equal( + isWorkspaceToolResult(previewRequest, { + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', + content: 'goodbye BYOM', + hasUtf8Bom: false, + baseSha256: 'a'.repeat(64), + replacements: 2, + bytesWritten: 12, + }), + true, + ); + assert.equal( + isWorkspaceToolRequest({ + ...editRequest, + expectedBaseSha256: 'b'.repeat(64), + }), + true, + ); + assert.equal( + isWorkspaceToolRequest({ + ...editRequest, + expectedBaseSha256: 'not-a-sha', + }), + false, + ); }); test('workspace commands require bounded sandbox inputs and outputs', () => { diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 28e99f90..3bbd83cb 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -9,6 +9,7 @@ export const BRIDGE_WORKSPACE_PATH_MAX_LENGTH = 4096; export const BRIDGE_WORKSPACE_QUERY_MAX_LENGTH = 4096; export const BRIDGE_WORKSPACE_READ_MAX_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_WRITE_MAX_BYTES = 1024 * 1024; +export const BRIDGE_WORKSPACE_EDIT_MAX_EDITS = 100; export const BRIDGE_WORKSPACE_READ_MAX_LINES = 500; export const BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS = 200; export const BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH = 2000; @@ -27,9 +28,15 @@ export type BridgeWorkspaceToolOperation = | 'search_text' | 'list_files' | 'write_file' + | 'preview_edit' | 'edit_file' | 'execute_command'; +export type WorkspaceWriteFileMode = 'replace' | 'create'; +export type WorkspaceEditFileMode = 'single' | 'batch'; +export type WorkspaceEditFileFeature = 'expected_base_sha256'; +export type WorkspaceListFileFeature = 'after_path'; + export interface BridgeWorkspaceDescriptor { id: string; name?: string; @@ -41,6 +48,14 @@ export interface BridgeWorkspaceToolCapabilities { protocolVersion: BridgeProtocolVersion; operations: BridgeWorkspaceToolOperation[]; workspaces: BridgeWorkspaceDescriptor[]; + /** Omitted by legacy workers, which only accept replacement writes. */ + writeFileModes?: WorkspaceWriteFileMode[]; + /** Omitted by legacy workers, which only accept single exact replacements. */ + editFileModes?: WorkspaceEditFileMode[]; + /** Omitted by workers that cannot fence edits against a preview revision. */ + editFileFeatures?: WorkspaceEditFileFeature[]; + /** Omitted by workers that cannot continue a bounded file listing. */ + listFileFeatures?: WorkspaceListFileFeature[]; } export interface WorkspaceReadFileRequest { @@ -94,6 +109,8 @@ export interface WorkspaceListFilesRequest { workspaceId: string; path?: string; maxResults?: number; + /** Continue strictly after this canonical path from a previous page. */ + afterPath?: string; } export interface WorkspaceListFilesResult { @@ -102,6 +119,8 @@ export interface WorkspaceListFilesResult { workspaceId: string; paths: string[]; truncated: boolean; + /** Last returned path; pass as afterPath to fetch the next page. */ + nextAfterPath?: string; } export interface WorkspaceWriteFileRequest { @@ -110,6 +129,8 @@ export interface WorkspaceWriteFileRequest { workspaceId: string; path: string; content: string; + /** False requires an atomic create and refuses to replace an existing file. */ + overwrite?: boolean; } export interface WorkspaceWriteFileResult { @@ -121,11 +142,37 @@ export interface WorkspaceWriteFileResult { bytesWritten: number; } -export interface WorkspaceEditFileRequest { +interface WorkspaceEditFileRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'edit_file'; workspaceId: string; path: string; + /** Refuses the mutation unless current file bytes match this preview revision. */ + expectedBaseSha256?: string; +} + +export interface WorkspaceSingleEditFileRequest + extends WorkspaceEditFileRequestBase { + /** Legacy single-edit form. */ + oldText: string; + /** Legacy single-edit form. */ + newText: string; + edits?: never; +} + +export interface WorkspaceBatchEditFileRequest + extends WorkspaceEditFileRequestBase { + /** Ordered exact replacements applied atomically as one file mutation. */ + edits: WorkspaceTextEdit[]; + oldText?: never; + newText?: never; +} + +export type WorkspaceEditFileRequest = + | WorkspaceSingleEditFileRequest + | WorkspaceBatchEditFileRequest; + +export interface WorkspaceTextEdit { oldText: string; newText: string; } @@ -135,7 +182,44 @@ export interface WorkspaceEditFileResult { operation: 'edit_file'; workspaceId: string; path: string; - replacements: 1; + replacements: number; + bytesWritten: number; +} + +interface WorkspacePreviewEditRequestBase { + protocolVersion: BridgeProtocolVersion; + operation: 'preview_edit'; + workspaceId: string; + path: string; +} + +export interface WorkspaceSinglePreviewEditRequest + extends WorkspacePreviewEditRequestBase { + oldText: string; + newText: string; + edits?: never; +} + +export interface WorkspaceBatchPreviewEditRequest + extends WorkspacePreviewEditRequestBase { + edits: WorkspaceTextEdit[]; + oldText?: never; + newText?: never; +} + +export type WorkspacePreviewEditRequest = + | WorkspaceSinglePreviewEditRequest + | WorkspaceBatchPreviewEditRequest; + +export interface WorkspacePreviewEditResult { + protocolVersion: BridgeProtocolVersion; + operation: 'preview_edit'; + workspaceId: string; + path: string; + content: string; + hasUtf8Bom: boolean; + baseSha256: string; + replacements: number; bytesWritten: number; } @@ -169,6 +253,7 @@ export type WorkspaceToolRequest = | WorkspaceSearchTextRequest | WorkspaceListFilesRequest | WorkspaceWriteFileRequest + | WorkspacePreviewEditRequest | WorkspaceEditFileRequest | WorkspaceExecuteCommandRequest; export type WorkspaceToolResult = @@ -176,6 +261,7 @@ export type WorkspaceToolResult = | WorkspaceSearchTextResult | WorkspaceListFilesResult | WorkspaceWriteFileResult + | WorkspacePreviewEditResult | WorkspaceEditFileResult | WorkspaceExecuteCommandResult; @@ -201,6 +287,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'workspaceId', 'path', 'maxResults', + 'afterPath', ]); const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -208,6 +295,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'workspaceId', 'path', 'content', + 'overwrite', ]); const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -216,7 +304,19 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'path', 'oldText', 'newText', + 'edits', + 'expectedBaseSha256', ]); +const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'oldText', + 'newText', + 'edits', +]); +const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', @@ -250,6 +350,7 @@ const WORKSPACE_LIST_RESULT_KEYS = new Set([ 'workspaceId', 'paths', 'truncated', + 'nextAfterPath', ]); const WORKSPACE_WRITE_RESULT_KEYS = new Set([ 'protocolVersion', @@ -267,6 +368,17 @@ const WORKSPACE_EDIT_RESULT_KEYS = new Set([ 'replacements', 'bytesWritten', ]); +const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'content', + 'hasUtf8Bom', + 'baseSha256', + 'replacements', + 'bytesWritten', +]); const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -311,6 +423,14 @@ export interface BridgeWorkerRegistrationResponse { leaseTtlMs: number; /** Operations this Code API can dispatch after the worker advertises them. */ supportedWorkspaceToolOperations?: BridgeWorkspaceToolOperation[]; + /** Write modes this Code API can safely route to a capability-aware worker. */ + supportedWorkspaceWriteFileModes?: WorkspaceWriteFileMode[]; + /** Edit modes this Code API can safely route to a capability-aware worker. */ + supportedWorkspaceEditFileModes?: WorkspaceEditFileMode[]; + /** Edit features this Code API can safely route to a capability-aware worker. */ + supportedWorkspaceEditFileFeatures?: WorkspaceEditFileFeature[]; + /** Listing features this Code API can safely route to a capability-aware worker. */ + supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; } export interface BridgePairingRedemption { @@ -476,6 +596,26 @@ function normalizePortableRelativePath(value: string): string { ); } +/** Compare path segments in ripgrep's sorted, depth-first traversal order. */ +export function comparePortableRelativePaths(left: string, right: string): number { + const encoder = new TextEncoder(); + const leftSegments = left.split('/'); + const rightSegments = right.split('/'); + const segmentCount = Math.min(leftSegments.length, rightSegments.length); + for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex += 1) { + const leftBytes = encoder.encode(leftSegments[segmentIndex]); + const rightBytes = encoder.encode(rightSegments[segmentIndex]); + const byteCount = Math.min(leftBytes.length, rightBytes.length); + for (let byteIndex = 0; byteIndex < byteCount; byteIndex += 1) { + const difference = leftBytes[byteIndex] - rightBytes[byteIndex]; + if (difference !== 0) return difference; + } + const lengthDifference = leftBytes.length - rightBytes.length; + if (lengthDifference !== 0) return lengthDifference; + } + return leftSegments.length - rightSegments.length; +} + function isWithinRequestedPath(candidate: string, requested?: string): boolean { if (requested == null) return true; const normalizedCandidate = normalizePortableRelativePath(candidate); @@ -487,6 +627,55 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } +function isValidWorkspaceEditRequest(request: Record): boolean { + const hasBatch = request.edits !== undefined; + if (hasBatch && (request.oldText !== undefined || request.newText !== undefined)) { + return false; + } + const edits = hasBatch + ? request.edits + : [{ oldText: request.oldText, newText: request.newText }]; + if ( + !Array.isArray(edits) || + edits.length < 1 || + edits.length > BRIDGE_WORKSPACE_EDIT_MAX_EDITS + ) { + return false; + } + let totalBytes = 0; + for (const edit of edits) { + if ( + typeof edit !== 'object' || + edit === null || + !hasOnlyKeys(edit as Record, WORKSPACE_TEXT_EDIT_KEYS) + ) { + return false; + } + const candidate = edit as Record; + if ( + typeof candidate.oldText !== 'string' || + candidate.oldText.length === 0 || + Buffer.from(candidate.oldText).toString('utf8') !== candidate.oldText || + typeof candidate.newText !== 'string' || + Buffer.from(candidate.newText).toString('utf8') !== candidate.newText + ) { + return false; + } + const oldBytes = new TextEncoder().encode(candidate.oldText).byteLength; + const newBytes = new TextEncoder().encode(candidate.newText).byteLength; + totalBytes += oldBytes + newBytes; + if ( + (hasBatch && totalBytes > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) || + (!hasBatch && + (oldBytes > BRIDGE_WORKSPACE_WRITE_MAX_BYTES || + newBytes > BRIDGE_WORKSPACE_WRITE_MAX_BYTES)) + ) { + return false; + } + } + return true; +} + function hasOnlyKeys( value: Record, allowed: ReadonlySet, @@ -544,6 +733,10 @@ export function isWorkspaceToolRequest( hasOnlyKeys(request, WORKSPACE_LIST_REQUEST_KEYS) && (request.path === undefined || isSafePortableRelativePath(request.path)) && + (request.afterPath === undefined || + (isSafePortableRelativePath(request.afterPath) && + normalizePortableRelativePath(request.afterPath) === request.afterPath && + isWithinRequestedPath(request.afterPath, request.path))) && (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && @@ -557,22 +750,26 @@ export function isWorkspaceToolRequest( typeof request.content === 'string' && Buffer.from(request.content).toString('utf8') === request.content && new TextEncoder().encode(request.content).byteLength <= - BRIDGE_WORKSPACE_WRITE_MAX_BYTES + BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + (request.overwrite === undefined || + typeof request.overwrite === 'boolean') + ); + } + if (request.operation === 'preview_edit') { + return ( + hasOnlyKeys(request, WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS) && + isSafePortableRelativePath(request.path) && + isValidWorkspaceEditRequest(request) ); } if (request.operation === 'edit_file') { return ( hasOnlyKeys(request, WORKSPACE_EDIT_REQUEST_KEYS) && isSafePortableRelativePath(request.path) && - typeof request.oldText === 'string' && - request.oldText.length > 0 && - Buffer.from(request.oldText).toString('utf8') === request.oldText && - new TextEncoder().encode(request.oldText).byteLength <= - BRIDGE_WORKSPACE_WRITE_MAX_BYTES && - typeof request.newText === 'string' && - Buffer.from(request.newText).toString('utf8') === request.newText && - new TextEncoder().encode(request.newText).byteLength <= - BRIDGE_WORKSPACE_WRITE_MAX_BYTES + (request.expectedBaseSha256 === undefined || + (typeof request.expectedBaseSha256 === 'string' && + /^[a-f0-9]{64}$/.test(request.expectedBaseSha256))) && + isValidWorkspaceEditRequest(request) ); } if (request.operation === 'execute_command') { @@ -603,6 +800,7 @@ export function isWorkspaceToolRequest( export function isWorkspaceToolResult( request: WorkspaceToolRequest, value: unknown, + capabilities?: Pick, ): value is WorkspaceToolResult { if (typeof value !== 'object' || value === null) return false; const result = value as Record; @@ -662,6 +860,14 @@ export function isWorkspaceToolResult( return false; } const normalizedPaths = new Set(); + const normalizedAfterPath = + request.afterPath === undefined + ? undefined + : normalizePortableRelativePath(request.afterPath); + const enforcesPaginationContract = + capabilities === undefined || + capabilities.listFileFeatures?.includes('after_path') === true; + let previousPath = normalizedAfterPath; for (const path of result.paths) { if ( !isSafePortableRelativePath(path) || @@ -670,10 +876,26 @@ export function isWorkspaceToolResult( return false; } const normalizedPath = normalizePortableRelativePath(path); - if (normalizedPaths.has(normalizedPath)) return false; + if ( + normalizedPaths.has(normalizedPath) || + (enforcesPaginationContract && + (normalizedPath !== path || + (previousPath !== undefined && + comparePortableRelativePaths(normalizedPath, previousPath) <= 0))) + ) { + return false; + } normalizedPaths.add(normalizedPath); + previousPath = normalizedPath; } - return true; + if (!enforcesPaginationContract) { + return result.nextAfterPath === undefined; + } + if (result.truncated !== true) return result.nextAfterPath === undefined; + return ( + result.paths.length > 0 && + result.nextAfterPath === result.paths[result.paths.length - 1] + ); } if (request.operation === 'write_file') { @@ -681,6 +903,7 @@ export function isWorkspaceToolResult( hasOnlyKeys(result, WORKSPACE_WRITE_RESULT_KEYS) && result.path === request.path && typeof result.created === 'boolean' && + (request.overwrite !== false || result.created === true) && Number.isSafeInteger(result.bytesWritten) && Number(result.bytesWritten) === new TextEncoder().encode(request.content).byteLength @@ -688,16 +911,37 @@ export function isWorkspaceToolResult( } if (request.operation === 'edit_file') { + const replacements = request.edits?.length ?? 1; return ( hasOnlyKeys(result, WORKSPACE_EDIT_RESULT_KEYS) && result.path === request.path && - result.replacements === 1 && + result.replacements === replacements && Number.isSafeInteger(result.bytesWritten) && Number(result.bytesWritten) >= 0 && Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES ); } + if (request.operation === 'preview_edit') { + const replacements = request.edits?.length ?? 1; + const content = typeof result.content === 'string' ? result.content : null; + return ( + hasOnlyKeys(result, WORKSPACE_PREVIEW_EDIT_RESULT_KEYS) && + result.path === request.path && + content !== null && + Buffer.from(content).toString('utf8') === content && + typeof result.hasUtf8Bom === 'boolean' && + typeof result.baseSha256 === 'string' && + /^[a-f0-9]{64}$/.test(result.baseSha256) && + result.replacements === replacements && + Number.isSafeInteger(result.bytesWritten) && + Number(result.bytesWritten) === + new TextEncoder().encode(content).byteLength + + (result.hasUtf8Bom ? 3 : 0) && + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + ); + } + if (request.operation === 'execute_command') { const stdout = typeof result.stdout === 'string' ? result.stdout : null; const stderr = typeof result.stderr === 'string' ? result.stderr : null; @@ -761,13 +1005,14 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !Array.isArray(capabilities.operations) || capabilities.operations.length < 1 || - capabilities.operations.length > 6 || + capabilities.operations.length > 7 || !capabilities.operations.every( (operation) => operation === 'read_file' || operation === 'search_text' || operation === 'list_files' || operation === 'write_file' || + operation === 'preview_edit' || operation === 'edit_file' || operation === 'execute_command', ) || @@ -779,6 +1024,57 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.writeFileModes !== undefined && + (!Array.isArray(capabilities.writeFileModes) || + capabilities.writeFileModes.length < 1 || + capabilities.writeFileModes.length > 2 || + !capabilities.operations.includes('write_file') || + !capabilities.writeFileModes.every( + (mode) => mode === 'replace' || mode === 'create', + ) || + new Set(capabilities.writeFileModes).size !== + capabilities.writeFileModes.length) + ) { + return false; + } + + if ( + capabilities.editFileModes !== undefined && + (!Array.isArray(capabilities.editFileModes) || + capabilities.editFileModes.length < 1 || + capabilities.editFileModes.length > 2 || + (!capabilities.operations.includes('edit_file') && + !capabilities.operations.includes('preview_edit')) || + !capabilities.editFileModes.every( + (mode) => mode === 'single' || mode === 'batch', + ) || + new Set(capabilities.editFileModes).size !== + capabilities.editFileModes.length) + ) { + return false; + } + + if ( + capabilities.editFileFeatures !== undefined && + (!Array.isArray(capabilities.editFileFeatures) || + capabilities.editFileFeatures.length !== 1 || + !capabilities.operations.includes('edit_file') || + capabilities.editFileFeatures[0] !== 'expected_base_sha256') + ) { + return false; + } + + if ( + capabilities.listFileFeatures !== undefined && + (!Array.isArray(capabilities.listFileFeatures) || + capabilities.listFileFeatures.length !== 1 || + !capabilities.operations.includes('list_files') || + capabilities.listFileFeatures[0] !== 'after_path') + ) { + return false; + } + const workspaceIds = new Set(); return capabilities.workspaces.every((workspace) => { if (typeof workspace !== 'object' || workspace === null) return false; diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 0b6f5c13..657ba726 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -19,6 +19,7 @@ import type { BridgeWorkerCapabilities, BridgeWorkerCredentialResponse, BridgeWorkerRegistrationResponse, + BridgeWorkspaceToolOperation, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -137,6 +138,24 @@ function workspaceCapabilitiesMatch( advertised.operations.every( (operation, index) => operation === executor.operations[index], ) && + advertised.writeFileModes?.length === executor.writeFileModes?.length && + (advertised.writeFileModes?.every( + (mode, index) => mode === executor.writeFileModes?.[index], + ) ?? executor.writeFileModes == null) && + advertised.editFileModes?.length === executor.editFileModes?.length && + (advertised.editFileModes?.every( + (mode, index) => mode === executor.editFileModes?.[index], + ) ?? executor.editFileModes == null) && + advertised.editFileFeatures?.length === + executor.editFileFeatures?.length && + (advertised.editFileFeatures?.every( + (feature, index) => feature === executor.editFileFeatures?.[index], + ) ?? executor.editFileFeatures == null) && + advertised.listFileFeatures?.length === + executor.listFileFeatures?.length && + (advertised.listFileFeatures?.every( + (feature, index) => feature === executor.listFileFeatures?.[index], + ) ?? executor.listFileFeatures == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -191,10 +210,17 @@ function registrationCompatibleCapabilities( const { workspaceTools: _workspaceTools, ...compatible } = capabilities; return compatible; } + const { + writeFileModes: _writeFileModes, + editFileModes: _editFileModes, + editFileFeatures: _editFileFeatures, + listFileFeatures: _listFileFeatures, + ...compatibleWorkspaceTools + } = workspaceTools; return { ...capabilities, workspaceTools: { - ...workspaceTools, + ...compatibleWorkspaceTools, operations, workspaces, }, @@ -208,10 +234,50 @@ function supportedWorkspaceCapabilities( const desired = capabilities.workspaceTools; const supported = registration.supportedWorkspaceToolOperations; if (desired == null || !Array.isArray(supported)) return undefined; - const operations = desired.operations.filter((operation) => + let operations = desired.operations.filter((operation) => supported.includes(operation), ); if (operations.length === 0) return undefined; + let writeFileModes: typeof desired.writeFileModes; + if (operations.includes('write_file')) { + const desiredModes = desired.writeFileModes ?? ['replace']; + const serverModes = registration.supportedWorkspaceWriteFileModes ?? [ + 'replace', + ]; + const commonModes = desiredModes.filter((mode) => + serverModes.includes(mode), + ); + if (commonModes.length === 0) { + operations = operations.filter((operation) => operation !== 'write_file'); + } else if (registration.supportedWorkspaceWriteFileModes != null) { + writeFileModes = commonModes; + } + } + const editOperations = new Set([ + 'preview_edit', + 'edit_file', + ]); + let editFileModes: typeof desired.editFileModes; + if (operations.some((operation) => editOperations.has(operation))) { + const desiredModes = desired.editFileModes ?? ['single']; + const serverModes = registration.supportedWorkspaceEditFileModes ?? [ + 'single', + ]; + const commonModes = desiredModes.filter((mode) => + serverModes.includes(mode), + ); + if (commonModes.length === 0) { + operations = operations.filter( + (operation) => !editOperations.has(operation), + ); + } else if (registration.supportedWorkspaceEditFileModes != null) { + editFileModes = commonModes; + } + } + if (operations.length === 0) return undefined; + const supportsEditRequests = operations.some((operation) => + editOperations.has(operation), + ); const workspaces = desired.workspaces.flatMap((workspace) => { if (workspace.operations == null) return [workspace]; const workspaceOperations = workspace.operations.filter((operation) => @@ -222,12 +288,37 @@ function supportedWorkspaceCapabilities( : [{ ...workspace, operations: workspaceOperations }]; }); if (workspaces.length === 0) return undefined; + const editFileFeatures = desired.editFileFeatures?.filter((feature) => + registration.supportedWorkspaceEditFileFeatures?.includes(feature), + ); + const listFileFeatures = desired.listFileFeatures?.filter((feature) => + registration.supportedWorkspaceListFileFeatures?.includes(feature), + ); + const { + writeFileModes: _writeFileModes, + editFileModes: _editFileModes, + editFileFeatures: _editFileFeatures, + listFileFeatures: _listFileFeatures, + ...compatibleDesired + } = desired; return { ...capabilities, workspaceTools: { - ...desired, + ...compatibleDesired, operations, workspaces, + ...(operations.includes('write_file') && writeFileModes?.length + ? { writeFileModes } + : {}), + ...(supportsEditRequests && editFileModes?.length + ? { editFileModes } + : {}), + ...(operations.includes('edit_file') && editFileFeatures?.length + ? { editFileFeatures } + : {}), + ...(operations.includes('list_files') && listFileFeatures?.length + ? { listFileFeatures } + : {}), }, }; } @@ -875,6 +966,52 @@ export class BridgeWorker { 'Workspace tool operation is not advertised for workspace', ); } + if (workspaceRequest.operation === 'write_file') { + const mode = + workspaceRequest.overwrite === false ? 'create' : 'replace'; + const modes = advertised.writeFileModes; + if ( + (workspaceRequest.overwrite !== undefined && modes == null) || + (modes != null && !modes.includes(mode)) + ) { + throw new BridgeProtocolError( + 'Workspace write mode is not advertised', + ); + } + } + if ( + workspaceRequest.operation === 'preview_edit' || + workspaceRequest.operation === 'edit_file' + ) { + const mode = workspaceRequest.edits === undefined ? 'single' : 'batch'; + const modes = advertised.editFileModes; + if ( + (modes == null && mode !== 'single') || + (modes != null && !modes.includes(mode)) + ) { + throw new BridgeProtocolError( + 'Workspace edit mode is not advertised', + ); + } + if ( + workspaceRequest.operation === 'edit_file' && + workspaceRequest.expectedBaseSha256 !== undefined && + !advertised.editFileFeatures?.includes('expected_base_sha256') + ) { + throw new BridgeProtocolError( + 'Workspace edit feature is not advertised', + ); + } + } + if ( + workspaceRequest.operation === 'list_files' && + workspaceRequest.afterPath !== undefined && + !advertised.listFileFeatures?.includes('after_path') + ) { + throw new BridgeProtocolError( + 'Workspace listing feature is not advertised', + ); + } const isMutation = workspaceRequest.operation === 'write_file' || workspaceRequest.operation === 'edit_file' || @@ -898,6 +1035,17 @@ export class BridgeWorker { workspaceRequest, executionController.signal, ); + if ( + workspaceRequest.operation === 'list_files' && + !advertised.listFileFeatures?.includes('after_path') && + 'nextAfterPath' in payload + ) { + const { + nextAfterPath: _nextAfterPath, + ...compatiblePayload + } = payload; + payload = compatiblePayload; + } workspaceMutationApplied = isMutation; if (isMutation && !isWorkspaceToolResult(workspaceRequest, payload)) { throw new BridgeProtocolError( diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 169bdf50..c1bc7fce 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -151,8 +151,12 @@ test('CLI advertises explicitly enabled writes without exposing the workspace ro 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', ], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], }), ); return; @@ -215,8 +219,12 @@ test('CLI advertises explicitly enabled writes without exposing the workspace ro 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', ], + writeFileModes: ['replace', 'create'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['expected_base_sha256'], workspaces: [ { id: 'root-workspace', @@ -226,6 +234,7 @@ test('CLI advertises explicitly enabled writes without exposing the workspace ro 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', ], }, diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index c98182b0..43c820fe 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -15,9 +15,13 @@ const listWorkspaceCapabilities = { 'list_files' as const, ], workspaces: [{ id: 'primary' }], + listFileFeatures: ['after_path' as const], }; -function registrationResponse(supportsList: boolean): Response { +function registrationResponse( + supportsList: boolean, + supportsPagination = false, +): Response { return Response.json({ protocolVersion: 1, workerId: 'vm-1', @@ -31,6 +35,9 @@ function registrationResponse(supportsList: boolean): Response { 'search_text', 'list_files', ], + ...(supportsPagination + ? { supportedWorkspaceListFileFeatures: ['after_path'] } + : {}), } : {}), }); @@ -131,6 +138,67 @@ test('worker re-registers list_files after the Code API advertises support', asy ]); }); +test('worker omits pagination fields until Code API negotiates them', async () => { + let settlement: Record | undefined; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: listWorkspaceCapabilities, + }, + workspaceTools: { + capabilities: listWorkspaceCapabilities, + async execute() { + return { + protocolVersion: 1 as const, + operation: 'list_files' as const, + workspaceId: 'primary', + paths: ['first.txt'], + truncated: true, + nextAfterPath: 'first.txt', + }; + }, + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/register')) return registrationResponse(true); + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.register(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-list-legacy-consumer', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 1, + }, + }); + + assert.deepEqual(settlement?.result, { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['first.txt'], + truncated: true, + }); +}); + test('worker omits restricted workspaces that legacy registration would widen', async () => { const registrations: Array<{ operations: string[]; @@ -220,6 +288,7 @@ test('worker promotes only operations understood by an older Code API', async () const registrations: Array<{ operations: string[]; workspaces: Array>; + writeFileModes?: string[]; }> = []; const workspaceCapabilities = { protocolVersion: 1 as const, @@ -230,6 +299,7 @@ test('worker promotes only operations understood by an older Code API', async () 'write_file' as const, 'edit_file' as const, ], + writeFileModes: ['replace' as const, 'create' as const], workspaces: [ { id: 'primary', @@ -268,6 +338,7 @@ test('worker promotes only operations understood by an older Code API', async () workspaceTools: { operations: string[]; workspaces: Array>; + writeFileModes?: string[]; }; }; }; @@ -312,6 +383,7 @@ test('worker retains per-workspace restrictions during partial mutation promotio const registrations: Array<{ operations: string[]; workspaces: Array>; + writeFileModes?: string[]; }> = []; const workspaceCapabilities = { protocolVersion: 1 as const, @@ -322,6 +394,7 @@ test('worker retains per-workspace restrictions during partial mutation promotio 'write_file' as const, 'edit_file' as const, ], + writeFileModes: ['replace' as const, 'create' as const], workspaces: [ { id: 'readonly', @@ -368,6 +441,7 @@ test('worker retains per-workspace restrictions during partial mutation promotio workspaceTools: { operations: string[]; workspaces: Array>; + writeFileModes?: string[]; }; }; }; @@ -384,6 +458,7 @@ test('worker retains per-workspace restrictions during partial mutation promotio 'list_files', 'write_file', ], + supportedWorkspaceWriteFileModes: ['replace', 'create'], }); }, }); @@ -393,6 +468,7 @@ test('worker retains per-workspace restrictions during partial mutation promotio assert.deepEqual(registrations[1], { protocolVersion: 1, operations: ['read_file', 'search_text', 'list_files', 'write_file'], + writeFileModes: ['replace', 'create'], workspaces: [ { id: 'readonly', @@ -411,6 +487,220 @@ test('worker retains per-workspace restrictions during partial mutation promotio }); }); +test('worker omits write modes not negotiated by an older Code API', async () => { + const registrations: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'write_file' as const], + writeFileModes: ['replace' as const, 'create' as const], + workspaces: [ + { + id: 'primary', + operations: ['read_file' as const, 'write_file' as const], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { workspaceTools?: Record }; + }; + registrations.push(body.capabilities.workspaceTools ?? {}); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'write_file'], + }); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + { + protocolVersion: 1, + operations: ['read_file', 'write_file'], + workspaces: [ + { id: 'primary', operations: ['read_file', 'write_file'] }, + ], + }, + ]); +}); + +test('worker advertises only edit modes and features negotiated by Code API', async () => { + const registrations: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'edit_file' as const], + editFileModes: ['single' as const, 'batch' as const], + editFileFeatures: ['expected_base_sha256' as const], + workspaces: [ + { + id: 'primary', + operations: ['read_file' as const, 'edit_file' as const], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { workspaceTools?: Record }; + }; + registrations.push(body.capabilities.workspaceTools ?? {}); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'edit_file'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + }); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + { + protocolVersion: 1, + operations: ['read_file', 'edit_file'], + editFileModes: ['single', 'batch'], + workspaces: [ + { id: 'primary', operations: ['read_file', 'edit_file'] }, + ], + }, + ]); +}); + +test('worker drops file operations when no request mode is compatible', async () => { + const registrations: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: [ + 'read_file' as const, + 'write_file' as const, + 'preview_edit' as const, + ], + writeFileModes: ['create' as const], + editFileModes: ['batch' as const], + workspaces: [ + { + id: 'primary', + operations: [ + 'read_file' as const, + 'write_file' as const, + 'preview_edit' as const, + ], + }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new Error('not executed'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { + capabilities: { workspaceTools?: Record }; + }; + registrations.push(body.capabilities.workspaceTools ?? {}); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'write_file', + 'preview_edit', + ], + supportedWorkspaceWriteFileModes: ['replace'], + supportedWorkspaceEditFileModes: ['single'], + }); + }, + }); + + await worker.register(); + + assert.deepEqual(registrations, [ + { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary', operations: ['read_file'] }], + }, + ]); +}); + test('worker retains per-workspace restrictions during read-only promotion', async () => { const registrations: Array<{ operations: string[]; @@ -2126,3 +2416,61 @@ test('worker rejects workspace operations outside its advertised capability', as assert.equal(settlement?.status, 'rejected'); assert.match(String(settlement?.error), /operation is not advertised/i); }); + +test('worker rejects legacy replacement writes outside its advertised mode', async () => { + let executions = 0; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + writeFileModes: ['create' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + executions += 1; + throw new Error('must not execute'); + }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-workspace-replace-mode', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'blocked', + }, + }); + + assert.equal(executions, 0); + assert.equal(settlement?.status, 'rejected'); + assert.match(String(settlement?.error), /write mode is not advertised/i); +}); diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index e0cacb48..25894773 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -201,6 +201,23 @@ test('lists workspace files deterministically with a hard result bound', async ( workspaceId: 'primary', paths: ['docs/guide.md', 'src/app.ts'], truncated: true, + nextAfterPath: 'src/app.ts', + }, + ); + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 2, + afterPath: 'src/app.ts', + }), + { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['src/worker.ts'], + truncated: false, }, ); assert.deepEqual( @@ -221,6 +238,96 @@ test('lists workspace files deterministically with a hard result bound', async ( ); }); +test('continues a workspace listing beyond the protocol result ceiling', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await Promise.all( + Array.from({ length: 501 }, (_, index) => + writeFile(join(root, `file-${String(index).padStart(3, '0')}.txt`), 'x'), + ), + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const firstPage = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 500, + }); + assert.equal(firstPage.operation, 'list_files'); + if (firstPage.operation !== 'list_files') assert.fail('expected list result'); + assert.equal(firstPage.paths.length, 500); + assert.equal(firstPage.truncated, true); + assert.equal(firstPage.nextAfterPath, 'file-499.txt'); + + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 500, + afterPath: firstPage.nextAfterPath, + }), + { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['file-500.txt'], + truncated: false, + }, + ); +}); + +test( + 'continues listings across directory and file prefix siblings', + async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + await writeFile(join(root, 'src', 'app.ts'), 'nested'); + await writeFile(join(root, 'src.ts'), 'sibling'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + + const firstPage = await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 1, + }); + assert.deepEqual(firstPage, { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['src/app.ts'], + truncated: true, + nextAfterPath: 'src/app.ts', + }); + if (firstPage.operation !== 'list_files') { + assert.fail('expected list result'); + } + assert.deepEqual( + await tools.execute({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 1, + afterPath: firstPage.nextAfterPath, + }), + { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + paths: ['src.ts'], + truncated: false, + }, + ); + }, +); + test('rejects listing through a directory symlink that leaves the workspace', async (t) => { const parent = await mkdtemp( join(tmpdir(), 'librechat-code-workspace-parent-'), @@ -767,6 +874,7 @@ test('advertises workspace IDs and names without exposing host roots', async (t) protocolVersion: 1, operations: ['read_file', 'search_text', 'list_files'], workspaces: [{ id: 'primary', name: 'LibreChat' }], + listFileFeatures: ['after_path'], }); assert.equal(JSON.stringify(tools.capabilities).includes(root), false); }); @@ -805,6 +913,7 @@ test('writable workspaces create, replace, and exactly edit files', async (t) => 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', ], workspaces: [ @@ -816,10 +925,15 @@ test('writable workspaces create, replace, and exactly edit files', async (t) => 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', ], }, ], + writeFileModes: ['replace', 'create'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['expected_base_sha256'], + listFileFeatures: ['after_path'], }); await tools.execute({ protocolVersion: 1, @@ -847,6 +961,183 @@ test('writable workspaces create, replace, and exactly edit files', async (t) => assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'hello BYOM'); }); +test('workspace writes can require an atomic create without replacement', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'existing.txt'), 'preserve me'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'existing.txt', + content: 'replace me', + overwrite: false, + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT', + ); + assert.equal(await readFile(join(root, 'existing.txt'), 'utf8'), 'preserve me'); + + const created = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'created.txt', + content: 'new file', + overwrite: false, + }); + assert.deepEqual(created, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'created.txt', + created: true, + bytesWritten: 8, + }); + assert.equal(await readFile(join(root, 'created.txt'), 'utf8'), 'new file'); + + const competingWrites = await Promise.allSettled([ + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'raced.txt', + content: 'first', + overwrite: false, + }), + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'raced.txt', + content: 'second', + overwrite: false, + }), + ]); + assert.equal( + competingWrites.filter((result) => result.status === 'fulfilled').length, + 1, + ); + const rejected = competingWrites.find( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ); + assert.ok(rejected?.reason instanceof WorkspaceToolError); + assert.equal(rejected.reason.code, 'EDIT_CONFLICT'); + assert.match(await readFile(join(root, 'raced.txt'), 'utf8'), /^(first|second)$/); +}); + +test('workspace batch edits commit all replacements atomically', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'batch.txt'), 'alpha beta gamma'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'batch.txt', + edits: [ + { oldText: 'alpha', newText: 'one' }, + { oldText: 'gamma', newText: 'three' }, + ], + }); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'batch.txt', + replacements: 2, + bytesWritten: 14, + }); + assert.equal(await readFile(join(root, 'batch.txt'), 'utf8'), 'one beta three'); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'batch.txt', + edits: [ + { oldText: 'one', newText: 'partial' }, + { oldText: 'missing', newText: 'never' }, + ], + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT', + ); + assert.equal(await readFile(join(root, 'batch.txt'), 'utf8'), 'one beta three'); +}); + +test('workspace edit previews are non-mutating and fence the commit revision', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'preview.txt'), 'prefix SEC suffix'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const preview = await tools.execute({ + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'preview.txt', + oldText: ' suffix', + newText: 'RET suffix', + }); + assert.equal(preview.operation, 'preview_edit'); + assert.equal(preview.content, 'prefix SECRET suffix'); + assert.equal(preview.hasUtf8Bom, false); + assert.match(preview.baseSha256, /^[a-f0-9]{64}$/); + assert.equal(await readFile(join(root, 'preview.txt'), 'utf8'), 'prefix SEC suffix'); + + await writeFile(join(root, 'preview.txt'), 'changed SEC suffix'); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'preview.txt', + oldText: ' suffix', + newText: 'RET suffix', + expectedBaseSha256: preview.baseSha256, + }), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT', + ); + assert.equal(await readFile(join(root, 'preview.txt'), 'utf8'), 'changed SEC suffix'); +}); + +test('workspace edit previews strip a UTF-8 BOM while retaining its byte count', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'bom.txt'), Buffer.from('\ufeffbefore', 'utf8')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const preview = await tools.execute({ + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'bom.txt', + oldText: 'before', + newText: 'after', + }); + assert.equal(preview.operation, 'preview_edit'); + assert.equal(preview.content, 'after'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(preview.bytesWritten, 8); + assert.equal(await readFile(join(root, 'bom.txt'), 'utf8'), '\ufeffbefore'); +}); + test('workspace mutations sync the containing directory after replacement', async (t) => { if (process.platform === 'win32') { t.skip('Directory fsync is unavailable on Windows'); @@ -889,6 +1180,42 @@ test('workspace mutations sync the containing directory after replacement', asyn assert.equal(syncCalls, 5); }); +test('atomic creates remove staging before syncing the directory', async (t) => { + if (process.platform === 'win32') { + t.skip('Directory fsync is unavailable on Windows'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const probe = await open(root, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + sync(): Promise; + }; + await probe.close(); + const originalSync = fileHandlePrototype.sync; + let syncCalls = 0; + t.mock.method(fileHandlePrototype, 'sync', async function (this: FileHandle) { + syncCalls += 1; + if (syncCalls === 2) { + assert.deepEqual(await readdir(root), ['created.txt']); + } + await originalSync.call(this); + }); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'created.txt', + content: 'durable create', + overwrite: false, + }); + assert.equal(syncCalls, 2); +}); + test('workspace mutations report uncertain commit when directory sync fails', async (t) => { if (process.platform === 'win32') { t.skip('Directory fsync is unavailable on Windows'); @@ -1588,9 +1915,16 @@ test('composes sandboxed commands without exposing them on unconfigured workspac 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', 'execute_command', ]); + assert.deepEqual(tools.capabilities.writeFileModes, ['replace', 'create']); + assert.deepEqual(tools.capabilities.editFileModes, ['single', 'batch']); + assert.deepEqual(tools.capabilities.editFileFeatures, [ + 'expected_base_sha256', + ]); + assert.deepEqual(tools.capabilities.listFileFeatures, ['after_path']); assert.deepEqual( tools.capabilities.workspaces.find(({ id }) => id === 'sandboxed')?.operations, tools.capabilities.operations, diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 3f8ed7b6..727955cd 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,7 +1,7 @@ import { spawn } from 'node:child_process'; -import { randomBytes } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { lstat, open, realpath, rename, stat, unlink } from 'node:fs/promises'; +import { link, lstat, open, realpath, rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -14,6 +14,7 @@ import { BRIDGE_WORKSPACE_LIST_MAX_RESULTS, BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS, BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, + comparePortableRelativePaths, isSafePortableRelativePath, isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, @@ -27,6 +28,8 @@ import type { WorkspaceReadFileResult, WorkspaceEditFileRequest, WorkspaceEditFileResult, + WorkspacePreviewEditRequest, + WorkspacePreviewEditResult, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, WorkspaceListFilesRequest, @@ -47,6 +50,8 @@ export type { WorkspaceReadFileResult, WorkspaceEditFileRequest, WorkspaceEditFileResult, + WorkspacePreviewEditRequest, + WorkspacePreviewEditResult, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, WorkspaceListFilesRequest, @@ -119,7 +124,7 @@ const READ_OPERATIONS = [ 'search_text', 'list_files', ] as const; -const WRITE_OPERATIONS = ['write_file', 'edit_file'] as const; +const WRITE_OPERATIONS = ['write_file', 'preview_edit', 'edit_file'] as const; function isUtf8ScalarString(value: string): boolean { return Buffer.from(value).toString('utf8') === value; @@ -455,6 +460,7 @@ async function atomicWriteConfinedFile( content: Buffer, signal?: AbortSignal, expected?: { dev: bigint | number; ino: bigint | number; content: Buffer }, + allowOverwrite = true, ): Promise<{ created: boolean }> { throwIfAborted(signal); if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { @@ -488,6 +494,12 @@ async function atomicWriteConfinedFile( if (existing?.isSymbolicLink() || (existing != null && !existing.isFile())) { throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } + if (!allowOverwrite && existing != null) { + throw new WorkspaceToolError( + 'Workspace file already exists', + 'EDIT_CONFLICT', + ); + } if ( expected != null && (existing == null || @@ -506,6 +518,7 @@ async function atomicWriteConfinedFile( ); const installTarget = resolve(canonicalParent, basename(candidate)); let handle: FileHandle | undefined; + let temporaryNeedsCleanup = true; let staged: { dev: bigint | number; ino: bigint | number; content: Buffer }; try { handle = await open( @@ -588,10 +601,36 @@ async function atomicWriteConfinedFile( staged, signal, ); + temporaryNeedsCleanup = false; return { created: false }; } throwIfAborted(signal); - await rename(temporary, installTarget); + if (allowOverwrite) { + await rename(temporary, installTarget); + temporaryNeedsCleanup = false; + } else { + try { + await link(temporary, installTarget); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new WorkspaceToolError( + 'Workspace file already exists', + 'EDIT_CONFLICT', + ); + } + throw error; + } + try { + await unlink(temporary); + temporaryNeedsCleanup = false; + } catch { + throw new WorkspaceToolError( + 'Workspace create cleanup could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + } + } await confirmInstalledMutation(root, installTarget, staged); return { created: existing == null }; } catch (error) { @@ -602,7 +641,9 @@ async function atomicWriteConfinedFile( ); } finally { await handle?.close().catch(() => undefined); - await unlink(temporary).catch(() => undefined); + if (temporaryNeedsCleanup) { + await unlink(temporary).catch(() => undefined); + } } } @@ -617,6 +658,8 @@ async function writeWorkspaceFile( request.path, content, signal, + undefined, + request.overwrite !== false, ); return { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -660,34 +703,17 @@ async function editWorkspaceFile( ); } const original = await readBoundedEditFile(opened); - const hasBom = - original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf; - const body = hasBom ? original.subarray(3) : original; - const text = body.toString('utf8'); - if (!Buffer.from(text, 'utf8').equals(body)) { - throw new WorkspaceToolError( - 'Workspace file is not UTF-8 text', - 'INVALID_REQUEST', - ); - } - const first = text.indexOf(request.oldText); if ( - first < 0 || - text.indexOf(request.oldText, first + 1) >= 0 + request.expectedBaseSha256 !== undefined && + createHash('sha256').update(original).digest('hex') !== + request.expectedBaseSha256 ) { throw new WorkspaceToolError( - 'Workspace edit must match exactly once', + 'Workspace file changed after edit preview', 'EDIT_CONFLICT', ); } - const updatedText = - text.slice(0, first) + - request.newText + - text.slice(first + request.oldText.length); - const updatedBody = Buffer.from(updatedText, 'utf8'); - const updated = hasBom - ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), updatedBody]) - : updatedBody; + const { updated, replacements } = applyWorkspaceEdits(original, request); await atomicWriteConfinedFile( root, request.path, @@ -704,7 +730,7 @@ async function editWorkspaceFile( operation: 'edit_file', workspaceId: request.workspaceId, path: request.path, - replacements: 1, + replacements, bytesWritten: updated.byteLength, }; } catch (error) { @@ -715,6 +741,84 @@ async function editWorkspaceFile( } } +function applyWorkspaceEdits( + original: Buffer, + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, +): { updated: Buffer; replacements: number } { + const hasBom = + original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf; + const body = hasBom ? original.subarray(3) : original; + const text = body.toString('utf8'); + if (!Buffer.from(text, 'utf8').equals(body)) { + throw new WorkspaceToolError( + 'Workspace file is not UTF-8 text', + 'INVALID_REQUEST', + ); + } + const edits = request.edits ?? [ + { oldText: request.oldText ?? '', newText: request.newText ?? '' }, + ]; + let updatedText = text; + for (const edit of edits) { + const first = updatedText.indexOf(edit.oldText); + if (first < 0 || updatedText.indexOf(edit.oldText, first + 1) >= 0) { + throw new WorkspaceToolError( + 'Workspace edit must match exactly once', + 'EDIT_CONFLICT', + ); + } + updatedText = + updatedText.slice(0, first) + + edit.newText + + updatedText.slice(first + edit.oldText.length); + } + const updatedBody = Buffer.from(updatedText, 'utf8'); + const updated = hasBom + ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), updatedBody]) + : updatedBody; + if (updated.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + return { updated, replacements: edits.length }; +} + +async function previewWorkspaceEdit( + root: string, + request: WorkspacePreviewEditRequest, + signal?: AbortSignal, +): Promise { + const original = await readConfinedFileBuffer(root, request.path); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + const { updated, replacements } = applyWorkspaceEdits(original, request); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + const hasUtf8Bom = + updated[0] === 0xef && updated[1] === 0xbb && updated[2] === 0xbf; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit', + workspaceId: request.workspaceId, + path: request.path, + content: decodeWorkspaceText(updated), + hasUtf8Bom, + baseSha256: createHash('sha256').update(original).digest('hex'), + replacements, + bytesWritten: updated.byteLength, + }; +} + interface SearchCandidates { paths: string[]; truncated: boolean; @@ -1049,6 +1153,7 @@ async function listWorkspaceFiles( .filter((segment) => segment.length > 0 && segment !== '.') .join('/'); const requestedResultPath = normalizedRequestedResultPath || undefined; + const afterPath = request.afterPath; const candidates: Array<{ filesystemPath: string; resultPath: string }> = []; let truncated = false; @@ -1127,6 +1232,12 @@ async function listWorkspaceFiles( ? `${requestedResultPath}${normalizedPath.slice(portableCanonicalListPath.length)}` : normalizedPath; if (!isSafePortableRelativePath(resultPath)) return; + if ( + afterPath !== undefined && + comparePortableRelativePaths(resultPath, afterPath) <= 0 + ) { + return; + } candidates.push({ filesystemPath: normalizedPath, resultPath }); }; @@ -1234,6 +1345,9 @@ async function listWorkspaceFiles( workspaceId: request.workspaceId, paths, truncated, + ...(truncated && paths.length > 0 + ? { nextAfterPath: paths[paths.length - 1] } + : {}), }; } @@ -1298,11 +1412,19 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { private readonly roots: ReadonlyMap, operations: BridgeWorkspaceToolCapabilities['operations'], workspaces: BridgeWorkspaceDescriptor[], + writeFileModes?: BridgeWorkspaceToolCapabilities['writeFileModes'], + editFileModes?: BridgeWorkspaceToolCapabilities['editFileModes'], + editFileFeatures?: BridgeWorkspaceToolCapabilities['editFileFeatures'], + listFileFeatures?: BridgeWorkspaceToolCapabilities['listFileFeatures'], ) { this.capabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, operations, workspaces, + ...(writeFileModes != null ? { writeFileModes } : {}), + ...(editFileModes != null ? { editFileModes } : {}), + ...(editFileFeatures != null ? { editFileFeatures } : {}), + ...(listFileFeatures != null ? { listFileFeatures } : {}), }; } @@ -1335,6 +1457,12 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { protocolVersion: BRIDGE_PROTOCOL_VERSION, operations, workspaces, + ...(anyWritable ? { writeFileModes: ['replace', 'create'] } : {}), + ...(anyWritable ? { editFileModes: ['single', 'batch'] } : {}), + ...(anyWritable + ? { editFileFeatures: ['expected_base_sha256'] } + : {}), + listFileFeatures: ['after_path'], }; if (!isValidBridgeWorkspaceToolCapabilities(capabilities)) { throw new WorkspaceToolError( @@ -1358,7 +1486,15 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { writable: workspace.writable === true, }); } - return new LocalWorkspaceTools(roots, operations, workspaces); + return new LocalWorkspaceTools( + roots, + operations, + workspaces, + capabilities.writeFileModes, + capabilities.editFileModes, + capabilities.editFileFeatures, + capabilities.listFileFeatures, + ); } async execute( @@ -1383,7 +1519,11 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { } const { root } = workspace; - if (request.operation === 'write_file' || request.operation === 'edit_file') { + if ( + request.operation === 'write_file' || + request.operation === 'preview_edit' || + request.operation === 'edit_file' + ) { if (!workspace.writable) { throw new WorkspaceToolError( 'Workspace mutations are disabled by the worker', @@ -1396,8 +1536,11 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { 'EXECUTION_ABORTED', ); } - return request.operation === 'write_file' - ? writeWorkspaceFile(root, request, signal) + if (request.operation === 'write_file') { + return writeWorkspaceFile(root, request, signal); + } + return request.operation === 'preview_edit' + ? previewWorkspaceEdit(root, request, signal) : editWorkspaceFile(root, request, signal); } @@ -1482,6 +1625,18 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { this.capabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, operations: [...new Set([...base.operations, 'execute_command' as const])], + ...(base.writeFileModes != null + ? { writeFileModes: base.writeFileModes } + : {}), + ...(base.editFileModes != null + ? { editFileModes: base.editFileModes } + : {}), + ...(base.editFileFeatures != null + ? { editFileFeatures: base.editFileFeatures } + : {}), + ...(base.listFileFeatures != null + ? { listFileFeatures: base.listFileFeatures } + : {}), workspaces: base.workspaces.map((workspace) => ({ ...workspace, operations: [ diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 2c0d8998..11b0eb31 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -284,9 +284,14 @@ describe('paired bridge HTTP API', () => { 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', 'execute_command', ], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceListFileFeatures: ['after_path'], }); const crossDeploymentRevoke = await fetch( diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index b764bb39..7fa87b93 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -384,9 +384,14 @@ router.post( 'search_text', 'list_files', 'write_file', + 'preview_edit', 'edit_file', 'execute_command', ], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceListFileFeatures: ['after_path'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index b0a5c2fa..03694b1b 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -76,13 +76,41 @@ function supportsWorkspaceTool( const workspace = capabilities?.workspaces.find( (candidate) => candidate.id === request.workspaceId, ); - return ( + const supportsOperation = capabilities != null && capabilities.operations.includes(request.operation) && workspace != null && (workspace.operations == null || - workspace.operations.includes(request.operation)) - ); + workspace.operations.includes(request.operation)); + if (!supportsOperation) { + return supportsOperation; + } + if (request.operation === 'list_files' && request.afterPath !== undefined) { + return capabilities?.listFileFeatures?.includes('after_path') === true; + } + if (request.operation === 'write_file') { + const mode = request.overwrite === false ? 'create' : 'replace'; + const modes = capabilities?.writeFileModes; + return request.overwrite === undefined && modes == null + ? true + : modes?.includes(mode) === true; + } + if ( + request.operation === 'preview_edit' || + request.operation === 'edit_file' + ) { + const mode = request.edits === undefined ? 'single' : 'batch'; + const modes = capabilities?.editFileModes; + const supportsMode = modes == null ? mode === 'single' : modes.includes(mode); + if (request.operation === 'preview_edit') return supportsMode; + return ( + supportsMode && + (request.expectedBaseSha256 === undefined || + capabilities?.editFileFeatures?.includes('expected_base_sha256') === + true) + ); + } + return true; } function workerKey(workerId: string): string { @@ -485,22 +513,28 @@ export class RedisBridgeStore { 'Invalid workspace tool request', ); } - const settlement = (await this.dispatch({ + return (await this.dispatch({ ...args, body: {} as t.PayloadBody, headers: {}, workspaceRequest: args.request, + finalize: async (settlement, registration) => { + if ( + settlement.status === 'fulfilled' && + !isWorkspaceToolResult( + args.request, + settlement.result, + registration.capabilities.workspaceTools, + ) + ) { + throw new BridgeStoreError( + 'RESULT_INVALID', + 'Bridge worker returned an invalid workspace tool result', + ); + } + return settlement; + }, })) as unknown as CodeBridgeWorkspaceSettlement; - if ( - settlement.status === 'fulfilled' && - !isWorkspaceToolResult(args.request, settlement.result) - ) { - throw new BridgeStoreError( - 'RESULT_INVALID', - 'Bridge worker returned an invalid workspace tool result', - ); - } - return settlement; } async dispatch(args: { @@ -515,6 +549,7 @@ export class RedisBridgeStore { signal: AbortSignal; finalize?: ( settlement: CodeBridgeSettlement, + registration: RegisteredBridgeWorker, ) => Promise; }): Promise { this.assertDispatchActive(args.signal, args.deadlineAtMs); @@ -694,7 +729,7 @@ export class RedisBridgeStore { const result = args.finalize == null ? settlement - : await args.finalize(settlement); + : await args.finalize(settlement, registration); await this.commitPendingWorkspace( assignment, settlement, diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index 44ab66fe..a23274d8 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -44,7 +44,11 @@ test('dispatches a workspace tool only to a worker advertising its workspace and signal: new AbortController().signal, }); - const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + const assignment = await store.lease( + 'workspace-worker', + incarnationId, + 1_000, + ); expect(assignment).toMatchObject({ executionKind: 'workspace_tool', request, @@ -106,6 +110,142 @@ test('rejects a workspace tool that the selected worker did not advertise', asyn expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); }); +test('rejects listing continuation without the negotiated feature', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['list_files'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 10, + afterPath: 'src/app.ts', + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + +test('rejects pagination fields from a worker without the negotiated feature', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['list_files'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 1, + }, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: 'primary', + paths: ['first.txt'], + truncated: true, + nextAfterPath: 'first.txt', + }, + }); + + await expect(completion).rejects.toMatchObject({ code: 'RESULT_INVALID' }); +}); + +test('accepts a legacy truncated listing without a pagination cursor', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['list_files'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: 'primary', + maxResults: 2, + }, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + + const assignment = await store.lease( + 'workspace-worker', + incarnationId, + 1_000, + ); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: 'primary', + paths: ['src//z.ts', 'src/a.ts'], + truncated: true, + }, + }); + + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { paths: ['src//z.ts', 'src/a.ts'], truncated: true }, + }); +}); + test('rejects an operation omitted from the selected workspace capability', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -143,6 +283,181 @@ test('rejects an operation omitted from the selected workspace capability', asyn expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); }); +test('rejects create-only writes from workers without the negotiated mode', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['write_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'create me', + overwrite: false, + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + +test('rejects legacy replacement writes from create-only workers', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['write_file'], + writeFileModes: ['create'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'write_file', + workspaceId: 'primary', + path: 'notes.txt', + content: 'replace me', + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + +test('rejects batch edits from workers without the negotiated mode', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['edit_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'before', newText: 'after' }], + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + +test('rejects fenced edits from workers without the negotiated feature', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['edit_file'], + editFileModes: ['single'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'before', + newText: 'after', + expectedBaseSha256: 'a'.repeat(64), + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + +test('rejects batch previews from workers without the negotiated mode', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'before', newText: 'after' }], + }, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + test('rejects a fulfilled workspace settlement that violates the result contract', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, From 5510e76fd36e47ebefd23990e33e3589e6289bc0 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 4 Sep 2026 12:02:46 -0400 Subject: [PATCH 035/116] =?UTF-8?q?=F0=9F=94=A4=20fix:=20Stabilize=20Trunc?= =?UTF-8?q?ated=20Workspace=20Search=20Results=20(#109)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `list_files` passes `--sort path`; the search candidate listing did not, so ripgrep's parallel directory walk decided the order. Two consequences: search results came back in filesystem-dependent order, and - the part that matters - when the candidate limit or maxResults truncates, *which* files get searched was arbitrary, so a bounded search could silently return different matches on identical input. `search decodes BOM-marked UTF-16 text` asserts a fixed order and fails 12/12 on ext4 with ripgrep 13.0.0; it passes only where the walk happens to emit sorted order. Sorting the candidates matches list_files and makes the bound reproducible: 0/10 failures after. Sorting disables ripgrep's parallel walk, but the cost is immaterial against the 10s search budget: 16ms versus 8ms over a 4,856-file tree, the same tradeoff list_files already makes. --- packages/code/src/workspace.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 727955cd..0fa3c8ba 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -840,6 +840,10 @@ async function listSearchCandidates( '--files', '--no-config', '--no-follow', + /* Deterministic order, as list_files uses: without it ripgrep's parallel + * walk decides both result order and which files survive truncation. */ + '--sort', + 'path', '--path-separator', '/', '--null', From 003524b73ca576a1be58dac61f454203f9f2633f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 4 Sep 2026 12:13:48 -0400 Subject: [PATCH 036/116] =?UTF-8?q?=F0=9F=A7=AB=20ci:=20Run=20Code=20Packa?= =?UTF-8?q?ge=20Tests=20on=20Pull=20Requests=20(#110)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `packages/code` had no CI job, so its 234 tests never ran on a pull request - including the ones covering the bridge identity, the mutation quarantine, and the workspace tools. The job pins Node 24.16.0 rather than tracking the engines range: the suite fails 47 worker tests on Node 22 despite the package declaring ">=20.11", and CI should pin the version it is actually green on rather than advertise a range nobody verifies. Ripgrep is installed explicitly because list_files and search_text shell out to it; without it 37 tests fail with LIST_UNAVAILABLE and SEARCH_UNAVAILABLE, so it is a dependency of the suite, not an assumption about the runner image. --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd89d5da..97d76161 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,37 @@ jobs: - name: Bun tests run: bun run test + code-package-tests: + name: Code Package Tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: packages/code + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + # The suite fails 47 worker tests on Node 22 despite the package's + # ">=20.11" engines range, so CI pins the version it is green on. + node-version: 24.16.0 + cache: npm + cache-dependency-path: packages/code/package-lock.json + + - name: Install ripgrep + # list_files and search_text shell out to rg. Without it the workspace + # tools degrade to LIST_UNAVAILABLE / SEARCH_UNAVAILABLE and 37 tests + # fail, so the dependency is part of the job, not an assumption. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ripgrep + + - name: Install dependencies + run: npm ci + + - name: Tests + run: npm test + lambda-microvm-provisioning: name: Lambda MicroVM Provisioning runs-on: ubuntu-latest From 6a3bb9c5cfe5072f7c635b691dc3706d8b64393c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 4 Sep 2026 13:49:32 -0400 Subject: [PATCH 037/116] =?UTF-8?q?=F0=9F=8E=96=EF=B8=8F=20fix:=20Enforce?= =?UTF-8?q?=20Owner-Only=20Worker=20Credentials=20(#107)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(code): fail closed when worker credentials cannot be owner-only `chmod` reports success without effect on mounts that do not implement POSIX permissions. On WSL2 DrvFs (`/mnt/`) the paired identity was written and reported as saved while remaining world-accessible: POSIX mode 777, and a Windows ACL granting `Authenticated Users: Modify`. Any local authenticated user could read the worker's Ed25519 private key, contradicting the owner-only guarantee the README documents. Tighten explicitly, then verify, and fail closed when group or other access survives - on the write path and the read path both, since hardening only writes would leave already-paired workers booting on an exposed key. The identity verdict is taken against the still-empty temporary file, so no private key is written to a world-readable path, and the load paths validate and read through a single descriptor so a retargeted symlink cannot make the file that was judged differ from the file that is read. Symlinks resolve for the verdict: a link's own mode is always 0777 and ignored by the kernel, so the file the bytes live in is what counts. Skipped on win32, where POSIX mode bits are not meaningful and NTFS ACLs govern access. A quarantine marker that cannot be protected is removed rather than left unparseable for every later load, and that removal is synced as durably as the write it undoes. * docs(code): state what the mode check does not cover The verdict reads POSIX mode bits. A Linux POSIX ACL surfaces its mask in the group bits and is caught, but a macOS extended ACL inherited from the parent directory is invisible to `stat` and survives `chmod`, and Windows is exempt outright. Say so at the check rather than let the name imply more than it verifies. --- packages/code/src/storage.test.ts | 164 +++++++++++++++++++++++++++++- packages/code/src/storage.ts | 115 ++++++++++++++++++--- 2 files changed, 265 insertions(+), 14 deletions(-) diff --git a/packages/code/src/storage.test.ts b/packages/code/src/storage.test.ts index 3a17705a..b45d351d 100644 --- a/packages/code/src/storage.test.ts +++ b/packages/code/src/storage.test.ts @@ -1,5 +1,15 @@ import assert from 'node:assert/strict'; -import { mkdtemp, open, rm, stat, writeFile } from 'node:fs/promises'; +import { + chmod, + mkdir, + mkdtemp, + open, + readdir, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -170,7 +180,8 @@ test('workspace mutation quarantine persists until explicitly cleared', async (t await clearWorkspaceMutationQuarantine(path); assert.equal(syncCalls, process.platform === 'win32' ? 1 : 4); assert.equal(await loadWorkspaceMutationQuarantine(path), undefined); - await writeFile(path, '{bad json', 'utf8'); + /* Owner-only, so this exercises the parse failure and not the mode check. */ + await writeFile(path, '{bad json', { encoding: 'utf8', mode: 0o600 }); await assert.rejects( loadWorkspaceMutationQuarantine(path), /invalid workspace quarantine file/i, @@ -213,3 +224,152 @@ test('workspace mutation quarantine cannot be replaced or cleared by another own await rm(directory, { recursive: true, force: true }); } }); + +const SAMPLE_IDENTITY = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', +}; + +/** + * Locate a mount that ignores POSIX permissions (WSL2 DrvFs under `/mnt/`). + * Returns undefined on hosts where every writable filesystem honours chmod. + */ +async function findChmodIgnoringDirectory(): Promise { + const roots: string[] = []; + const configured = process.env.LIBRECHAT_CODE_TEST_NONPOSIX_DIR?.trim(); + if (configured) roots.push(configured); + try { + for (const entry of await readdir('/mnt')) roots.push(join('/mnt', entry)); + } catch { + /* No /mnt on this host. */ + } + for (const root of roots) { + let directory: string | undefined; + try { + directory = await mkdtemp(join(root, 'librechat-code-mode-')); + const probe = join(directory, 'probe'); + await writeFile(probe, '', { mode: 0o600 }); + await chmod(probe, 0o600); + if (((await stat(probe)).mode & 0o077) !== 0) return directory; + } catch { + /* Root is absent or not writable. */ + } + if (directory) await rm(directory, { recursive: true, force: true }); + } + return undefined; +} + +test('credential storage fails closed on filesystems that ignore chmod', async (t) => { + const directory = await findChmodIgnoringDirectory(); + if (!directory) { + t.skip('no chmod-ignoring filesystem available on this host'); + return; + } + try { + const identityPath = join(directory, 'worker.json'); + await assert.rejects( + saveBridgeIdentity(identityPath, SAMPLE_IDENTITY), + /owner-only access/, + ); + /* The private key must not be left behind on a world-readable path. */ + await assert.rejects(stat(identityPath), { code: 'ENOENT' }); + + await assert.rejects( + ensurePrivateWorkspaceDirectory(join(directory, 'workspace')), + /owner-only access/, + ); + + const quarantinePath = join(directory, 'quarantine.json'); + await assert.rejects( + saveWorkspaceMutationQuarantine(quarantinePath, { + version: 1, + workerId: 'vm-1', + workspaceId: 'primary', + quarantinedAt: new Date().toISOString(), + reason: 'test', + }), + /owner-only access/, + ); + /* An unreadable half-written marker would wedge every later load. */ + await assert.rejects(stat(quarantinePath), { code: 'ENOENT' }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('an already-exposed identity is refused on load', async (t) => { + const directory = await findChmodIgnoringDirectory(); + if (!directory) { + t.skip('no chmod-ignoring filesystem available on this host'); + return; + } + try { + /* Written the way a release without the save-time check would have. */ + const path = join(directory, 'legacy-worker.json'); + await writeFile(path, JSON.stringify(SAMPLE_IDENTITY), { mode: 0o600 }); + await assert.rejects(loadBridgeIdentity(path), /accessible beyond its owner/); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('an already-exposed quarantine marker is refused on load', async (t) => { + const directory = await findChmodIgnoringDirectory(); + if (!directory) { + t.skip('no chmod-ignoring filesystem available on this host'); + return; + } + try { + const path = join(directory, 'quarantine.json'); + await writeFile( + path, + JSON.stringify({ + version: 1, + workerId: 'vm-1', + workspaceId: 'primary', + ownerId: 'incarnation-1', + quarantinedAt: new Date().toISOString(), + reason: 'test', + }), + { mode: 0o600 }, + ); + await assert.rejects( + loadWorkspaceMutationQuarantine(path), + /accessible beyond its owner/, + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('a symlinked owner-only identity is accepted', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const target = join(base, 'real.json'); + await saveBridgeIdentity(target, SAMPLE_IDENTITY); + /* A link's own mode is always 0777; the credential's mode is the target's. */ + const link = join(base, 'link.json'); + await symlink(target, link); + assert.deepEqual(await loadBridgeIdentity(link), SAMPLE_IDENTITY); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('default workspace directories are tightened when they already exist', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const workspace = join(base, 'workspace'); + await mkdir(workspace, { mode: 0o777 }); + await chmod(workspace, 0o777); + await ensurePrivateWorkspaceDirectory(workspace); + assert.equal((await stat(workspace)).mode & 0o777, 0o700); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 2a7eef5c..8c0d1cbf 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes } from 'node:crypto'; -import { chmod, lstat, mkdir, open, readFile, rename, rm } from 'node:fs/promises'; +import { chmod, lstat, mkdir, open, readFile, rename, rm, stat } from 'node:fs/promises'; import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -149,6 +149,67 @@ export function defaultWorkspaceQuarantinePath( ); } +/** + * Verify a path really is owner-only. `chmod` reports success without effect on + * mounts that do not implement POSIX permissions - notably WSL2 DrvFs + * (`/mnt/`), where the result stays world-accessible - so a credential + * that cannot be protected must fail closed rather than appear protected. + * + * Symlinks are resolved: a link's own mode is always `0777` and ignored by the + * kernel, so the file the bytes live in is what counts. + * + * This reads POSIX mode bits, which is not the whole access story everywhere. + * A Linux POSIX ACL surfaces its mask in the group bits and so is caught, but + * a macOS extended ACL inherited from the parent directory is invisible here + * and survives `chmod`, and Windows is exempt entirely. Establishing owner-only + * storage on those needs real ACL inspection; until then this verifies what the + * mode can express and nothing more. + */ +async function groupOrOtherAccessMode( + path: string, +): Promise { + if (process.platform === 'win32') return undefined; + const mode = (await stat(path)).mode & 0o777; + return (mode & 0o077) === 0 ? undefined : mode; +} + +async function assertOwnerOnlyPath( + path: string, + reportedPath: string = path, +): Promise { + const mode = await groupOrOtherAccessMode(path); + if (mode === undefined) return; + throw new BridgeProtocolError( + `Cannot restrict ${reportedPath} to owner-only access (mode ${mode.toString(8)}). ` + + 'Filesystems that ignore POSIX permissions, such as Windows drives mounted ' + + 'under /mnt, cannot protect worker credentials or workspaces. Use a path on a ' + + 'native Linux filesystem.', + ); +} + +/** + * Validate and read through one descriptor. Checking a path and then reading it + * resolves the name twice, so a symlink retargeted in between would let the file + * that was judged differ from the file that is read. + */ +async function readGuardedFile( + path: string, + exposed: (mode: string) => string, +): Promise { + const handle = await open(path, 'r'); + try { + if (process.platform !== 'win32') { + const mode = (await handle.stat()).mode & 0o777; + if ((mode & 0o077) !== 0) { + throw new BridgeProtocolError(exposed(mode.toString(8))); + } + } + return await handle.readFile('utf8'); + } finally { + await handle.close(); + } +} + export async function ensurePrivateWorkspaceDirectory( path: string, ): Promise { @@ -158,6 +219,7 @@ export async function ensurePrivateWorkspaceDirectory( throw new BridgeProtocolError('Default workspace path must be a directory'); } await chmod(path, 0o700); + await assertOwnerOnlyPath(path); } export async function saveBridgeIdentity( @@ -169,6 +231,10 @@ export async function saveBridgeIdentity( try { const file = await open(temporaryPath, 'wx', 0o600); try { + /* Tighten every way available before judging, and judge before the key + * is written, so no private key reaches a world-readable path. */ + await file.chmod(0o600); + await assertOwnerOnlyPath(temporaryPath, path); await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); await file.sync(); } finally { @@ -205,10 +271,24 @@ export async function saveWorkspaceMutationQuarantine( await ensureDurableDirectory(dirname(path)); const file = await open(path, 'wx', 0o600); try { - await file.writeFile(`${JSON.stringify(record, null, 2)}\n`, 'utf8'); - await file.sync(); - } finally { - await file.close(); + try { + await file.chmod(0o600); + await assertOwnerOnlyPath(path); + await file.writeFile(`${JSON.stringify(record, null, 2)}\n`, 'utf8'); + await file.sync(); + } finally { + await file.close(); + } + } catch (error) { + /* A partial marker fails every later load, so undoing it has to reach the + * disk as durably as the write it is undoing. */ + await rm(path, { force: true }); + try { + await syncParentDirectory(path); + } catch { + /* Surface the original failure, not a cleanup-durability one. */ + } + throw error; } await syncParentDirectory(path); } @@ -218,13 +298,15 @@ export async function loadWorkspaceMutationQuarantine( ): Promise { let content: string; try { - content = await readFile(path, 'utf8'); + content = await readGuardedFile( + path, + (mode) => + `Workspace quarantine ${path} is accessible beyond its owner (mode ${mode}). ` + + 'Another local account could clear or forge it. Keep worker state on a ' + + 'native Linux filesystem.', + ); } catch (error) { - if ( - isRecord(error) && - 'code' in error && - error.code === 'ENOENT' - ) { + if (isMissingPathError(error)) { return undefined; } throw error; @@ -278,7 +360,16 @@ export async function assertWorkspaceMutationQuarantineOwner( export async function loadBridgeIdentity( path: string, ): Promise { - const identity = JSON.parse(await readFile(path, 'utf8')) as unknown; + /* An identity written before this check, or by an older release, is still a + * private key other local accounts can read. Refuse it rather than booting. */ + const content = await readGuardedFile( + path, + (mode) => + `Bridge identity ${path} is accessible beyond its owner (mode ${mode}). ` + + 'Treat its private key as compromised: revoke the worker and pair again with an ' + + 'identity path on a native Linux filesystem.', + ); + const identity = JSON.parse(content) as unknown; if (!isPairedIdentity(identity)) { throw new BridgeProtocolError(`Invalid bridge identity file: ${path}`); } From 81610a73fac195c8005fb7ea3fece5c137678a00 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 4 Sep 2026 15:46:48 -0400 Subject: [PATCH 038/116] =?UTF-8?q?=F0=9F=8F=AF=20fix:=20Guard=20Worker=20?= =?UTF-8?q?Credentials=20From=20Local=20Accounts=20(#112)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything here defends a threat the merged owner-only fix does not: another account on the same host. It is separate deliberately, because BYOM's stated model is a worker on the user's own machine or VM with the sandboxed command as the adversary, and these checks buy that defence by trading deployment flexibility for it. Ownership. Mode bits do not establish trust: a 0600 file owned by another account is unreadable by others yet fully rewritable by its owner, who then controls the credential the worker loads - or, for a quarantine marker, can delete it and let mutations resume. The containing directory is judged the same way, since an owner lacking write bits today can grant them tomorrow. Root counts as the trust root. `--default-workspace` is application-owned by contract, so a pre-existing one under another account is refused too. Containers. A 0600 file in a directory others can write can be unlinked and replaced. Publishing goes through `rename`, which replaces the named entry, so the write path judges the entry's directory; reading follows the link, so both ends are judged. The sticky bit counts as protection, keeping /tmp-style parents usable. Only the immediate container is inspected. Pairing preflight. `pair` redeemed the one-time code before the destination was known usable, so an unusable path cost the code and left an orphaned remote pairing. The destination is now validated - rejecting a directory, a foreign-owned file under a sticky bit that `rename` could not replace, and a parent that denies the sibling temporary file the publish needs - and then claimed, so another account cannot take the name while the pairing request is in flight. The claim records its inode and is released only if the file is still that inode and still empty, so a concurrent pairing that published a real identity over the name is never destroyed by another invocation's unwind. Known gaps, left explicit rather than half-done: only the immediate container is checked, so a writable ancestor could still rename a private directory out from under the worker; a bind-mounted destination still fails at `rename` with EBUSY because every way to detect it either false-positives on btrfs subvolumes or races; and macOS extended ACLs and Windows ACLs are both outside what a mode check can see, which needs real ACL inspection on a host that can validate it. --- packages/code/src/cli.ts | 11 +- packages/code/src/storage.test.ts | 348 +++++++++++++++++++++++++++--- packages/code/src/storage.ts | 274 +++++++++++++++++++++-- 3 files changed, 578 insertions(+), 55 deletions(-) diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 06aec93b..a13eb011 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -8,6 +8,7 @@ import { pairBridgeWorker } from './pairing.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { + assertIdentityPathIsPrivate, assertWorkspaceMutationQuarantineOwner, clearWorkspaceMutationQuarantine, defaultBridgeIdentityPath, @@ -128,8 +129,14 @@ async function pair(args: string[]): Promise { option(args, '--identity') ?? process.env.LIBRECHAT_CODE_IDENTITY_FILE ?? defaultBridgeIdentityPath(workerId); - const identity = await pairBridgeWorker({ codeApiUrl, workerId, code }); - await saveBridgeIdentity(identityPath, identity); + const reservation = await assertIdentityPathIsPrivate(identityPath); + try { + const identity = await pairBridgeWorker({ codeApiUrl, workerId, code }); + await saveBridgeIdentity(identityPath, identity); + } catch (error) { + await reservation.release(); + throw error; + } process.stdout.write( `Paired worker ${workerId}. Identity saved to ${identityPath}\n`, ); diff --git a/packages/code/src/storage.test.ts b/packages/code/src/storage.test.ts index b45d351d..f3e8243e 100644 --- a/packages/code/src/storage.test.ts +++ b/packages/code/src/storage.test.ts @@ -1,15 +1,5 @@ import assert from 'node:assert/strict'; -import { - chmod, - mkdir, - mkdtemp, - open, - readdir, - rm, - stat, - symlink, - writeFile, -} from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, open, readdir, rm, stat, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -17,6 +7,7 @@ import test from 'node:test'; import type { FileHandle } from 'node:fs/promises'; import { + assertIdentityPathIsPrivate, clearWorkspaceMutationQuarantine, defaultBridgeIdentityPath, defaultWorkspaceQuarantinePath, @@ -225,16 +216,6 @@ test('workspace mutation quarantine cannot be replaced or cleared by another own } }); -const SAMPLE_IDENTITY = { - protocolVersion: 1 as const, - workerId: 'vm-1', - codeApiUrl: 'https://code.example/v1', - credential: 'credential', - expiresAt: new Date(Date.now() + 60_000).toISOString(), - publicKey: 'public', - privateKey: 'private', -}; - /** * Locate a mount that ignores POSIX permissions (WSL2 DrvFs under `/mnt/`). * Returns undefined on hosts where every writable filesystem honours chmod. @@ -273,7 +254,15 @@ test('credential storage fails closed on filesystems that ignore chmod', async ( try { const identityPath = join(directory, 'worker.json'); await assert.rejects( - saveBridgeIdentity(identityPath, SAMPLE_IDENTITY), + saveBridgeIdentity(identityPath, { + protocolVersion: 1, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }), /owner-only access/, ); /* The private key must not be left behind on a world-readable path. */ @@ -309,15 +298,72 @@ test('an already-exposed identity is refused on load', async (t) => { return; } try { - /* Written the way a release without the save-time check would have. */ + /* Write it the way a release without the save-time check would have. */ const path = join(directory, 'legacy-worker.json'); - await writeFile(path, JSON.stringify(SAMPLE_IDENTITY), { mode: 0o600 }); + await writeFile( + path, + JSON.stringify({ + protocolVersion: 1, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }), + { mode: 0o600 }, + ); await assert.rejects(loadBridgeIdentity(path), /accessible beyond its owner/); } finally { await rm(directory, { recursive: true, force: true }); } }); +test('the identity destination is rejected before a pairing code is spent', async (t) => { + const directory = await findChmodIgnoringDirectory(); + if (!directory) { + t.skip('no chmod-ignoring filesystem available on this host'); + return; + } + try { + const identityPath = join(directory, 'worker.json'); + await assert.rejects( + assertIdentityPathIsPrivate(identityPath), + /owner-only access/, + ); + /* The probe must not survive the rejection. */ + assert.deepEqual( + (await readdir(directory)).filter((name) => name.includes('.probe')), + [], + ); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('a symlinked owner-only identity is accepted', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + const target = join(base, 'real.json'); + await saveBridgeIdentity(target, identity); + /* A link's own mode is always 0777; the credential's mode is the target's. */ + const link = join(base, 'link.json'); + await symlink(target, link); + assert.deepEqual(await loadBridgeIdentity(link), identity); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + test('an already-exposed quarantine marker is refused on load', async (t) => { const directory = await findChmodIgnoringDirectory(); if (!directory) { @@ -325,6 +371,7 @@ test('an already-exposed quarantine marker is refused on load', async (t) => { return; } try { + /* Write it the way a release without the save-time check would have. */ const path = join(directory, 'quarantine.json'); await writeFile( path, @@ -347,16 +394,257 @@ test('an already-exposed quarantine marker is refused on load', async (t) => { } }); -test('a symlinked owner-only identity is accepted', async () => { +test('a directory at the identity path is rejected before the code is spent', async () => { const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); try { - const target = join(base, 'real.json'); - await saveBridgeIdentity(target, SAMPLE_IDENTITY); - /* A link's own mode is always 0777; the credential's mode is the target's. */ - const link = join(base, 'link.json'); + const identityPath = join(base, 'worker.json'); + await mkdir(identityPath); + await assert.rejects( + assertIdentityPathIsPrivate(identityPath), + /is a directory/, + ); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('re-pairing over an existing owner-only identity is still allowed', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + const identityPath = join(base, 'worker.json'); + await saveBridgeIdentity(identityPath, identity); + await assertIdentityPathIsPrivate(identityPath); + const replacement = { ...identity, credential: 'rotated' }; + await saveBridgeIdentity(identityPath, replacement); + assert.deepEqual(await loadBridgeIdentity(identityPath), replacement); + /* No probe may survive a successful preflight either. */ + assert.deepEqual( + (await readdir(base)).filter((name) => name.includes('.probe')), + [], + ); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('an owned identity file in a sticky directory is still replaceable', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + /* Ownership only blocks rename under the sticky bit, and only for a file + * this account does not own - which /tmp-style directories make common. */ + const sticky = join(base, 'sticky'); + await mkdir(sticky); + await chmod(sticky, 0o1777); + const identityPath = join(sticky, 'worker.json'); + await writeFile(identityPath, '{}', { mode: 0o600 }); + await assertIdentityPathIsPrivate(identityPath); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('a credential in a shared writable directory is refused', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + const shared = join(base, 'shared'); + await mkdir(shared); + const identityPath = join(shared, 'worker.json'); + await saveBridgeIdentity(identityPath, identity); + /* 0600 still, but anyone may now unlink and substitute it. */ + await chmod(shared, 0o777); + await assert.rejects( + loadBridgeIdentity(identityPath), + /writable by other accounts/, + ); + await assert.rejects( + assertIdentityPathIsPrivate(identityPath), + /writable by other accounts/, + ); + + /* The sticky bit restores owner-only unlink, so /tmp-style parents work. */ + await chmod(shared, 0o1777); + assert.deepEqual(await loadBridgeIdentity(identityPath), identity); + await assertIdentityPathIsPrivate(identityPath); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('a symlink entry in a shared writable directory is refused', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + /* Credential is private; the link naming it is not, so the link can be + * repointed at an attacker's file. */ + const priv = join(base, 'private'); + await mkdir(priv, { mode: 0o700 }); + const target = join(priv, 'worker.json'); + await saveBridgeIdentity(target, identity); + const shared = join(base, 'shared'); + await mkdir(shared); + const link = join(shared, 'worker.json'); await symlink(target, link); - assert.deepEqual(await loadBridgeIdentity(link), SAMPLE_IDENTITY); + assert.deepEqual(await loadBridgeIdentity(link), identity); + await chmod(shared, 0o777); + await assert.rejects( + loadBridgeIdentity(link), + /writable by other accounts/, + ); + /* Pairing publishes by replacing the link entry, so the same directory + * governs the write path too. */ + await assert.rejects( + assertIdentityPathIsPrivate(link), + /writable by other accounts/, + ); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('a symlink into a shared writable directory is refused', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + const shared = join(base, 'shared'); + await mkdir(shared); + const target = join(shared, 'worker.json'); + await saveBridgeIdentity(target, identity); + /* The link sits in a private directory; the credential does not. */ + const priv = join(base, 'private'); + await mkdir(priv, { mode: 0o700 }); + const link = join(priv, 'worker.json'); + await symlink(target, link); + assert.deepEqual(await loadBridgeIdentity(link), identity); + await chmod(shared, 0o777); + await assert.rejects( + loadBridgeIdentity(link), + /writable by other accounts/, + ); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('the identity destination is reserved across pairing and released on failure', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identityPath = join(base, 'worker.json'); + const reservation = await assertIdentityPathIsPrivate(identityPath); + /* The name is claimed while the pairing request is in flight, so another + * account cannot take it and strand a spent code. */ + assert.equal((await stat(identityPath)).mode & 0o777, 0o600); + await reservation.release(); + await assert.rejects(stat(identityPath), { code: 'ENOENT' }); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('releasing never removes a pre-existing identity', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + const identityPath = join(base, 'worker.json'); + await saveBridgeIdentity(identityPath, identity); + const reservation = await assertIdentityPathIsPrivate(identityPath); + await reservation.release(); + assert.deepEqual(await loadBridgeIdentity(identityPath), identity); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('releasing never removes an identity another pairing published', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + const identityPath = join(base, 'worker.json'); + const first = await assertIdentityPathIsPrivate(identityPath); + /* A concurrent pair publishes over the reserved name and completes. */ + await saveBridgeIdentity(identityPath, identity); + /* The first invocation then fails and unwinds; its credential is gone, but + * the one that succeeded must survive. */ + await first.release(); + assert.deepEqual(await loadBridgeIdentity(identityPath), identity); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + +test('an identity in a directory that denies creation is rejected before pairing', async () => { + const base = await mkdtemp(join(tmpdir(), 'librechat-code-storage-')); + const locked = join(base, 'locked'); + try { + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'credential', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: 'public', + privateKey: 'private', + }; + await mkdir(locked, { mode: 0o700 }); + const identityPath = join(locked, 'worker.json'); + await saveBridgeIdentity(identityPath, identity); + /* Readable and owner-only, but the publish writes a sibling first. */ + await chmod(locked, 0o500); + await assert.rejects( + assertIdentityPathIsPrivate(identityPath), + /could not be published there/, + ); } finally { + await chmod(locked, 0o700).catch(() => undefined); await rm(base, { recursive: true, force: true }); } }); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 8c0d1cbf..af3e90a7 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,5 +1,15 @@ import { createHash, randomBytes } from 'node:crypto'; -import { chmod, lstat, mkdir, open, readFile, rename, rm, stat } from 'node:fs/promises'; +import { + chmod, + lstat, + mkdir, + open, + readFile, + realpath, + rename, + rm, + stat, +} from 'node:fs/promises'; import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -169,28 +179,95 @@ async function groupOrOtherAccessMode( path: string, ): Promise { if (process.platform === 'win32') return undefined; + /* Resolve symlinks: the bytes live at the target, and a link's own mode is + * always 0777 and ignored by the kernel. */ const mode = (await stat(path)).mode & 0o777; return (mode & 0o077) === 0 ? undefined : mode; } -async function assertOwnerOnlyPath( +/** + * A `0600` file in a directory other accounts can write is not owner-only in + * practice: they cannot read it, but they can unlink and substitute it, so a + * swapped credential or a forged quarantine marker would be trusted. The sticky + * bit counts as protection, which keeps shared `/tmp`-style parents usable. A + * writable ancestor above a private directory could still have that directory + * renamed out from under us, which is broader hardening than this addresses. + * + * Deliberately not applied to the registered workspace, which is the user's own + * project directory and may legitimately be shared. + */ +async function assertDirectoryNotSharedWritable( + directory: string, path: string, - reportedPath: string = path, ): Promise { - const mode = await groupOrOtherAccessMode(path); - if (mode === undefined) return; + const metadata = await stat(directory); + const mode = metadata.mode & 0o7777; + const uid = process.getuid?.(); + if (uid !== undefined && !isTrustedOwner(metadata.uid, uid)) { + throw new BridgeProtocolError( + `Directory ${directory} is owned by another account (uid ${metadata.uid}), ` + + `which can grant itself write access and replace ${path}. Keep worker ` + + 'credentials in a directory this account owns.', + ); + } + if ((mode & 0o022) === 0) return; + if ((mode & 0o1000) !== 0 && (metadata.uid === uid || metadata.uid === 0)) { + return; + } throw new BridgeProtocolError( - `Cannot restrict ${reportedPath} to owner-only access (mode ${mode.toString(8)}). ` + - 'Filesystems that ignore POSIX permissions, such as Windows drives mounted ' + - 'under /mnt, cannot protect worker credentials or workspaces. Use a path on a ' + - 'native Linux filesystem.', + `Directory ${directory} is writable by other accounts (mode ${mode.toString(8)}), ` + + `so ${path} can be replaced even while owner-only. Keep worker credentials ` + + 'in a directory only this account can write.', + ); +} + +/** Publishing goes through `rename`, which replaces the named entry itself. */ +async function assertWriteContainerPrivate(path: string): Promise { + if (process.platform === 'win32' || process.getuid === undefined) return; + await assertDirectoryNotSharedWritable(await realpath(dirname(path)), path); +} + +/** + * Reading follows the link, so both the entry and the file it names are trust + * boundaries: a writable directory at either end allows a substitution. + */ +async function assertReadPathPrivate(path: string): Promise { + if (process.platform === 'win32' || process.getuid === undefined) return; + const entryDirectory = await realpath(dirname(path)); + await assertDirectoryNotSharedWritable(entryDirectory, path); + const targetDirectory = dirname(await realpath(path)); + if (targetDirectory !== entryDirectory) { + await assertDirectoryNotSharedWritable(targetDirectory, path); + } +} + +/** Root is the trust root; anyone else holding a credential path is not. */ +function isTrustedOwner(uid: number, self: number): boolean { + return uid === self || uid === 0; +} + +/** + * Mode bits alone do not establish trust. A `0600` file owned by another + * account is unreadable by others yet fully rewritable by its owner, who then + * controls the credential the worker loads - or, for a quarantine marker, can + * delete it and let mutations resume. + */ +async function assertOwnedByWorker(path: string): Promise { + const self = process.getuid?.(); + if (self === undefined) return; + const { uid } = await stat(path); + if (isTrustedOwner(uid, self)) return; + throw new BridgeProtocolError( + `${path} is owned by another account (uid ${uid}), which can rewrite it. ` + + 'Keep worker credentials on a path this account owns.', ); } /** - * Validate and read through one descriptor. Checking a path and then reading it - * resolves the name twice, so a symlink retargeted in between would let the file - * that was judged differ from the file that is read. + * Validate and read through one descriptor. Re-resolving the path after a + * check lets a multi-hop symlink be toggled in between, so the file that was + * judged need not be the file that is read; holding the descriptor removes the + * second resolution entirely. */ async function readGuardedFile( path: string, @@ -198,11 +275,17 @@ async function readGuardedFile( ): Promise { const handle = await open(path, 'r'); try { + const stats = await handle.stat(); + const self = process.getuid?.(); + if (self !== undefined && !isTrustedOwner(stats.uid, self)) { + throw new BridgeProtocolError( + `${path} is owned by another account (uid ${stats.uid}), which can rewrite it. ` + + 'Keep worker credentials on a path this account owns.', + ); + } if (process.platform !== 'win32') { - const mode = (await handle.stat()).mode & 0o777; - if ((mode & 0o077) !== 0) { - throw new BridgeProtocolError(exposed(mode.toString(8))); - } + const mode = stats.mode & 0o777; + if ((mode & 0o077) !== 0) throw new BridgeProtocolError(exposed(mode.toString(8))); } return await handle.readFile('utf8'); } finally { @@ -210,6 +293,20 @@ async function readGuardedFile( } } +async function assertOwnerOnlyPath( + path: string, + reportedPath: string = path, +): Promise { + const mode = await groupOrOtherAccessMode(path); + if (mode === undefined) return; + throw new BridgeProtocolError( + `Cannot restrict ${reportedPath} to owner-only access (mode ${mode.toString(8)}). ` + + 'Filesystems that ignore POSIX permissions, such as Windows drives mounted ' + + 'under /mnt, cannot protect worker credentials or workspaces. Use a path on a ' + + 'native Linux filesystem.', + ); +} + export async function ensurePrivateWorkspaceDirectory( path: string, ): Promise { @@ -220,6 +317,135 @@ export async function ensurePrivateWorkspaceDirectory( } await chmod(path, 0o700); await assertOwnerOnlyPath(path); + /* This directory is application-owned by contract; a pre-existing one under + * another account lets that owner alter workspace inputs and results. */ + await assertOwnedByWorker(path); +} + +/** + * `saveBridgeIdentity` publishes by `rename`, which cannot replace a directory + * and cannot replace a file this account does not own in a sticky directory. + * Probing a sibling path alone would miss both, and the failure would land + * after the code was already spent. + */ +async function assertIdentityDestinationIsReplaceable( + path: string, +): Promise { + let metadata; + try { + metadata = await lstat(path); + } catch (error) { + if (isMissingPathError(error)) return; + throw error; + } + if (metadata.isDirectory()) { + throw new BridgeProtocolError( + `Bridge identity path ${path} is a directory. Point --identity at a file.`, + ); + } + const uid = process.platform === 'win32' ? undefined : process.getuid?.(); + if (uid === undefined || uid === 0 || metadata.uid === uid) return; + /* Ownership only blocks `rename` under the sticky bit, and owning the + * directory is enough there; elsewhere the parent's write bit decides. */ + const parent = await stat(dirname(path)); + if ((parent.mode & 0o1000) === 0 || parent.uid === uid) return; + throw new BridgeProtocolError( + `Bridge identity path ${path} is owned by another account (uid ${metadata.uid}) ` + + `inside the sticky directory ${dirname(path)}, so it cannot be replaced. ` + + 'Choose a path this account owns.', + ); +} + +/** + * Probe the identity destination before a one-time pairing code is redeemed, so + * a filesystem that cannot hold the credential fails validation instead of + * burning the code and leaving an orphaned remote pairing. + */ +export interface IdentityPathReservation { + /** Drop a destination this call created, when pairing does not reach a save. */ + release(): Promise; +} + +/** + * `saveBridgeIdentity` publishes by writing `..tmp` beside the + * destination and renaming it over. A directory that holds a readable identity + * but denies creation - `0500`, say - passes every check on the file itself and + * still fails the save, so exercise the sibling write rather than infer it. + */ +async function assertSiblingPublishable(path: string): Promise { + const probePath = `${path}.${randomBytes(8).toString('hex')}.probe`; + try { + await (await open(probePath, 'wx', 0o600)).close(); + } catch (error) { + throw new BridgeProtocolError( + `Cannot create a temporary file beside ${path} (${ + isRecord(error) && typeof error.code === 'string' ? error.code : 'unknown' + }), so the identity could not be published there. Choose a writable directory.`, + ); + } finally { + await rm(probePath, { force: true }); + } +} + +export async function assertIdentityPathIsPrivate( + path: string, +): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await assertIdentityDestinationIsReplaceable(path); + let created = false; + let reservedInode: bigint | undefined; + try { + /* Claiming the destination itself, rather than probing a sibling and + * letting go, is what keeps the verdict true across the pairing request: + * a shared sticky directory otherwise lets another account take the name + * in that window, and the save would fail with the code already spent. */ + const reserved = await open(path, 'wx', 0o600); + try { + created = true; + await reserved.chmod(0o600); + await assertOwnerOnlyPath(path); + reservedInode = (await reserved.stat({ bigint: true })).ino; + } finally { + await reserved.close(); + } + } catch (error) { + if (created) { + await rm(path, { force: true }); + throw error; + } + if (!isRecord(error) || error.code !== 'EEXIST') throw error; + /* Already present: judge what is there instead of the placeholder, and + * prove the publish itself is possible - an unwritable parent would + * otherwise surface as EACCES only after the code was spent. */ + await assertOwnedByWorker(path); + await assertOwnerOnlyPath(path); + await assertSiblingPublishable(path); + } + try { + /* After the mode verdict, so a filesystem that cannot hold an owner-only + * file keeps the more specific diagnosis. */ + await assertWriteContainerPrivate(path); + } catch (error) { + if (created) await rm(path, { force: true }); + throw error; + } + return { + async release(): Promise { + if (!created || reservedInode === undefined) return; + /* Only ever drop the placeholder this call made. A concurrent `pair` + * may have published a real identity over the name since, and removing + * that would destroy a credential whose code is already spent. */ + const current = await lstat(path, { bigint: true }).catch(() => undefined); + if ( + current === undefined || + current.ino !== reservedInode || + current.size !== 0n + ) { + return; + } + await rm(path, { force: true }); + }, + }; } export async function saveBridgeIdentity( @@ -231,8 +457,6 @@ export async function saveBridgeIdentity( try { const file = await open(temporaryPath, 'wx', 0o600); try { - /* Tighten every way available before judging, and judge before the key - * is written, so no private key reaches a world-readable path. */ await file.chmod(0o600); await assertOwnerOnlyPath(temporaryPath, path); await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); @@ -274,14 +498,15 @@ export async function saveWorkspaceMutationQuarantine( try { await file.chmod(0o600); await assertOwnerOnlyPath(path); + await assertWriteContainerPrivate(path); await file.writeFile(`${JSON.stringify(record, null, 2)}\n`, 'utf8'); await file.sync(); } finally { await file.close(); } } catch (error) { - /* A partial marker fails every later load, so undoing it has to reach the - * disk as durably as the write it is undoing. */ + /* A partially written marker would fail every later load, so the removal + * has to reach the disk as durably as the write it is undoing. */ await rm(path, { force: true }); try { await syncParentDirectory(path); @@ -296,6 +521,8 @@ export async function saveWorkspaceMutationQuarantine( export async function loadWorkspaceMutationQuarantine( path: string, ): Promise { + /* A marker another account can rewrite is not a control: it could be cleared + * to resume mutations, or forged to wedge the worker under a foreign owner. */ let content: string; try { content = await readGuardedFile( @@ -305,10 +532,9 @@ export async function loadWorkspaceMutationQuarantine( 'Another local account could clear or forge it. Keep worker state on a ' + 'native Linux filesystem.', ); + await assertReadPathPrivate(path); } catch (error) { - if (isMissingPathError(error)) { - return undefined; - } + if (isMissingPathError(error)) return undefined; throw error; } let record: unknown; @@ -369,6 +595,8 @@ export async function loadBridgeIdentity( 'Treat its private key as compromised: revoke the worker and pair again with an ' + 'identity path on a native Linux filesystem.', ); + /* After the file's own verdict, so an exposed mode keeps its diagnosis. */ + await assertReadPathPrivate(path); const identity = JSON.parse(content) as unknown; if (!isPairedIdentity(identity)) { throw new BridgeProtocolError(`Invalid bridge identity file: ${path}`); From 3968527680497246b689fcfa3f0d6256ee91a76d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 4 Sep 2026 17:16:55 -0400 Subject: [PATCH 039/116] =?UTF-8?q?=F0=9F=93=A1=20feat:=20Report=20BYOM=20?= =?UTF-8?q?Worker=20Readiness=20(#114)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/code/src/protocol.ts | 11 +++++ service/src/bridge/router.test.ts | 73 +++++++++++++++++++++++++++++++ service/src/bridge/router.ts | 17 +++++++ service/src/bridge/store.test.ts | 28 ++++++++++++ service/src/bridge/store.ts | 66 ++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 3bbd83cb..37288d63 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -433,6 +433,17 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; } +/** Administrator-visible liveness for a configured worker. Credentials, + * bindings, host paths, and worker identity material are deliberately omitted. */ +export interface BridgeWorkerStatusResponse { + protocolVersion: BridgeProtocolVersion; + workerId: string; + online: boolean; + ready: boolean; + leaseExpiresInMs?: number; + capabilities?: BridgeWorkerCapabilities; +} + export interface BridgePairingRedemption { protocolVersion: BridgeProtocolVersion; workerId: string; diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 11b0eb31..764613a5 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -25,6 +25,79 @@ afterEach(async () => { }); describe('paired bridge HTTP API', () => { + test('reports authenticated worker readiness without exposing identity or binding data', async () => { + const store = new RedisBridgeStore(redis); + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'paired', + adminToken: 'strong-administrator-bootstrap-token', + allowDynamicWorkers: true, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + const baseUrl = `http://127.0.0.1:${address.port}/v1/bridge`; + const headers = { Authorization: 'Bearer strong-administrator-bootstrap-token' }; + + const offline = await fetch(`${baseUrl}/workers/user-vm/status`, { headers }); + expect(offline.status).toBe(200); + await expect(offline.json()).resolves.toEqual({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'user-vm', + online: false, + ready: false, + }); + + const capabilities = { + statefulWorkspace: true, + sandboxProfile: 'native-srt', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }; + const registrationGeneration = await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'user-vm', + incarnationId: 'incarnation-00000001', + capabilities, + binding: { + tenantId: 'tenant-1', + principal: { type: 'user', id: 'user-1' }, + }, + credentialId: 'secret-credential-id', + identityId: 'secret-identity-id', + }); + + const starting = await fetch(`${baseUrl}/workers/user-vm/status`, { headers }); + expect(starting.status).toBe(200); + await expect(starting.json()).resolves.toMatchObject({ + workerId: 'user-vm', + online: true, + ready: false, + capabilities, + }); + expect( + JSON.stringify(await (await fetch(`${baseUrl}/workers/user-vm/status`, { headers })).json()), + ).not.toMatch(/tenant-1|user-1|secret-credential-id|secret-identity-id/); + + await store.confirmReady('user-vm', 'incarnation-00000001', registrationGeneration); + const ready = await fetch(`${baseUrl}/workers/user-vm/status`, { headers }); + const status = (await ready.json()) as Record; + expect(status).toMatchObject({ workerId: 'user-vm', online: true, ready: true, capabilities }); + expect(status.leaseExpiresInMs).toBeNumber(); + + const unauthorized = await fetch(`${baseUrl}/workers/user-vm/status`); + expect(unauthorized.status).toBe(401); + }); + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 7fa87b93..f428bdbe 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -312,6 +312,23 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { }), ); + router.get( + '/workers/:workerId/status', + adminAuth, + asyncRoute(async (req, res) => { + const workerId = req.params.workerId; + if (!validWorkerId(workerId) || !configuredWorker(workerId)) { + res.status(400).json({ error: 'Invalid bridge worker ID' }); + return; + } + const status = await options.store.workerStatus(workerId); + res.json({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + ...status, + }); + }), + ); router.post( '/workers/register', diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts index e08273f9..9d271551 100644 --- a/service/src/bridge/store.test.ts +++ b/service/src/bridge/store.test.ts @@ -27,6 +27,34 @@ afterEach(async () => { }); describe('RedisBridgeStore', () => { + test('reports an atomic, capability-limited worker status snapshot', async () => { + const store = new RedisBridgeStore(redis); + const capabilities = { + statefulWorkspace: true, + sandboxProfile: 'native-srt', + runtimes: ['bash'], + requiresReadyConfirmation: true, + }; + expect(await store.workerStatus('vm-status')).toEqual({ online: false, ready: false }); + + const generation = await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-status', + incarnationId: 'incarnation-status-01', + capabilities, + }); + expect(await store.workerStatus('vm-status')).toMatchObject({ + online: true, + ready: false, + capabilities, + }); + + await store.confirmReady('vm-status', 'incarnation-status-01', generation); + const status = await store.workerStatus('vm-status'); + expect(status).toMatchObject({ online: true, ready: true, capabilities }); + expect(status.leaseExpiresInMs).toBeGreaterThan(0); + }); + test('rejects a registration whose authenticated identity was replaced', async () => { await redis.set( 'codeapi:bridge:v1:identity:fenced-worker', diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 03694b1b..da3956d7 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -12,6 +12,8 @@ import type { import { BRIDGE_PROTOCOL_VERSION, + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, isWorkspaceToolRequest, isWorkspaceToolResult, } from '../../../packages/code/src/protocol'; @@ -68,6 +70,13 @@ export interface RegisteredBridgeWorker extends BridgeWorkerRegistration { identityId?: string; } +export interface BridgeWorkerStatus { + online: boolean; + ready: boolean; + leaseExpiresInMs?: number; + capabilities?: BridgeWorkerRegistration['capabilities']; +} + function supportsWorkspaceTool( registration: RegisteredBridgeWorker, request: WorkspaceToolRequest, @@ -309,6 +318,63 @@ export class RedisBridgeStore { ); } + /** Returns only the worker's ephemeral registration state. The registration + * is the heartbeat: when its TTL expires the worker is offline. */ + async workerStatus(workerId: string): Promise { + const snapshot = (await boundedCommand( + this.redis.eval( + [ + "local registration = redis.call('GET', KEYS[1])", + "if not registration then return { false, false, false, -2 } end", + 'return {', + ' registration,', + " redis.call('GET', KEYS[2]) or false,", + " redis.call('GET', KEYS[3]) or false,", + " redis.call('PTTL', KEYS[1])", + '}', + ].join('\n'), + 3, + workerKey(workerId), + workerReadyKey(workerId), + workerRegistrationGenerationKey(workerId), + ), + this.redisCommandTimeoutMs, + 'Bridge worker status', + )) as [string | null, string | null, string | null, number]; + const [rawRegistration, readyToken, registrationGeneration, leaseExpiresInMs] = snapshot; + if (rawRegistration == null || rawRegistration === '' || leaseExpiresInMs <= 0) { + return { online: false, ready: false }; + } + + let registration: RegisteredBridgeWorker; + try { + registration = JSON.parse(rawRegistration) as RegisteredBridgeWorker; + } catch { + return { online: false, ready: false }; + } + if ( + registration.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + !isValidBridgeWorkerId(registration.workerId) || + registration.workerId !== workerId || + typeof registration.incarnationId !== 'string' || + !isValidBridgeWorkerCapabilities(registration.capabilities) + ) { + return { online: false, ready: false }; + } + + const requiresConfirmation = registration.capabilities.requiresReadyConfirmation === true; + const ready = + !requiresConfirmation || + (registrationGeneration != null && + readyToken === workerReadyToken(registration.incarnationId, Number(registrationGeneration))); + return { + online: true, + ready, + leaseExpiresInMs, + capabilities: registration.capabilities, + }; + } + async register( registration: RegisteredBridgeWorker, authorization?: string | { From 6043181bae865aaab1851bc194bc450de9acf502 Mon Sep 17 00:00:00 2001 From: SSIG-IT Date: Sat, 5 Sep 2026 04:49:47 +0200 Subject: [PATCH 040/116] =?UTF-8?q?=F0=9F=9A=A2=20fix:=20Pack=20Code=20Pro?= =?UTF-8?q?tocol=20Into=20Egress=20Gateway=20Image=20(#108)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The egress-gateway container crash-loops on startup: error: Cannot find module '../../packages/code/src/protocol' from '/app/src/secure-startup.ts' `service/src/egress-gateway.ts` imports `./secure-startup`, and `service/src/secure-startup.ts` imports `../../packages/code/src/protocol`. `service/Dockerfile.egress-gateway` copies `service/src` and `shared` but never copies `packages/code/src`, so from `/app/src/…` the relative import resolves to `/packages/code/src/protocol`, which is absent from the image. The other service images (`Dockerfile.node`, `Dockerfile.worker`) already `COPY packages/code/src /packages/code/src` next to `COPY shared /shared`; this adds the same line to both the production and development stages of the egress-gateway image, matching that convention. Tested by building `service/Dockerfile.egress-gateway` and confirming the container reaches its `/health` endpoint on 3190 instead of exiting. Co-authored-by: Paul <200737214+SSIG-IT@users.noreply.github.com> --- service/Dockerfile.egress-gateway | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/Dockerfile.egress-gateway b/service/Dockerfile.egress-gateway index bd431ab0..ca9e9a9d 100644 --- a/service/Dockerfile.egress-gateway +++ b/service/Dockerfile.egress-gateway @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf COPY --from=install /temp/prod/node_modules ./node_modules COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src EXPOSE 3190 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD curl -f http://localhost:3190/health || exit 1 @@ -28,5 +29,6 @@ ENV NODE_ENV=development COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY shared /shared +COPY packages/code/src /packages/code/src EXPOSE 3190 9230 CMD ["bun", "run", "--watch", "src/egress-gateway.ts"] From 8b6b2de333eb089a36ae3654cfe26d406d6f3bf4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 5 Sep 2026 23:55:43 -0400 Subject: [PATCH 041/116] =?UTF-8?q?=F0=9F=94=90=20feat:=20Add=20Sandboxed?= =?UTF-8?q?=20GitHub=20Authentication=20(#115)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add sandboxed GitHub authentication * fix: Harden GitHub credential startup * fix: Guard GitHub App key loading * fix: Serialize credential masking * fix: Preserve Git LFS policy * fix: Inject Git LFS config into SRT * fix: Override ambient Git filter config * fix: Preserve composed Git sandbox config * fix: Compose trusted Git environment * fix: Sanitize composed Git config slots * fix: Bind credential policy identity --- packages/code/README.md | 41 ++++ packages/code/package.json | 4 + packages/code/src/cli.test.ts | 114 +++++++++- packages/code/src/cli.ts | 235 +++++++++++++++++--- packages/code/src/github.test.ts | 231 +++++++++++++++++++ packages/code/src/github.ts | 268 +++++++++++++++++++++++ packages/code/src/index.ts | 1 + packages/code/src/native-sandbox.test.ts | 238 +++++++++++++++++++- packages/code/src/native-sandbox.ts | 139 ++++++++++-- 9 files changed, 1227 insertions(+), 44 deletions(-) create mode 100644 packages/code/src/github.test.ts create mode 100644 packages/code/src/github.ts diff --git a/packages/code/README.md b/packages/code/README.md index f53741a8..54284c56 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -86,6 +86,47 @@ policy: an allowed destination can receive workspace data. The normalized allowlist is included in the worker policy digest. Tool approval hooks remain the user-facing allow/deny boundary for each invocation. +### GitHub authentication + +The native BYOM worker can provide Git HTTPS authentication without exposing a +real token to the command sandbox. Prefer a GitHub App installed only on the +repositories the agent may access: + +```bash +LIBRECHAT_CODE_GITHUB_APP_ID=12345 \ +LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 \ +LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/secure/librechat-agent.pem \ +librechat-code run --worker-dir /path/to/project --allow-workspace-commands +``` + +The private key must be an owner-only regular file outside the workspace. It is +read only by the trusted worker, which mints and refreshes short-lived +installation tokens. A personal access token is supported as a fallback with +`LIBRECHAT_CODE_GITHUB_TOKEN`, but the GitHub App is the safer default because +its repository access and permissions can be narrowly installed and revoked. +Native Windows currently requires token mode because the worker cannot +reliably validate private-key ACLs there; use WSL2 for GitHub App mode. + +Git receives authentication through process-scoped `GIT_CONFIG_*` variables. +The same isolated config supplies the standard Git LFS filters; hosts using LFS +must install `git-lfs`, and checkout fails instead of silently leaving pointer +files when it is unavailable. +SRT replaces only the bearer-token portion with a sentinel inside the sandbox +and substitutes the real value in its host proxy only for `github.com` HTTPS +traffic. TLS termination is enabled for that substitution. The worker restores +the parent environment immediately after constructing the sandbox command; it +never writes credentials into the repository, a remote URL, or Git config. +GitHub's required domains are added to the command egress allowlist only when +authentication is configured. The worker identity, GitHub App key path, token +source variables, and mutation-quarantine record remain denied to sandboxed +commands. + +For GitHub Enterprise Server, set `LIBRECHAT_CODE_GITHUB_HOST` to its hostname +and `LIBRECHAT_CODE_GITHUB_API_URL` to its HTTPS API base URL. GitHub +authentication currently requires the `native-srt` command sandbox. Every +clone, commit, or push command still crosses LibreChat's tool-approval policy; +the credential boundary does not grant approval by itself. + Select the backend explicitly when desired: ```bash diff --git a/packages/code/package.json b/packages/code/package.json index cfb97ddd..c9f810b7 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -34,6 +34,10 @@ "./native-sandbox": { "types": "./dist/native-sandbox.d.ts", "import": "./dist/native-sandbox.js" + }, + "./github": { + "types": "./dist/github.d.ts", + "import": "./dist/github.js" } }, "bin": { diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 7bdb3d30..ba195dbc 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -94,6 +94,105 @@ test('CLI rejects an unknown command sandbox before entering the run loop', () = ); }); +test('CLI rejects incomplete GitHub App authentication before worker registration', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: undefined, + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /GitHub App authentication requires/); +}); + +test('CLI refuses GitHub credentials without native sandboxed commands', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_GITHUB_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /GitHub authentication requires workspace commands/, + ); +}); + +test('CLI rejects a GitHub App API URL that does not match its Git host', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_HOST: 'github.example.test', + LIBRECHAT_CODE_GITHUB_API_URL: 'https://other.example.test/api/v3', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /LIBRECHAT_CODE_GITHUB_HOST must match the GitHub App API hostname/, + ); +}); + +test('CLI validates GitHub App credentials before worker registration', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/exist/app.pem', + LIBRECHAT_CODE_GITHUB_HOST: undefined, + LIBRECHAT_CODE_GITHUB_API_URL: undefined, + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /ENOENT|no such file/i); + assert.doesNotMatch(result.stderr, /fetch failed/); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, @@ -132,7 +231,10 @@ test('CLI requires the macOS NsJail seccomp profile', () => { ); assert.notEqual(result.status, 0); - assert.match(result.stderr, /LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE is required/); + assert.match( + result.stderr, + /LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE is required/, + ); }); test('CLI requires a package mount for the macOS NsJail profile', () => { @@ -154,7 +256,10 @@ test('CLI requires a package mount for the macOS NsJail profile', () => { ); assert.notEqual(result.status, 0); - assert.match(result.stderr, /LIBRECHAT_CODE_DOCKER_PACKAGES_PATH is required/); + assert.match( + result.stderr, + /LIBRECHAT_CODE_DOCKER_PACKAGES_PATH is required/, + ); }); test('CLI reset does not require Docker runtime launch inputs', () => { @@ -176,6 +281,7 @@ test('CLI reset does not require Docker runtime launch inputs', () => { LIBRECHAT_CODE_RUNTIME_IMAGE: undefined, LIBRECHAT_CODE_DOCKER_SECCOMP_PROFILE: undefined, LIBRECHAT_CODE_DOCKER_PACKAGES_PATH: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', }, }, ); @@ -185,6 +291,10 @@ test('CLI reset does not require Docker runtime launch inputs', () => { result.stderr, /LIBRECHAT_CODE_(?:RUNTIME_IMAGE|DOCKER_SECCOMP_PROFILE|DOCKER_PACKAGES_PATH) is required/, ); + assert.doesNotMatch( + result.stderr, + /GitHub authentication requires workspace commands/, + ); }); test('CLI relay requires a fixed upstream URL', () => { diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index a13eb011..e9442df9 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -21,14 +21,25 @@ import { saveWorkspaceMutationQuarantine, } from './storage.js'; import { BridgeWorker } from './worker.js'; +import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; import { - LocalWorkspaceTools, - SandboxWorkspaceTools, -} from './workspace.js'; -import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor } from './runtime.js'; + DockerRuntimeSupervisor, + EndpointRuntimeSupervisor, +} from './runtime.js'; import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { + GITHUB_ALLOWED_DOMAINS, + GITHUB_CREDENTIAL_ENV_NAME, + GitHubAppCredentialProvider, + StaticGitHubCredentialProvider, + gitHubAuthenticationPolicyIdentity, + gitHubCredentialEnvironment, + normalizeGitHubHost, + wrapGitHubCredentialCommand, +} from './github.js'; import type { RuntimeSupervisor } from './runtime.js'; +import type { GitHubCredentialProvider } from './github.js'; import type { WorkspaceToolExecutor } from './workspace.js'; import { BRIDGE_WORKSPACE_NAME_MAX_LENGTH, @@ -74,7 +85,11 @@ function list(value: string | undefined): string[] { ); } -function positiveInteger(name: string, value: string | undefined, fallback: number): number { +function positiveInteger( + name: string, + value: string | undefined, + fallback: number, +): number { if (value == null || value.trim().length === 0) return fallback; const parsed = Number(value); if (!Number.isSafeInteger(parsed) || parsed <= 0) { @@ -103,14 +118,107 @@ const MACOS_NSJAIL_CAPABILITIES = [ function option(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index >= 0) return args[index + 1]; - return args.find((value) => value.startsWith(`${name}=`))?.slice(name.length + 1); + return args + .find((value) => value.startsWith(`${name}=`)) + ?.slice(name.length + 1); } function nonEmpty(value: string | undefined): string | undefined { return value?.trim().length ? value : undefined; } -function defaultWorkspaceName(workerDirectory: string, workspaceId: string): string { +function githubCredentials(): { + provider?: GitHubCredentialProvider; + host: string; + privateKeyPath?: string; + mode?: 'app' | 'token'; + policyIdentity: string; +} { + const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); + const appId = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_APP_ID); + const installationId = nonEmpty( + process.env.LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, + ); + const privateKeyPath = nonEmpty( + process.env.LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE, + ); + const appValues = [appId, installationId, privateKeyPath]; + const hasApp = appValues.some(Boolean); + if (hasApp && !appValues.every(Boolean)) { + throw new Error( + 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE', + ); + } + if (hasApp && token) { + throw new Error( + 'Configure either GitHub App authentication or a GitHub token, not both', + ); + } + const configuredHostValue = nonEmpty( + process.env.LIBRECHAT_CODE_GITHUB_HOST, + ); + const configuredHost = configuredHostValue + ? normalizeGitHubHost(configuredHostValue) + : undefined; + const apiUrl = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_API_URL); + let apiHost: string | undefined; + if (apiUrl) { + let parsedApiUrl: URL; + try { + parsedApiUrl = new URL(apiUrl); + } catch { + throw new Error('LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL'); + } + apiHost = + parsedApiUrl.hostname.toLowerCase() === 'api.github.com' + ? 'github.com' + : parsedApiUrl.hostname.toLowerCase(); + } + if (configuredHost && apiHost && configuredHost.toLowerCase() !== apiHost) { + throw new Error( + 'LIBRECHAT_CODE_GITHUB_HOST must match the GitHub App API hostname', + ); + } + const host = normalizeGitHubHost(configuredHost ?? apiHost ?? 'github.com'); + if (hasApp) { + return { + host, + mode: 'app', + policyIdentity: gitHubAuthenticationPolicyIdentity({ + mode: 'app', + host, + appId, + installationId, + }), + privateKeyPath, + provider: new GitHubAppCredentialProvider({ + appId: appId!, + installationId: installationId!, + privateKeyPath: privateKeyPath!, + apiUrl, + }), + }; + } + return { + host, + policyIdentity: gitHubAuthenticationPolicyIdentity({ + mode: token ? 'token' : undefined, + host, + token, + }), + ...(token + ? { + provider: new StaticGitHubCredentialProvider(token), + mode: 'token' as const, + } + : {}), + }; +} + +function defaultWorkspaceName( + workerDirectory: string, + workspaceId: string, +): string { const directoryName = basename(resolve(workerDirectory)); return directoryName.trim().length > 0 && directoryName.length <= BRIDGE_WORKSPACE_NAME_MAX_LENGTH @@ -168,7 +276,9 @@ async function relay(): Promise { 8, ), }); - process.stdout.write(`librechat-code: file relay listening at ${handle.url}\n`); + process.stdout.write( + `librechat-code: file relay listening at ${handle.url}\n`, + ); await new Promise((resolve) => { process.once('SIGINT', resolve); process.once('SIGTERM', resolve); @@ -176,9 +286,13 @@ async function relay(): Promise { await handle.close(); } -async function run(runtimeSessionId?: string, args: string[] = []): Promise { +async function run( + runtimeSessionId?: string, + args: string[] = [], +): Promise { const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); - const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); + const configuredIdentityPath = + process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); const identityPath = configuredIdentityPath ?? @@ -211,7 +325,8 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise path != null, - ), + protectedPaths: [ + identityPath, + mutationQuarantinePath, + github.privateKeyPath, + ].filter((path): path is string => path != null), allowedDomains: commandAllowedDomains, + ...(github.provider + ? { + maskedEnvironment: { + variables: [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: [github.host], + }, + ], + async resolve(signal?: AbortSignal) { + return gitHubCredentialEnvironment( + await github.provider!.getCredential(signal), + ); + }, + wrapCommand(command: string, platform: NodeJS.Platform) { + return wrapGitHubCredentialCommand( + command, + github.host, + platform, + ); + }, + }, + } + : {}), }) : undefined; if (allowWorkspaceCommands && workspaceTools) { @@ -538,7 +720,7 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise undefined); @@ -659,7 +842,8 @@ async function run(runtimeSessionId?: string, args: string[] = []): Promise { const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); - const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); + const configuredIdentityPath = + process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); const identityPath = configuredIdentityPath ?? @@ -682,7 +866,8 @@ async function clearMutationQuarantine(args: string[]): Promise { process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; const explicitWorkerDirectory = nonEmpty( - option(args, '--worker-dir') ?? process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), + option(args, '--worker-dir') ?? + process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), ); const useDefaultWorkspace = args.includes('--default-workspace') || diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts new file mode 100644 index 00000000..8006f2fe --- /dev/null +++ b/packages/code/src/github.test.ts @@ -0,0 +1,231 @@ +import { generateKeyPairSync } from 'node:crypto'; +import { + chmod, + mkdtemp, + mkdir, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + GITHUB_ALLOWED_DOMAINS, + GitHubAppCredentialProvider, + StaticGitHubCredentialProvider, + gitHubAuthenticationPolicyIdentity, + GITHUB_CREDENTIAL_ENV_NAME, + gitHubCredentialEnvironment, + normalizeGitHubHost, + wrapGitHubCredentialCommand, +} from './github.js'; + +test('normalizes GitHub DNS hostnames before policy and allowlist use', () => { + assert.equal(normalizeGitHubHost('GitHub.COM'), 'github.com'); + assert.throws(() => normalizeGitHubHost('.github.com'), /DNS hostname/); +}); + +test('binds the GitHub App installation to the public policy identity', () => { + assert.equal( + gitHubAuthenticationPolicyIdentity({ + mode: 'app', + host: 'github.com', + appId: '123', + installationId: '456', + }), + 'github-auth:app:github.com:app:123:installation:456', + ); + assert.notEqual( + gitHubAuthenticationPolicyIdentity({ + mode: 'app', + host: 'github.com', + appId: '123', + installationId: '456', + }), + gitHubAuthenticationPolicyIdentity({ + mode: 'app', + host: 'github.com', + appId: '123', + installationId: '789', + }), + ); +}); + +test('binds token credentials to policy identity without exposing the token', () => { + const firstToken = 'github_pat_abcdefghijklmnopqrstuvwxyz'; + const secondToken = 'github_pat_zyxwvutsrqponmlkjihgfedcba'; + const first = gitHubAuthenticationPolicyIdentity({ + mode: 'token', + host: 'github.com', + token: firstToken, + }); + const second = gitHubAuthenticationPolicyIdentity({ + mode: 'token', + host: 'github.com', + token: secondToken, + }); + + assert.notEqual(first, second); + assert.ok(!first.includes(firstToken)); +}); + +test('rejects GitHub App authentication where key ACLs cannot be validated', () => { + assert.throws( + () => + new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath: 'C:\\secure\\app.pem', + platform: 'win32', + }), + /private key ACLs cannot be validated securely/, + ); +}); + +test('mints and caches a short-lived GitHub App installation token', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + await chmod(privateKeyPath, 0o600); + let calls = 0; + const request = async ( + _input: string | URL | Request, + init?: RequestInit, + ) => { + calls += 1; + assert.match( + String(new Headers(init?.headers).get('authorization')), + /^Bearer eyJ/, + ); + return new Response( + JSON.stringify({ + token: 'ghs_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }), + { status: 201 }, + ); + }; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + fetch: request as typeof fetch, + now: () => new Date('2030-01-01T00:00:00Z'), + }); + + assert.equal( + (await provider.getCredential()).value, + 'ghs_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal( + (await provider.getCredential()).value, + 'ghs_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal(calls, 1); +}); + +test('builds process-scoped Git HTTPS authorization without embedding credentials in URLs', async () => { + const provider = new StaticGitHubCredentialProvider( + 'github_pat_abcdefghijklmnopqrstuvwxyz', + ); + assert.deepEqual( + gitHubCredentialEnvironment(await provider.getCredential()), + { + [GITHUB_CREDENTIAL_ENV_NAME]: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }, + ); +}); + +test('composes the masked credential with SRT Git configuration inside the sandbox', () => { + const wrapped = wrapGitHubCredentialCommand( + 'git push', + 'github.com', + 'darwin', + ); + assert.match(wrapped, /http\.proxyAuthMethod=basic/); + assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); + assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); + assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); + assert.equal(wrapped.match(/Authorization: Bearer/g)?.length, 1); + assert.ok(!wrapped.includes('github_pat_')); +}); + +test('rejects an insecure GitHub App API endpoint before reading the private key', () => { + assert.throws( + () => + new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath: '/does/not/matter', + apiUrl: 'http://github.example.test/api/v3', + }), + /must be an HTTPS URL/, + ); +}); + +test('rejects a GitHub App key in a shared writable directory', async (t) => { + if (process.platform === 'win32') { + t.skip('POSIX directory permissions are unavailable on Windows'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-github-')); + t.after(() => rm(root, { recursive: true, force: true })); + const directory = join(root, 'shared'); + await mkdir(directory, { mode: 0o700 }); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + await chmod(directory, 0o777); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + }); + + await assert.rejects( + provider.getCredential(), + /private key directory must not be writable/, + ); +}); + +test('rejects a symlinked GitHub App key without reopening its target', async (t) => { + if (process.platform === 'win32') { + t.skip('O_NOFOLLOW is unavailable on Windows'); + return; + } + const root = await mkdtemp(join(tmpdir(), 'librechat-code-github-')); + t.after(() => rm(root, { recursive: true, force: true })); + const target = join(root, 'target.pem'); + const link = join(root, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(target, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + await symlink(target, link); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath: link, + }); + + await assert.rejects(provider.getCredential(), /ELOOP|symbolic link/i); +}); + +test('allows the GitHub LFS object delivery hosts', () => { + assert.ok(GITHUB_ALLOWED_DOMAINS.includes('objects.githubusercontent.com')); + assert.ok(GITHUB_ALLOWED_DOMAINS.includes('*.githubusercontent.com')); + assert.ok(GITHUB_ALLOWED_DOMAINS.includes('github-cloud.s3.amazonaws.com')); +}); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts new file mode 100644 index 00000000..9d265e58 --- /dev/null +++ b/packages/code/src/github.ts @@ -0,0 +1,268 @@ +import { constants } from 'node:fs'; +import { createHash, createPrivateKey, sign } from 'node:crypto'; +import { open, realpath, stat } from 'node:fs/promises'; +import { dirname } from 'node:path'; + +export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; +export const GITHUB_ALLOWED_DOMAINS = [ + 'github.com', + '*.github.com', + 'api.github.com', + 'lfs.github.com', + 'objects.githubusercontent.com', + '*.githubusercontent.com', + 'github-cloud.s3.amazonaws.com', +] as const; + +export interface GitHubCredential { + value: string; + expiresAt?: Date; +} + +export interface GitHubCredentialProvider { + getCredential(signal?: AbortSignal): Promise; +} + +export interface GitHubAppCredentialProviderOptions { + appId: string; + installationId: string; + privateKeyPath: string; + apiUrl?: string; + fetch?: typeof globalThis.fetch; + now?: () => Date; + platform?: NodeJS.Platform; +} + +function base64UrlJson(value: unknown): string { + return Buffer.from(JSON.stringify(value)).toString('base64url'); +} + +function assertPositiveIdentifier(name: string, value: string): void { + if (!/^[1-9][0-9]*$/.test(value)) { + throw new Error(`${name} must be a positive decimal identifier`); + } +} + +async function readPrivateKey(path: string): Promise { + if (process.platform !== 'win32') { + const directory = await stat(await realpath(dirname(path))); + const uid = process.getuid?.(); + if (uid !== undefined && directory.uid !== uid && directory.uid !== 0) { + throw new Error( + 'GitHub App private key directory must be owned by this user or root', + ); + } + const mode = directory.mode & 0o7777; + const protectedByStickyBit = + (mode & 0o1000) !== 0 && (directory.uid === uid || directory.uid === 0); + if ((mode & 0o022) !== 0 && !protectedByStickyBit) { + throw new Error( + 'GitHub App private key directory must not be writable by group or other users', + ); + } + } + + const handle = await open( + path, + constants.O_RDONLY | + (process.platform === 'win32' ? 0 : constants.O_NOFOLLOW), + ); + try { + const metadata = await handle.stat(); + if (!metadata.isFile()) { + throw new Error('GitHub App private key must be a regular file'); + } + const uid = process.getuid?.(); + if (uid !== undefined && metadata.uid !== uid && metadata.uid !== 0) { + throw new Error( + 'GitHub App private key must be owned by this user or root', + ); + } + if (process.platform !== 'win32' && (metadata.mode & 0o077) !== 0) { + throw new Error( + 'GitHub App private key must not be accessible by group or other users', + ); + } + return await handle.readFile('utf8'); + } finally { + await handle.close(); + } +} + +function createAppJwt(appId: string, privateKey: string, now: Date): string { + const issuedAt = Math.floor(now.getTime() / 1000) - 60; + const header = base64UrlJson({ alg: 'RS256', typ: 'JWT' }); + const payload = base64UrlJson({ + iss: appId, + iat: issuedAt, + exp: issuedAt + 600, + }); + const unsigned = `${header}.${payload}`; + const signature = sign( + 'RSA-SHA256', + Buffer.from(unsigned), + createPrivateKey(privateKey), + ); + return `${unsigned}.${signature.toString('base64url')}`; +} + +export class GitHubAppCredentialProvider implements GitHubCredentialProvider { + private cached?: GitHubCredential; + + constructor(private readonly options: GitHubAppCredentialProviderOptions) { + if ((options.platform ?? process.platform) === 'win32') { + throw new Error( + 'GitHub App authentication is unavailable on native Windows because private key ACLs cannot be validated securely; use a token or WSL2', + ); + } + assertPositiveIdentifier('GitHub App ID', options.appId); + assertPositiveIdentifier( + 'GitHub App installation ID', + options.installationId, + ); + if (options.apiUrl != null) { + const apiUrl = new URL(options.apiUrl); + if (apiUrl.protocol !== 'https:' || apiUrl.username || apiUrl.password) { + throw new Error( + 'GitHub API URL must be an HTTPS URL without credentials', + ); + } + } + } + + async getCredential(signal?: AbortSignal): Promise { + const now = (this.options.now ?? (() => new Date()))(); + if ( + this.cached?.expiresAt != null && + this.cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 + ) { + return this.cached; + } + const privateKey = await readPrivateKey(this.options.privateKeyPath); + const jwt = createAppJwt(this.options.appId, privateKey, now); + const apiUrl = (this.options.apiUrl ?? 'https://api.github.com').replace( + /\/+$/, + '', + ); + const request = this.options.fetch ?? globalThis.fetch; + const response = await request( + `${apiUrl}/app/installations/${this.options.installationId}/access_tokens`, + { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${jwt}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + signal, + }, + ); + if (!response.ok) { + throw new Error( + `GitHub App token request failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { + token?: unknown; + expires_at?: unknown; + }; + if ( + typeof body.token !== 'string' || + body.token.length < 20 || + typeof body.expires_at !== 'string' + ) { + throw new Error('GitHub App token response is invalid'); + } + const expiresAt = new Date(body.expires_at); + if ( + !Number.isFinite(expiresAt.getTime()) || + expiresAt.getTime() <= now.getTime() + ) { + throw new Error('GitHub App token expiry is invalid'); + } + this.cached = { value: body.token, expiresAt }; + return this.cached; + } +} + +export class StaticGitHubCredentialProvider implements GitHubCredentialProvider { + constructor(private readonly token: string) { + if (token.trim().length < 20 || /[\0\r\n]/.test(token)) { + throw new Error('GitHub token is invalid'); + } + } + + async getCredential(): Promise { + return { value: this.token }; + } +} + +export function gitHubCredentialEnvironment( + credential: GitHubCredential, +): Record { + return { + [GITHUB_CREDENTIAL_ENV_NAME]: credential.value, + }; +} + +export function gitHubAuthenticationPolicyIdentity(options: { + mode?: 'app' | 'token'; + host: string; + appId?: string; + installationId?: string; + token?: string; +}): string { + const identity = `github-auth:${options.mode ?? 'none'}:${options.host}`; + if (options.mode === 'token') { + if (!options.token) { + throw new Error('GitHub token policy identity requires a token'); + } + const fingerprint = createHash('sha256') + .update('librechat-code-github-token-v1\0') + .update(options.token) + .digest('hex'); + return `${identity}:fingerprint:${fingerprint}`; + } + if (options.mode !== 'app') return identity; + if (!options.appId || !options.installationId) { + throw new Error( + 'GitHub App policy identity requires an App and installation ID', + ); + } + return `${identity}:app:${options.appId}:installation:${options.installationId}`; +} + +export function normalizeGitHubHost(value: string): string { + const host = value.toLowerCase(); + if ( + !/^[a-z0-9.-]+$/.test(host) || + host.startsWith('.') || + host.endsWith('.') + ) { + throw new Error('LIBRECHAT_CODE_GITHUB_HOST must be a DNS hostname'); + } + return host; +} + +export function wrapGitHubCredentialCommand( + command: string, + host = 'github.com', + platform: NodeJS.Platform = process.platform, +): string { + const key = `http.https://${host}/.extraheader`; + if (platform === 'win32') { + return [ + 'set "GIT_CONFIG_GLOBAL=NUL"', + 'set "GIT_CONFIG_NOSYSTEM=1"', + `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Bearer %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, + command, + ].join(' && '); + } + return [ + 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', + `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Bearer \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, + command, + ].join(';\n'); +} diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 2e94a55c..74ed37d4 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -6,4 +6,5 @@ export * from './runtime.js'; export * from './workspace.js'; export * from './workspace-runtime.js'; export * from './native-sandbox.js'; +export * from './github.js'; export * from './worker.js'; diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 8cc7c6cf..50f8e5c3 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -28,9 +28,18 @@ const request = { maxOutputBytes: 64, }; -function fakeManager(options: { dependencyErrors?: string[] } = {}) { +function fakeManager( + options: { + dependencyErrors?: string[]; + beforeWrap?: () => Promise; + appendGitSafeDirectory?: boolean; + inheritedGitEnvironment?: Record; + } = {}, +) { let config: SandboxRuntimeConfig | undefined; let reset = false; + let credentialSeenDuringWrap: string | undefined; + let gitLfsRequiredSeenDuringWrap: string | undefined; const manager = { isSupportedPlatform: () => true, async checkDependenciesAsync() { @@ -40,9 +49,36 @@ function fakeManager(options: { dependencyErrors?: string[] } = {}) { config = value; }, async wrapWithSandboxArgv(command: string) { + await options.beforeWrap?.(); + credentialSeenDuringWrap = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + gitLfsRequiredSeenDuringWrap = process.env.GIT_CONFIG_VALUE_3; + const ambientGitEnvironment = Object.fromEntries( + Object.entries(process.env).filter( + ([name, value]) => name.startsWith('GIT_CONFIG_') && value != null, + ), + ); + let gitEnvironment = ambientGitEnvironment; + if (options.appendGitSafeDirectory) { + const index = Number(ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0'); + gitEnvironment = { + ...(options.inheritedGitEnvironment ?? {}), + GIT_CONFIG_COUNT: String(index + 1), + [`GIT_CONFIG_KEY_${index}`]: 'safe.directory', + [`GIT_CONFIG_VALUE_${index}`]: '/workspace', + }; + } return { argv: ['/bin/bash', '-c', command], - env: { PATH: process.env.PATH }, + env: { + PATH: process.env.PATH, + ...gitEnvironment, + ...(credentialSeenDuringWrap + ? { + LIBRECHAT_CODE_TEST_CREDENTIAL: + 'Authorization: Bearer srt-sentinel', + } + : {}), + }, }; }, annotateStderrWithSandboxFailures(_commandId: string, stderr: string) { @@ -61,6 +97,12 @@ function fakeManager(options: { dependencyErrors?: string[] } = {}) { get reset() { return reset; }, + get credentialSeenDuringWrap() { + return credentialSeenDuringWrap; + }, + get gitLfsRequiredSeenDuringWrap() { + return gitLfsRequiredSeenDuringWrap; + }, }; } @@ -106,6 +148,183 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.equal(fake.reset, true); }); +test('masks a host credential for only its injection host and restores the parent environment', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const original = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + delete process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + t.after(() => { + if (original === undefined) + delete process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + else process.env.LIBRECHAT_CODE_TEST_CREDENTIAL = original; + }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + allowedDomains: ['github.com'], + maskedEnvironment: { + variables: [ + { + name: 'LIBRECHAT_CODE_TEST_CREDENTIAL', + extract: '^Authorization: Bearer (.+)$', + injectHosts: ['github.com'], + }, + ], + async resolve() { + return { + LIBRECHAT_CODE_TEST_CREDENTIAL: 'Authorization: Bearer real-secret', + }; + }, + }, + manager: fake.manager, + }); + + const result = await sandbox.execute({ + ...request, + command: 'printf %s "$LIBRECHAT_CODE_TEST_CREDENTIAL"', + }); + + assert.equal( + fake.credentialSeenDuringWrap, + 'Authorization: Bearer real-secret', + ); + assert.equal(result.stdout, 'Authorization: Bearer srt-sentinel'); + assert.equal(process.env.LIBRECHAT_CODE_TEST_CREDENTIAL, undefined); + assert.deepEqual(fake.config?.network.tlsTerminate, {}); + assert.deepEqual(fake.config?.credentials?.envVars?.at(-1), { + name: 'LIBRECHAT_CODE_TEST_CREDENTIAL', + extract: '^Authorization: Bearer (.+)$', + injectHosts: ['github.com'], + mode: 'mask', + onExtractNoMatch: 'error', + }); +}); + +test('serializes credential handoff across concurrent sandbox instances', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + delete process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + t.after(() => { + if (original === undefined) + delete process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + else process.env.LIBRECHAT_CODE_TEST_CREDENTIAL = original; + }); + let firstEntered!: () => void; + const firstEnteredPromise = new Promise((resolve) => { + firstEntered = resolve; + }); + let releaseFirst!: () => void; + const firstGate = new Promise((resolve) => { + releaseFirst = resolve; + }); + let secondEntered = false; + const first = fakeManager({ + async beforeWrap() { + firstEntered(); + await firstGate; + }, + }); + const second = fakeManager({ + async beforeWrap() { + secondEntered = true; + }, + }); + const sandbox = ( + manager: ReturnType['manager'], + value: string, + ) => + new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + allowedDomains: ['github.com'], + maskedEnvironment: { + variables: [ + { + name: 'LIBRECHAT_CODE_TEST_CREDENTIAL', + injectHosts: ['github.com'], + }, + ], + async resolve() { + return { LIBRECHAT_CODE_TEST_CREDENTIAL: value }; + }, + }, + manager, + }); + + const firstExecution = sandbox(first.manager, 'first-secret').execute( + request, + ); + await firstEnteredPromise; + const secondExecution = sandbox(second.manager, 'second-secret').execute( + request, + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(secondEntered, false); + releaseFirst(); + await firstExecution; + await secondExecution; + + assert.equal(first.credentialSeenDuringWrap, 'first-secret'); + assert.equal(second.credentialSeenDuringWrap, 'second-secret'); + assert.equal(process.env.LIBRECHAT_CODE_TEST_CREDENTIAL, undefined); +}); + +test('isolates Git from host-level global and system configuration', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + const result = await sandbox.execute({ + ...request, + command: 'printf "%s|%s" "$GIT_CONFIG_GLOBAL" "$GIT_CONFIG_NOSYSTEM"', + }); + + assert.equal(result.stdout, '/dev/null|1'); +}); + +test('restores trusted Git LFS filters without reading host Git configuration', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager({ + appendGitSafeDirectory: true, + inheritedGitEnvironment: { + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'include.path', + GIT_CONFIG_VALUE_0: '/untrusted/host-config', + }, + }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { + ...process.env, + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'include.path', + GIT_CONFIG_VALUE_0: '/untrusted/host-config', + }, + manager: fake.manager, + }); + + const result = await sandbox.execute({ + ...request, + maxOutputBytes: 256, + command: + 'printf "%s|%s|%s|%s|%s|%s" "$(git config --get filter.lfs.clean)" "$(git config --get filter.lfs.smudge)" "$(git config --get filter.lfs.process)" "$(git config --get filter.lfs.required)" "$(git config --get safe.directory)" "$(git config --get include.path)"', + }); + + assert.equal( + result.stdout, + 'git-lfs clean -- %f|git-lfs smudge -- %f|git-lfs filter-process|true|/workspace|', + ); + assert.equal(fake.gitLfsRequiredSeenDuringWrap, 'true'); + const denied = fake.config?.credentials?.envVars?.map(({ name }) => name); + assert.ok(!denied?.includes('GIT_CONFIG_COUNT')); + assert.ok(!denied?.includes('GIT_CONFIG_KEY_0')); + assert.ok(!denied?.includes('GIT_CONFIG_VALUE_0')); +}); + test('filters environment names case-insensitively only on Windows', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -118,6 +337,19 @@ test('filters environment names case-insensitively only on Windows', async (t) = Path: 'C:\\Windows\\System32', LC_API_TOKEN: 'secret', librechat_code_worker_token: 'secret', + librechat_code_github_authorization: 'secret', + git_config_count: '1', + }, + maskedEnvironment: { + variables: [ + { + name: 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION', + injectHosts: ['github.com'], + }, + ], + async resolve() { + return {}; + }, }, manager: fake.manager, }); @@ -128,6 +360,8 @@ test('filters environment names case-insensitively only on Windows', async (t) = assert.ok(!denied?.includes('Path')); assert.ok(!denied?.includes('LC_API_TOKEN')); assert.ok(denied?.includes('librechat_code_worker_token')); + assert.ok(!denied?.includes('librechat_code_github_authorization')); + assert.ok(!denied?.includes('git_config_count')); }); test('fails closed when the configured POSIX shell is unavailable', async (t) => { diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 2136f9b3..1b1d96cb 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -48,6 +48,24 @@ const SAFE_CHILD_ENV_NAMES = new Set([ 'USER', ]); +let hostEnvironmentMutationQueue: Promise = Promise.resolve(); + +const TRUSTED_GIT_ENVIRONMENT = { + GIT_CONFIG_COUNT: '4', + GIT_CONFIG_KEY_0: 'filter.lfs.clean', + GIT_CONFIG_VALUE_0: 'git-lfs clean -- %f', + GIT_CONFIG_KEY_1: 'filter.lfs.smudge', + GIT_CONFIG_VALUE_1: 'git-lfs smudge -- %f', + GIT_CONFIG_KEY_2: 'filter.lfs.process', + GIT_CONFIG_VALUE_2: 'git-lfs filter-process', + GIT_CONFIG_KEY_3: 'filter.lfs.required', + GIT_CONFIG_VALUE_3: 'true', +} as const; +const { + GIT_CONFIG_COUNT: TRUSTED_GIT_CONFIG_COUNT, + ...TRUSTED_GIT_CONFIG_ENTRIES +} = TRUSTED_GIT_ENVIRONMENT; + interface NativeSandboxManager { isSupportedPlatform(): boolean; checkDependenciesAsync(): Promise<{ warnings: string[]; errors: string[] }>; @@ -83,6 +101,16 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { platform?: NodeJS.Platform; /** Trusted shell path used by SRT on POSIX hosts. */ shellPath?: string; + /** Host-owned credentials exposed only as SRT sentinels inside the sandbox. */ + maskedEnvironment?: { + variables: Array<{ + name: string; + injectHosts: string[]; + extract?: string; + }>; + resolve(signal?: AbortSignal): Promise>; + wrapCommand?(command: string, platform: NodeJS.Platform): string; + }; } function isWithin(root: string, candidate: string): boolean { @@ -135,6 +163,13 @@ function safeEnvironmentNames( .sort(); } +function normalizedEnvironmentName( + name: string, + platform: NodeJS.Platform, +): string { + return platform === 'win32' ? name.toUpperCase() : name; +} + export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private readonly manager: NativeSandboxManager; @@ -222,6 +257,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox strictAllowlist: true, allowAllUnixSockets: false, allowLocalBinding: false, + ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), }, filesystem: { denyRead: [home], @@ -235,10 +271,31 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox path, mode: 'deny' as const, })), - envVars: safeEnvironmentNames(this.environment, this.platform).map((name) => ({ - name, - mode: 'deny' as const, - })), + envVars: [ + ...safeEnvironmentNames(this.environment, this.platform) + .filter((name) => { + const normalized = normalizedEnvironmentName( + name, + this.platform, + ); + return ( + !Object.hasOwn(TRUSTED_GIT_ENVIRONMENT, normalized) && + !this.options.maskedEnvironment?.variables.some( + (variable) => + normalizedEnvironmentName( + variable.name, + this.platform, + ) === normalized, + ) + ); + }) + .map((name) => ({ name, mode: 'deny' as const })), + ...(this.options.maskedEnvironment?.variables.map((variable) => ({ + ...variable, + mode: 'mask' as const, + ...(variable.extract ? { onExtractNoMatch: 'error' as const } : {}), + })) ?? []), + ], }, allowAppleEvents: false, enableWeakerNestedSandbox: false, @@ -282,19 +339,34 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const commandId = `librechat-code-${randomUUID()}`; + const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand + ? this.options.maskedEnvironment.wrapCommand( + request.command, + this.platform, + ) + : request.command; let wrapped: Awaited< ReturnType >; try { - wrapped = await this.manager.wrapWithSandboxArgv( - request.command, - this.platform === 'win32' - ? undefined - : this.options.shellPath ?? '/bin/bash', - undefined, - signal, - cwd, - { commandId, commandText: request.command }, + const credentialEnvironment = + await this.options.maskedEnvironment?.resolve(signal); + wrapped = await this.withTemporaryHostEnvironment( + { + ...TRUSTED_GIT_ENVIRONMENT, + ...(credentialEnvironment ?? {}), + }, + () => + this.manager.wrapWithSandboxArgv( + sandboxedCommand, + this.platform === 'win32' + ? undefined + : (this.options.shellPath ?? '/bin/bash'), + undefined, + signal, + cwd, + { commandId, commandText: request.command }, + ), ); } catch (error) { if (signal?.aborted) { @@ -317,6 +389,32 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox return await this.runWrapped(request, wrapped, cwd, commandId, signal); } + private async withTemporaryHostEnvironment( + values: Record, + action: () => Promise, + ): Promise { + const previousMutation = hostEnvironmentMutationQueue; + let releaseMutation!: () => void; + hostEnvironmentMutationQueue = new Promise((resolve) => { + releaseMutation = resolve; + }); + await previousMutation; + const previous = new Map(); + try { + for (const [name, value] of Object.entries(values)) { + previous.set(name, process.env[name]); + process.env[name] = value; + } + return await action(); + } finally { + for (const [name, value] of previous) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + releaseMutation(); + } + } + private async runWrapped( request: WorkspaceExecuteCommandRequest, wrapped: { argv: string[]; env: NodeJS.ProcessEnv }, @@ -334,7 +432,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox try { child = this.spawnCommand(wrapped.argv[0], wrapped.argv.slice(1), { cwd, - env: wrapped.env, + env: { + ...wrapped.env, + ...TRUSTED_GIT_CONFIG_ENTRIES, + GIT_CONFIG_COUNT: + wrapped.env.GIT_CONFIG_COUNT ?? TRUSTED_GIT_CONFIG_COUNT, + GIT_CONFIG_GLOBAL: + this.platform === 'win32' ? 'NUL' : '/dev/null', + GIT_CONFIG_NOSYSTEM: '1', + }, detached: this.platform !== 'win32', shell: false, windowsHide: true, @@ -467,7 +573,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } private protocolExitCode(code: number | null): number { - return Number.isSafeInteger(code) && code != null && code >= 0 && code <= 255 + return Number.isSafeInteger(code) && + code != null && + code >= 0 && + code <= 255 ? code : 1; } From 595ade0fa56a7c16c4a8852c7688fb070246423d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 5 Sep 2026 23:56:04 -0400 Subject: [PATCH 042/116] fix: Forward hardened Compose bridge configuration (#118) --- .env.example | 14 +++++++--- .github/workflows/ci.yml | 3 +++ README.md | 9 +++++++ docker-compose.yaml | 8 ++++++ tests/compose-bridge-config.cjs | 45 +++++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 tests/compose-bridge-config.cjs diff --git a/.env.example b/.env.example index 98c97260..f1ead384 100644 --- a/.env.example +++ b/.env.example @@ -24,13 +24,21 @@ SANDBOX_RUN_CPU_TIME=10000 SANDBOX_RUN_TIMEOUT=15000 SANDBOX_OUTPUT_MAX_SIZE=65536 +# Docker Compose keeps hardened mode enabled. Its API exposes bridge routes even +# with the default HTTP sandbox backend, so configure a private enrollment token +# of at least 32 bytes before starting Compose (generate with: openssl rand -hex 32). +# Do not use a shared example token. These defaults accept paired dynamic workers; +# enrollment still requires the token. Never provide this token to sandbox code. +CODEAPI_BRIDGE_TOKEN= +CODEAPI_BRIDGE_AUTH_MODE=paired +CODEAPI_BRIDGE_DYNAMIC_WORKERS=true +# For a fixed worker, set DYNAMIC_WORKERS=false and WORKER_ID to that worker's ID. +CODEAPI_BRIDGE_WORKER_ID= + # Remote stateful code bridge (Code API deployment) # CODEAPI_SANDBOX_BACKEND=remote-bridge # CODEAPI_EXECUTION_PROFILE=stateful # CODEAPI_RUNTIME_SESSION_MODE=affinity -# CODEAPI_BRIDGE_WORKER_ID=my-vm -# CODEAPI_BRIDGE_TOKEN=replace-with-a-strong-random-secret -# CODEAPI_BRIDGE_AUTH_MODE=paired # Service Configuration PYTHON_CONCURRENCY=5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97d76161..d34395ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Bridge pairing rollout safety run: tests/bridge_pairing_rollout.sh + - name: Compose bridge configuration + run: node tests/compose-bridge-config.cjs + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/README.md b/README.md index 113cbb0e..c2eeb7eb 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,15 @@ cut. ## Local Development +Copy `.env.example` to `.env` and set `CODEAPI_BRIDGE_TOKEN` to a private value +of at least 32 bytes (generate one with `openssl rand -hex 32`). The API exposes +bridge routes even with the default HTTP sandbox backend, so hardened mode +requires this enrollment credential. Compose defaults to +`CODEAPI_BRIDGE_AUTH_MODE=paired` and `CODEAPI_BRIDGE_DYNAMIC_WORKERS=true`. +To restrict pairing to a fixed worker, set `CODEAPI_BRIDGE_DYNAMIC_WORKERS=false` +and `CODEAPI_BRIDGE_WORKER_ID` to its ID. Keep the token outside workspaces and +model-visible configuration. + ```bash docker-compose up --build ``` diff --git a/docker-compose.yaml b/docker-compose.yaml index 00cad657..3409554c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,6 +10,10 @@ services: environment: - LOCAL_MODE=${LOCAL_MODE:-true} - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true} + - CODEAPI_BRIDGE_TOKEN=${CODEAPI_BRIDGE_TOKEN:-} + - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} + - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} + - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-} @@ -54,6 +58,10 @@ services: environment: - LOCAL_MODE=${LOCAL_MODE:-true} - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true} + - CODEAPI_BRIDGE_TOKEN=${CODEAPI_BRIDGE_TOKEN:-} + - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} + - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} + - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-} diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs new file mode 100644 index 00000000..d6c9797a --- /dev/null +++ b/tests/compose-bridge-config.cjs @@ -0,0 +1,45 @@ +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); + +function render(overrides) { + return JSON.parse(execFileSync('docker', [ + 'compose', '--env-file', '/dev/null', '-f', 'docker-compose.yaml', + 'config', '--format', 'json', + ], { + encoding: 'utf8', + env: { + ...process.env, + CODEAPI_HARDENED_SANDBOX_MODE: '', + CODEAPI_BRIDGE_AUTH_MODE: '', + CODEAPI_BRIDGE_DYNAMIC_WORKERS: '', + CODEAPI_BRIDGE_WORKER_ID: '', + CODEAPI_BRIDGE_TOKEN: '', + ...overrides, + }, + })); +} + +const token = 'compose-test-token-never-use-in-production'; +for (const overrides of [ + { CODEAPI_BRIDGE_TOKEN: token }, + { + CODEAPI_BRIDGE_TOKEN: token, + CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', + CODEAPI_BRIDGE_WORKER_ID: 'test-worker', + }, +]) { + const config = render(overrides); + for (const name of ['api', 'service-worker']) { + const env = config.services[name].environment; + assert.equal(env.CODEAPI_HARDENED_SANDBOX_MODE, 'true'); + assert.equal(env.CODEAPI_BRIDGE_AUTH_MODE, 'paired'); + assert.equal(env.CODEAPI_BRIDGE_TOKEN, token); + assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); + assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); + } + for (const name of ['egress_gateway', 'sandbox-runner']) { + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); + } +} +assert.equal(render({}).services.api.environment.CODEAPI_BRIDGE_TOKEN, ''); +console.log('Compose bridge configuration passed (dynamic/fixed pairing, no default secret).'); From 9df5cf8ae007b7732f894a82596cc8db8985861f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 6 Sep 2026 14:30:10 -0400 Subject: [PATCH 043/116] =?UTF-8?q?=F0=9F=8F=98=EF=B8=8F=20feat:=20Add=20H?= =?UTF-8?q?osted=20App=20Control=20Plane=20(#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Lambda hosted-app control plane * fix: preserve safe hosted-app worker errors * fix: rate limit hosted-app starts * fix: forward trusted hosted-app origin * fix: constrain hosted-app browser capabilities * fix: disable hosted-app DNS prefetch * fix: harden hosted app lifecycle and preview access * fix: harden hosted app recovery and previews * fix: Preserve Hosted App Recovery and Preview Boundaries * fix: Bound Hosted Preview Refresh And Reconcile Process Status * fix: Preserve Hosted Revisions Beyond VM Leases And Align Deployment Policy * fix: Separate Hosted Recovery From Credential And Throttle Failures --- .github/workflows/ci.yml | 1 + docs/lambda-microvm/README.md | 141 ++++ docs/lambda-microvm/terraform/README.md | 20 +- docs/lambda-microvm/terraform/main.tf | 32 +- .../terraform/tests/hosted-app.tftest.hcl | 77 ++ docs/lambda-microvm/terraform/variables.tf | 10 + helm/codeapi/README.md | 9 + service/.env.example | 10 +- service/openapi.yml | 148 ++++ service/src/api-server.ts | 4 + service/src/config.ts | 34 + service/src/hosted-app/control-plane.test.ts | 632 +++++++++++++++ service/src/hosted-app/control-plane.ts | 748 ++++++++++++++++++ service/src/hosted-app/credential.test.ts | 48 ++ service/src/hosted-app/credential.ts | 98 +++ service/src/hosted-app/factory.ts | 145 ++++ service/src/hosted-app/jobs.ts | 32 + .../src/hosted-app/microvm-runtime.test.ts | 309 ++++++++ service/src/hosted-app/microvm-runtime.ts | 496 ++++++++++++ service/src/hosted-app/preview-access.test.ts | 72 ++ service/src/hosted-app/preview-access.ts | 144 ++++ service/src/hosted-app/preview-gateway.ts | 165 ++++ service/src/hosted-app/preview-proxy.test.ts | 166 ++++ service/src/hosted-app/preview-proxy.ts | 343 ++++++++ service/src/hosted-app/proxy-policy.test.ts | 67 ++ service/src/hosted-app/proxy-policy.ts | 105 +++ service/src/hosted-app/queue-deadline.test.ts | 15 + service/src/hosted-app/queue-deadline.ts | 15 + service/src/hosted-app/queue.ts | 78 ++ service/src/hosted-app/record.ts | 38 + service/src/hosted-app/router.ts | 217 +++++ .../src/hosted-app/source-checkpoint.test.ts | 75 ++ service/src/hosted-app/source-checkpoint.ts | 86 ++ service/src/hosted-app/spec.test.ts | 70 ++ service/src/hosted-app/spec.ts | 228 ++++++ service/src/hosted-app/worker.test.ts | 42 + service/src/hosted-app/worker.ts | 123 +++ service/src/lifecycle.ts | 41 +- service/src/middleware/httpMetrics.test.ts | 15 + service/src/middleware/httpMetrics.ts | 7 +- .../runtime-session/checkpoint-store.test.ts | 79 ++ .../src/runtime-session/checkpoint-store.ts | 77 ++ service/src/runtime-session/registry.ts | 5 + service/src/secure-startup.test.ts | 125 +++ service/src/secure-startup.ts | 116 +++ service/src/service-api.ts | 4 + service/src/worker-server.ts | 12 +- 47 files changed, 5507 insertions(+), 17 deletions(-) create mode 100644 docs/lambda-microvm/terraform/tests/hosted-app.tftest.hcl create mode 100644 service/src/hosted-app/control-plane.test.ts create mode 100644 service/src/hosted-app/control-plane.ts create mode 100644 service/src/hosted-app/credential.test.ts create mode 100644 service/src/hosted-app/credential.ts create mode 100644 service/src/hosted-app/factory.ts create mode 100644 service/src/hosted-app/jobs.ts create mode 100644 service/src/hosted-app/microvm-runtime.test.ts create mode 100644 service/src/hosted-app/microvm-runtime.ts create mode 100644 service/src/hosted-app/preview-access.test.ts create mode 100644 service/src/hosted-app/preview-access.ts create mode 100644 service/src/hosted-app/preview-gateway.ts create mode 100644 service/src/hosted-app/preview-proxy.test.ts create mode 100644 service/src/hosted-app/preview-proxy.ts create mode 100644 service/src/hosted-app/proxy-policy.test.ts create mode 100644 service/src/hosted-app/proxy-policy.ts create mode 100644 service/src/hosted-app/queue-deadline.test.ts create mode 100644 service/src/hosted-app/queue-deadline.ts create mode 100644 service/src/hosted-app/queue.ts create mode 100644 service/src/hosted-app/record.ts create mode 100644 service/src/hosted-app/router.ts create mode 100644 service/src/hosted-app/source-checkpoint.test.ts create mode 100644 service/src/hosted-app/source-checkpoint.ts create mode 100644 service/src/hosted-app/spec.test.ts create mode 100644 service/src/hosted-app/spec.ts create mode 100644 service/src/hosted-app/worker.test.ts create mode 100644 service/src/hosted-app/worker.ts create mode 100644 service/src/middleware/httpMetrics.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d34395ec..0b25285b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,6 +215,7 @@ jobs: terraform fmt -check -recursive terraform init -backend=false -input=false -lockfile=readonly terraform validate + terraform test lambda-microvm-runner-build: name: Lambda MicroVM Runner Image (arm64) diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 2283bbd7..4ec87d94 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -200,6 +200,108 @@ again after expiry. Static assets and request-shaped handlers should remain on cheaper stateless delivery paths; this target is only the resident-server adapter. +#### Hosted-app control plane and preview gateway + +The service-side control plane is stateful-profile only. It snapshots the +source runtime session under its existing lock, records an exact checkpoint and +AWS idempotency intent in the fenced Redis registry, then launches/restores the +dedicated app-host VM on an isolated BullMQ queue. API pods enqueue lifecycle +work and proxy preview bytes; only worker pods need Lambda MicroVM IAM. +Start requests share the authenticated execution rate limiter. Before enabling +this feature broadly for untrusted multi-tenant traffic, add a plan-aware cap on +active hosted-app leases per owner; the initial feature-flagged slice relies on +the deployment's Lambda MicroVM quota as its hard fleet ceiling. + +Authenticated API contract: + +```http +POST /v1/hosted-apps +Content-Type: application/json + +{ + "runtime_session_hint": "conversation-123", + "app_id": "my-app", + "revision": "rev-1", + "language": "node", + "version": ">=22", + "entrypoint": "server.js", + "cwd": ".", + "args": [], + "env": {} +} +``` + +`GET /v1/hosted-apps/:app_id?runtime_session_hint=...` returns status and a +fresh five-minute `preview_url`; `DELETE` on the same resource terminates the +lease. A revision is immutable. Retrying the identical spec reasserts the +resident process; changing code or launch settings requires a new revision and +captures a new exact checkpoint. An ambiguous provider launch is replayed only +with its persisted token and can never be overwritten by a newer revision. + +Preview traffic uses a wildcard **unprivileged origin**, not a path below the +CodeAPI or LibreChat origin. Configure wildcard DNS and TLS such that +`*.apps.example.net` reaches the stateful CodeAPI API service, then set the bare +origin `https://apps.example.net`. The short-lived URL capability is exchanged +for an HttpOnly, Secure, host-only cookie and redirected to `/`; every app gets +its own `happ-.apps.example.net` origin, so absolute asset paths work +without exposing privileged-origin cookies to AI-generated JavaScript. Use a +dedicated registrable domain in production—do not set broad parent-domain +cookies that also match the app domain. + +Set these on both API and worker pods: + +| Env | Default | Meaning | +|---|---|---| +| `CODEAPI_HOSTED_APPS_ENABLED` | `false` | Enables the stateful-only lifecycle API, isolated preview gateway, and worker. | +| `CODEAPI_HOSTED_APP_CREDENTIAL_KEY` | — | Base64 of 32 random bytes; AES-GCM encrypts the AWS preview credential stored in Redis. | + +Set these only on API pods (the signing key must differ from the credential +key): + +| Env | Default | Meaning | +|---|---|---| +| `CODEAPI_HOSTED_APP_PREVIEW_SIGNING_KEY` | — | Different base64 32-byte key for owner-bound preview URL/cookie capabilities. | +| `CODEAPI_HOSTED_APP_PREVIEW_ORIGIN` | — | Bare HTTPS origin for wildcard app hosts, for example `https://apps.example.net`. | + +Set these on worker pods in addition to the ordinary stateful/checkpoint +configuration: + +| Env | Default | Meaning | +|---|---|---| +| `LAMBDA_MICROVM_APP_IMAGE_ARN` | — | Dedicated `lambda-microvm-app-host` image ARN. | +| `LAMBDA_MICROVM_APP_IMAGE_VERSION` | — | Required pinned image version. | +| `LAMBDA_MICROVM_APP_MAX_DURATION_SECONDS` | `28800` | App-VM hard lifetime. The control plane relaunches an immutable revision after expiry. | +| `LAMBDA_MICROVM_APP_IDLE_SECONDS` | `300` | Seconds idle before AWS suspends the VM. | +| `LAMBDA_MICROVM_APP_SUSPEND_SECONDS` | `900` | Seconds suspended before AWS terminates the VM. Suspended VMs still consume quota. | + +The pinned app-host image contract fixes the root-owned control/checkpoint +listener at port 8080, the resident app at port 3000, and resident readiness at +30 seconds. `RunMicrovm` cannot override the image environment; changing this +contract requires publishing a matching image and control-plane revision. + +Generate the two keys independently: + +```bash +openssl rand -base64 32 # CODEAPI_HOSTED_APP_CREDENTIAL_KEY +openssl rand -base64 32 # CODEAPI_HOSTED_APP_PREVIEW_SIGNING_KEY +``` + +The preview proxy strips CodeAPI authorization, cookies, forwarded identity, +caller-provided AWS headers, app `Set-Cookie`, app-controlled caching, +cross-origin policy, and external redirects. Gateway responses are private and +non-storable so an older revision cannot survive through the browser cache. It +supports streamed HTTP/SSE and same-origin redirects. WebSockets +are not part of this first resident adapter. A gateway-owned CSP constrains +fetches and subresources to the app origin and disables workers/service workers, +so one app revision cannot leave a persistent worker controlling a later +revision. Top-level app JavaScript can still navigate the owner's browser to an +external origin; treat this experimental viewer as owner-trusted. Before broad +untrusted enablement, serve app content from a separate origin inside a sandboxed +gateway wrapper. The request `env` map is persisted +with the immutable launch spec in the registry; it is configuration, not a +secret store. Add a dedicated secret-reference flow before passing application +secrets to hosted code. + ### 3. Generate the split execution-manifest keys The worker signs each execution manifest; the runner only receives the public @@ -654,3 +756,42 @@ terraform -chdir=docs/lambda-microvm/terraform destroy MicroVM images are billed as stored snapshots; running VMs bill while RUNNING and suspended VMs bill at a reduced rate, so terminate stray VMs before deleting the image. + +## Hosted-app retention and rollout + +Hosted revisions retain a separate immutable checkpoint under the source session's +`hosted/` object prefix before releasing the source lease. Rolling workspace +checkpoint pruning never deletes these snapshots. Keep this prefix out of short +bucket lifecycle expiry policies; an operator may reclaim retained objects only +after retiring all hosted revisions that reference them. Automatic reclamation of +unreferenced hosted snapshots is not yet implemented. + +Hosted apps require the `lambda-microvm` backend and an HTTPS preview origin in +every environment, including development. Preview authentication uses Secure +host-only cookies. Worker shutdown allows the full hosted-operation budget plus +launch cleanup and a 30-second reserve; configure the orchestrator's termination +grace period to cover that same budget (19 minutes with default timeouts). + +Immutable revision manifests live in checkpoint storage separately from expiring +Redis VM records. They bind the owner, source, revision, spec fingerprint and +retained snapshot using create-only writes. Keep both manifests and hosted +snapshots until explicitly retiring those revisions; ordinary Redis expiry is +not a revision reset. Existing experimental records are migrated on reassertion; +already-expired pre-manifest records cannot be reconstructed automatically. +Apply the Terraform retention policy and `s3:PutObjectTagging` permission before +rolling these workers. Configure `hosted_app_image_arn` to match the dedicated +app-host image; its policy includes resume permission for ambiguous recovery. + +Roll the service binary to every hosted-app worker before enabling hosted apps +on API pods. For upgrades from experimental builds, disable hosted-app admission +on API pods and drain the hosted queue before replacing workers; re-enable it +after the workers are ready. Older workers do not recognize the status job and +must not consume jobs submitted by the new API. The feature defaults to disabled. + +Running status is a worker-reconciled observation of the resident process, not +just a cached VM lease. It is rate-limited like start, waits at most 15 seconds +for the worker result, and returns unavailable rather than a stale running +claim if reconciliation fails. A crashed process does not erase its VM identity: +stop can still terminate the VM and start can reassert the same revision. +Preview refresh waits at most two seconds (or half the remaining credential/VM +lifetime when shorter), then rereads and reauthorizes any still-valid credential. diff --git a/docs/lambda-microvm/terraform/README.md b/docs/lambda-microvm/terraform/README.md index f9863b18..ce4d952d 100644 --- a/docs/lambda-microvm/terraform/README.md +++ b/docs/lambda-microvm/terraform/README.md @@ -28,8 +28,10 @@ walkthrough. - **Worker control policy** — `Run/Get/TerminateMicrovm`, `CreateMicrovmAuthToken`, and the dependent `iam:PassRole` / `lambda:PassNetworkConnector` permissions required by `RunMicrovm`. The - worker does not call or receive permission for `SuspendMicrovm` or - `ResumeMicrovm`; the configured AWS idle policy performs those transitions. + ordinary runner policy does not grant `SuspendMicrovm` or `ResumeMicrovm`. + Set `hosted_app_image_arn` to the dedicated `LAMBDA_MICROVM_APP_IMAGE_ARN` + when enabling hosted apps. A separate image-scoped statement permits its + lifecycle operations and `ResumeMicrovm` for same-token suspended-launch recovery. - **CloudWatch log groups** — build (`/aws/lambda-microvms/`) and runtime. - **Checkpoint access** — an IAM policy for task-role/instance-profile/IRSA @@ -77,8 +79,20 @@ terraform output repository. The checked-in `terraform.tfvars.example` is specifically an AIML-dev/disposable-stack example and explicitly opts all three into destructive teardown; do not copy those overrides to retained environments. -- Current checkpoint versions expire after `checkpoint_retention_days`. +- Current checkpoint versions expire after `checkpoint_retention_days`; when + `hosted_app_image_arn` is configured, only `codeapi-retention=rolling` objects expire. Noncurrent S3 versions expire independently after `checkpoint_noncurrent_retention_days` (one day by default), so bucket versioning does not unexpectedly retain replaced checkpoint data for another full current-version window. +- Hosted snapshots and immutable revision manifests use `codeapi-retention=hosted` + and are excluded from automatic current/noncurrent expiry. The checkpoint IAM + policy grants `s3:PutObjectTagging` for these writes and server-side copies. + Apply this lifecycle policy before enabling hosted apps. Legacy untagged objects + no longer expire automatically: tag only verified ordinary rolling checkpoints + during migration, never hosted snapshots or revision manifests. Runtime rolling + checkpoint pruning continues to delete superseded ordinary checkpoints. +- Workers tag ordinary checkpoints only when hosted apps are enabled, so the + disabled/default deployment does not acquire a new tagging-permission requirement. + Never clear `hosted_app_image_arn` while retained hosted data remains: doing so + restores the ordinary bucket-wide expiry policy. diff --git a/docs/lambda-microvm/terraform/main.tf b/docs/lambda-microvm/terraform/main.tf index 4f899760..b1dae758 100644 --- a/docs/lambda-microvm/terraform/main.tf +++ b/docs/lambda-microvm/terraform/main.tf @@ -164,13 +164,26 @@ resource "aws_s3_bucket_lifecycle_configuration" "checkpoint" { rule { id = "expire-checkpoints" status = "Enabled" - filter {} + filter { + dynamic "tag" { + for_each = var.hosted_app_image_arn == "" ? [] : [1] + content { + key = "codeapi-retention" + value = "rolling" + } + } + } expiration { days = var.checkpoint_retention_days } noncurrent_version_expiration { noncurrent_days = var.checkpoint_noncurrent_retention_days } + } + rule { + id = "abort-incomplete-checkpoints" + status = "Enabled" + filter {} abort_incomplete_multipart_upload { days_after_initiation = 7 } @@ -317,6 +330,21 @@ resource "aws_iam_role_policy" "execution" { # for every ingress/egress connector supplied on the request. # -------------------------------------------------------------------------- data "aws_iam_policy_document" "worker_microvm_control" { + dynamic "statement" { + for_each = var.hosted_app_image_arn == "" ? [] : [var.hosted_app_image_arn] + content { + sid = "OperateHostedAppMicrovms" + effect = "Allow" + actions = [ + "lambda:RunMicrovm", + "lambda:GetMicrovm", + "lambda:CreateMicrovmAuthToken", + "lambda:TerminateMicrovm", + "lambda:ResumeMicrovm", + ] + resources = [statement.value] + } + } statement { sid = "OperateCodeapiMicrovms" effect = "Allow" @@ -370,7 +398,7 @@ data "aws_iam_policy_document" "checkpoint_access" { statement { sid = "CheckpointObjects" effect = "Allow" - actions = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"] + actions = ["s3:GetObject", "s3:PutObject", "s3:PutObjectTagging", "s3:DeleteObject"] resources = ["${aws_s3_bucket.checkpoint.arn}/*"] } statement { diff --git a/docs/lambda-microvm/terraform/tests/hosted-app.tftest.hcl b/docs/lambda-microvm/terraform/tests/hosted-app.tftest.hcl new file mode 100644 index 00000000..fbb64a0e --- /dev/null +++ b/docs/lambda-microvm/terraform/tests/hosted-app.tftest.hcl @@ -0,0 +1,77 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" } + } + mock_data "aws_caller_identity" { + defaults = { account_id = "123456789012" } + } + mock_data "aws_partition" { + defaults = { partition = "aws", dns_suffix = "amazonaws.com" } + } +} + +variables { + region = "us-east-2" + name_prefix = "hosted-test" + image_name = "runner" + hosted_app_image_arn = "arn:aws:lambda:us-east-2:123456789012:microvm-image:app-host" +} + +run "hosted_permissions_and_retention" { + command = plan + + assert { + condition = contains(one([ + for statement in data.aws_iam_policy_document.worker_microvm_control.statement : statement + if statement.sid == "OperateHostedAppMicrovms" + ]).actions, "lambda:ResumeMicrovm") + error_message = "Hosted suspended-launch recovery requires ResumeMicrovm." + } + assert { + condition = toset(one([ + for statement in data.aws_iam_policy_document.worker_microvm_control.statement : statement + if statement.sid == "OperateHostedAppMicrovms" + ]).resources) == toset([var.hosted_app_image_arn]) + error_message = "Hosted control permissions must be scoped to the dedicated image." + } + assert { + condition = contains(one([ + for statement in data.aws_iam_policy_document.checkpoint_access.statement : statement + if statement.sid == "CheckpointObjects" + ]).actions, "s3:PutObjectTagging") + error_message = "Checkpoint writes and retained copies require tagging permission." + } + assert { + condition = one(one(one([ + for rule in aws_s3_bucket_lifecycle_configuration.checkpoint.rule : rule + if rule.id == "expire-checkpoints" + ]).filter).tag).value == "rolling" + error_message = "Only tagged rolling checkpoints may expire, never hosted snapshots/manifests." + } + assert { + condition = one(one(one([ + for rule in aws_s3_bucket_lifecycle_configuration.checkpoint.rule : rule + if rule.id == "expire-checkpoints" + ]).filter).tag).key == "codeapi-retention" + error_message = "The expiry filter must use the runtime retention tag." + } +} + +run "ordinary_runner_has_no_hosted_permissions" { + command = plan + variables { hosted_app_image_arn = "" } + assert { + condition = alltrue([ + for statement in data.aws_iam_policy_document.worker_microvm_control.statement : + !contains(statement.actions, "lambda:ResumeMicrovm") + ]) + error_message = "Hosted resume permission must remain opt-in." + } + assert { + condition = length(one(one([ + for rule in aws_s3_bucket_lifecycle_configuration.checkpoint.rule : rule + if rule.id == "expire-checkpoints" + ]).filter).tag) == 0 + error_message = "Ordinary deployments retain their existing untagged checkpoint expiry policy." + } +} diff --git a/docs/lambda-microvm/terraform/variables.tf b/docs/lambda-microvm/terraform/variables.tf index caae4cef..60b0419b 100644 --- a/docs/lambda-microvm/terraform/variables.tf +++ b/docs/lambda-microvm/terraform/variables.tf @@ -10,6 +10,16 @@ variable "name_prefix" { default = "codeapi-microvm" } +variable "hosted_app_image_arn" { + description = "Optional dedicated app-host image ARN; grants hosted lifecycle and suspended-launch recovery permissions only on this image." + type = string + default = "" + validation { + condition = var.hosted_app_image_arn == "" || can(regex("^arn:[a-z0-9-]+:lambda:[a-z0-9-]+:[0-9]{12}:microvm-image:[A-Za-z0-9_-]+$", var.hosted_app_image_arn)) + error_message = "hosted_app_image_arn must be empty or an exact MicroVM image ARN without wildcards." + } +} + variable "image_name" { description = <<-EOT Name of the MicroVM image you will create with the SDK/CLI helper. Only used diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index 29dc894a..a063e597 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -118,6 +118,15 @@ hardening variables documented in `docs/lambda-microvm/README.md`. This chart still renders its bundled sandbox-runner, though a Lambda worker does not call it; a platform-specific stateful deployment may omit that component. +Resident hosted apps are an opt-in capability of that stateful deployment. +Configure `CODEAPI_HOSTED_APPS_ENABLED`, the preview origin, and both hosted-app +keys through `api.extraEnv`; configure only the feature flag and credential key +on `workerSandbox.extraEnv`, along with the dedicated app image settings. +Wildcard DNS and TLS for the preview origin must route to the API service +separately from the normal CodeAPI host. See “Hosted-app control plane and preview gateway” in +`docs/lambda-microvm/README.md`; do not serve previews beneath the privileged +LibreChat/CodeAPI origin. + For an existing affinity/strict deployment from before execution profiles, first roll the new binary to API and worker pods with `CODEAPI_EXECUTION_PROFILE` still unset. The inferred stateful compatibility diff --git a/service/.env.example b/service/.env.example index e06ec4c0..c9b5beaf 100644 --- a/service/.env.example +++ b/service/.env.example @@ -9,6 +9,14 @@ REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD=mysecretpassword +# Stateful Lambda resident hosted apps (optional; see docs/lambda-microvm/README.md) +# CODEAPI_HOSTED_APPS_ENABLED=true +# LAMBDA_MICROVM_APP_IMAGE_ARN=arn:aws:lambda:REGION:ACCOUNT:microvm-image:codeapi-app-host +# LAMBDA_MICROVM_APP_IMAGE_VERSION=1 +# CODEAPI_HOSTED_APP_PREVIEW_ORIGIN=https://apps.example.net +# CODEAPI_HOSTED_APP_CREDENTIAL_KEY= +# CODEAPI_HOSTED_APP_PREVIEW_SIGNING_KEY= + # ----------------------------------------------------------------------------- # Stripe: https://shipfa.st/docs/features/payments # ----------------------------------------------------------------------------- @@ -20,4 +28,4 @@ STRIPE_WEBHOOK_SECRET= # Mailgun: https://shipfa.st/docs/features/emails # ----------------------------------------------------------------------------- # EMAIL_SERVER=smtp://postmaster@[mail.yourdomain.com]:[copied_password]@smtp.mailgun.org:587 (without the brackets) -EMAIL_SERVER= \ No newline at end of file +EMAIL_SERVER= diff --git a/service/openapi.yml b/service/openapi.yml index 913e7809..78ba082f 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -227,7 +227,155 @@ components: type: string enum: [default, stateful] + HostedAppStartRequest: + type: object + required: + - runtime_session_hint + - app_id + - revision + - language + - version + - entrypoint + properties: + runtime_session_hint: + type: string + maxLength: 128 + pattern: '^[A-Za-z0-9._:-]+$' + adapter: + type: string + enum: [resident] + default: resident + app_id: + type: string + maxLength: 64 + pattern: '^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$' + revision: + type: string + maxLength: 128 + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' + language: + type: string + version: + type: string + entrypoint: + type: string + description: Canonical path relative to the restored stateful workspace. + cwd: + type: string + default: . + args: + type: array + maxItems: 64 + items: + type: string + env: + type: object + additionalProperties: + type: string + + HostedAppStatus: + type: object + required: [app_id, revision, state, preview_id, updated_at] + properties: + app_id: + type: string + revision: + type: string + state: + type: string + enum: [starting, running, stopping, stopped, failed] + preview_id: + type: string + description: Opaque hosted-app lease identity. + preview_url: + type: string + format: uri + description: Five-minute owner capability exchange URL on the isolated app origin. + hard_deadline_at: + type: integer + format: int64 + updated_at: + type: integer + format: int64 + error: + type: string + paths: + /hosted-apps: + post: + summary: Start or reassert a resident hosted app + description: >- + Stateful-profile only. Captures an exact workspace checkpoint and runs + the immutable revision in a dedicated Lambda MicroVM app-host image. + operationId: startHostedApp + parameters: + - $ref: '#/components/parameters/ExpectedExecutionProfile' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/HostedAppStartRequest' + responses: + '200': + description: Hosted app is running + content: + application/json: + schema: + $ref: '#/components/schemas/HostedAppStatus' + '400': + $ref: '#/components/responses/BadRequest' + '409': + $ref: '#/components/responses/Conflict' + '429': + description: Hosted app start rate limit exceeded + '503': + description: Lifecycle or provider operation unavailable + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /hosted-apps/{app_id}: + parameters: + - $ref: '#/components/parameters/ExpectedExecutionProfile' + - name: app_id + in: path + required: true + schema: + type: string + - name: runtime_session_hint + in: query + required: true + schema: + type: string + maxLength: 128 + pattern: '^[A-Za-z0-9._:-]+$' + get: + summary: Get hosted app status and a fresh preview URL + operationId: getHostedApp + responses: + '200': + description: Hosted app status + content: + application/json: + schema: + $ref: '#/components/schemas/HostedAppStatus' + '404': + description: Hosted app not found + delete: + summary: Terminate a hosted app lease + operationId: stopHostedApp + responses: + '200': + description: Hosted app stopped + content: + application/json: + schema: + $ref: '#/components/schemas/HostedAppStatus' + '404': + description: Hosted app not found + /exec: post: summary: Execute code diff --git a/service/src/api-server.ts b/service/src/api-server.ts index 4a098576..fd77b55d 100644 --- a/service/src/api-server.ts +++ b/service/src/api-server.ts @@ -27,6 +27,8 @@ import { executionProfileMiddleware } from './middleware/execution-profile'; import { traceHttpRequest } from './telemetry'; import { env } from './config'; import logger from './logger'; +import hostedAppRouter from './hosted-app/router'; +import { hostedAppPreviewGateway } from './hosted-app/preview-gateway'; const { LOCAL_MODE: isLocalMode } = env; @@ -36,6 +38,7 @@ app.set('trust proxy', 1); app.use(traceHttpRequest('codeapi.api.request')); app.use(httpMetricsMiddleware); app.use(executionProfileMiddleware); +app.use(hostedAppPreviewGateway); const v1 = Router(); @@ -57,6 +60,7 @@ v1.use('/bridge', bridgeRouter); v1.use(isLocalMode ? localAuth : apiKeyAuth); v1.use(workspaceToolsRouter); +v1.use('/hosted-apps', hostedAppRouter); v1.use(serviceRouter); v1.use(programmaticRouter); diff --git a/service/src/config.ts b/service/src/config.ts index 8a8208f9..53848103 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -237,6 +237,12 @@ export function resolveEgressGrantTtlSeconds(rawTtlSeconds: string | undefined, const lambdaMicrovmNumericConfig = resolveLambdaMicrovmNumericConfig(process.env); +export function hostedAppOperationTimeoutMs(): number { + return env.CHECKPOINT_TIMEOUT_MS * 9 + + env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS * 7 + + env.HOSTED_APP_START_TIMEOUT_MS + 30_000; +} + function configuredNumber(raw: string | undefined, fallback: number): number { return raw == null || raw.trim() === '' ? fallback : Number(raw); } @@ -448,6 +454,34 @@ export const env = { ), CHECKPOINT_TIMEOUT_MS: configuredNumber(process.env.CODEAPI_CHECKPOINT_TIMEOUT_MS, 60_000), CHECKPOINT_PREFIX: process.env.CODEAPI_CHECKPOINT_PREFIX ?? 'rtsx-checkpoints/', + /** Dedicated Lambda MicroVM resident-server fleet. This remains an explicit + * stateful-stack capability; the ordinary/default HTTP profile never starts + * or preserves application processes. */ + HOSTED_APPS_ENABLED: process.env.CODEAPI_HOSTED_APPS_ENABLED === 'true', + HOSTED_APP_IMAGE_ARN: process.env.LAMBDA_MICROVM_APP_IMAGE_ARN ?? '', + HOSTED_APP_IMAGE_VERSION: process.env.LAMBDA_MICROVM_APP_IMAGE_VERSION || undefined, + /* These values are part of the pinned app-host image contract. RunMicrovm + * cannot inject environment variables into the image, so exposing overrides + * here would only make the control plane call ports the runner never opened. */ + HOSTED_APP_CONTROL_PORT: 8080 as number, + HOSTED_APP_PREVIEW_PORT: 3000 as number, + HOSTED_APP_MAX_DURATION_SECONDS: configuredNumber( + process.env.LAMBDA_MICROVM_APP_MAX_DURATION_SECONDS, + 28_800, + ), + HOSTED_APP_IDLE_SECONDS: configuredNumber( + process.env.LAMBDA_MICROVM_APP_IDLE_SECONDS, + 300, + ), + HOSTED_APP_SUSPEND_SECONDS: configuredNumber( + process.env.LAMBDA_MICROVM_APP_SUSPEND_SECONDS, + 900, + ), + HOSTED_APP_START_TIMEOUT_MS: 30_000 as number, + HOSTED_APP_CREDENTIAL_KEY: process.env.CODEAPI_HOSTED_APP_CREDENTIAL_KEY ?? '', + HOSTED_APP_PREVIEW_ORIGIN: process.env.CODEAPI_HOSTED_APP_PREVIEW_ORIGIN ?? '', + HOSTED_APP_PREVIEW_SIGNING_KEY: + process.env.CODEAPI_HOSTED_APP_PREVIEW_SIGNING_KEY ?? '', }; const default_run_memory_limit = 256 * 1024 * 1024; diff --git a/service/src/hosted-app/control-plane.test.ts b/service/src/hosted-app/control-plane.test.ts new file mode 100644 index 00000000..51d7e708 --- /dev/null +++ b/service/src/hosted-app/control-plane.test.ts @@ -0,0 +1,632 @@ +import { describe, expect, test } from 'bun:test'; +import { randomBytes } from 'node:crypto'; +import type { CheckpointStore } from '../runtime-session/checkpoint-store'; +import type { MicrovmDescription } from '../runtime-session/lambda-client'; +import type { RuntimeSessionRecord } from '../runtime-session/registry'; +import { + HostedAppControlPlane, + HostedAppControlPlaneError, + hostedAppPublicStatus, + type HostedAppRegistry, +} from './control-plane'; +import { openHostedAppCredential } from './credential'; +import { + hostedAppLaunchClientToken, + hostedAppLaunchFingerprint, + hostedAppLaunchGenerationSeed, + hostedAppLaunchRequestFingerprint, + HostedAppMicrovmError, + type HostedAppMicrovmConfig, + type HostedAppMicrovmRuntime, +} from './microvm-runtime'; +import { hostedAppSpecFingerprint, type ResidentHostedAppSpec } from './spec'; +import type { HostedAppRevision } from './record'; + +const appSpec: ResidentHostedAppSpec = { + adapter: 'resident', + app_id: 'demo', + revision: 'rev-1', + language: 'node', + version: '>=22', + entrypoint: 'server.js', + cwd: '.', + args: [], + env: {}, +}; + +const runtimeConfig: HostedAppMicrovmConfig = { + imageArn: 'arn:aws:lambda:us-east-2:1:microvm-image:app-host', + imageVersion: '7', + executionRoleArn: 'arn:aws:iam::1:role/app-host', + ingressConnectorArns: ['arn:ingress/private'], + controlPort: 8080, + previewPort: 3000, + maximumDurationSeconds: 28_800, + idleSeconds: 300, + suspendedSeconds: 900, + authTokenTtlSeconds: 3_600, + launchTimeoutMs: 5_000, + healthTimeoutMs: 500, + appStartTimeoutMs: 2_000, + launchTps: 4, + tokenTps: 8, +}; + +class MemoryRegistry implements HostedAppRegistry { + record: RuntimeSessionRecord | null = null; + generation = hostedAppLaunchGenerationSeed(runtimeConfig); + writes: RuntimeSessionRecord[] = []; + allocations = 0; + + async waitForLock(): Promise { return 'lock'; } + async renewLock(): Promise<'held'> { return 'held'; } + async releaseLock(): Promise {} + async read(): Promise { + return this.record ? structuredClone(this.record) : null; + } + async write(record: RuntimeSessionRecord, token: string): Promise { + if (token !== 'lock') return false; + this.record = structuredClone(record); + this.writes.push(structuredClone(record)); + return true; + } + async allocateGeneration(): Promise { + this.allocations += 1; + return this.generation; + } +} + +class FakeRuntime { + residentState: 'running' | 'failed' = 'running'; + async residentAppState() { return this.residentState; } + readonly config = runtimeConfig; + launches: string[] = []; + starts: Array<{ vm: string; source: string; spec: ResidentHostedAppSpec }> = []; + healthChecks: string[] = []; + terminations: string[] = []; + previewMints = 0; + previewExpiresAt = 1_900_000_000_000; + startedAtMs?: number; + terminateSucceeds = true; + launchError?: Error; + healthError?: Error; + + async launch(clientToken: string): Promise<{ vm: MicrovmDescription; clientToken: string }> { + this.launches.push(clientToken); + if (this.launchError) throw this.launchError; + return { + vm: { + microvmId: 'vm-app-1', + endpoint: 'https://vm-app-1.test', + state: 'RUNNING', + startedAtMs: this.startedAtMs, + imageArn: runtimeConfig.imageArn, + imageVersion: runtimeConfig.imageVersion, + }, + clientToken, + }; + } + async waitForControlReady(vm: MicrovmDescription): Promise { + this.healthChecks.push(vm.microvmId); + if (this.healthError) { + const error = this.healthError; + this.healthError = undefined; + throw error; + } + } + async startResidentApp( + vm: MicrovmDescription, + source: string, + spec: ResidentHostedAppSpec, + ): Promise { + this.starts.push({ vm: vm.microvmId, source, spec }); + } + async previewToken() { + this.previewMints += 1; + return { + headerName: 'X-aws-proxy-auth', + token: `secret-token-${this.previewMints}`, + expiresAtMs: this.previewExpiresAt, + }; + } + async terminate(microvmId: string): Promise { + this.terminations.push(microvmId); + return this.terminateSucceeds; + } +} + +function fixture(options: { + registry?: MemoryRegistry; + runtime?: FakeRuntime; + restore?: 'restored' | 'absent' | 'fetch_failed' | 'push_failed'; +} = {}) { + const registry = options.registry ?? new MemoryRegistry(); + const runtime = options.runtime ?? new FakeRuntime(); + const credentialKey = randomBytes(32); + const captures: string[] = []; + const restores: string[] = []; + const revisions = new Map(); + const control = new HostedAppControlPlane({ + registry, + runtime: runtime as unknown as HostedAppMicrovmRuntime, + checkpointStore: {} as CheckpointStore, + readRevision: async (_id, revision) => revisions.get(revision) ?? null, + retainRevision: async (_id, revision) => { + if (!revisions.has(revision.revision)) revisions.set(revision.revision, structuredClone(revision)); + return revisions.get(revision.revision)!; + }, + checkpointConfig: { + port: 8080, + authTokenTtlSeconds: 3_600, + maxBytes: 1024, + timeoutMs: 1_000, + }, + credentialKey, + lockWaitMs: 50, + lockTtlMs: 5_000, + captureCheckpoint: async source => { + captures.push(source); + return `rtsx-checkpoints/${source}/0001.tar.gz`; + }, + restoreCheckpoint: async args => { + restores.push(args.checkpointKey); + return options.restore ?? 'restored'; + }, + startHeartbeat: () => ({ stop() {} }), + now: () => 1_800_000_000_000, + }); + return { control, registry, runtime, credentialKey, captures, restores, revisions }; +} + +const input = { + hostedAppRuntimeId: 'happ_123', + sourceRuntimeSessionId: 'rt_source', + tenantId: 'tenant-1', + canonicalUserId: 'user-1', + spec: appSpec, + signal: new AbortController().signal, +}; + +test('reconciles process failure without forgetting the live VM needed for cleanup', async () => { + const f = fixture(); + await f.control.start(input); + expect((await f.control.status(input.hostedAppRuntimeId, input, input.signal)).state).toBe('running'); + f.runtime.residentState = 'failed'; + const status = await f.control.status(input.hostedAppRuntimeId, input, input.signal); + expect(status.state).toBe('failed'); + expect(status.preview_url).toBeUndefined(); + expect(f.registry.record!.microvm_id).toBe('vm-app-1'); + await expect(f.control.status(input.hostedAppRuntimeId, { ...input, canonicalUserId: 'intruder' }, input.signal)) + .rejects.toThrow('Hosted app not found'); + await f.control.stop(input.hostedAppRuntimeId, input, input.signal); + expect(f.runtime.terminations).toEqual(['vm-app-1']); +}); + +test('replaces a running VM after its launch policy changes', async () => { + const f = fixture(); + await f.control.start(input); + f.registry.record!.launch_fingerprint = 'obsolete-policy'; + await f.control.start(input); + expect(f.runtime.terminations).toEqual(['vm-app-1']); + expect(f.runtime.launches).toHaveLength(2); + expect(f.captures).toHaveLength(1); + expect(f.restores[1]).toBe(f.restores[0]); +}); + +test('retains immutable revision bytes and settings after Redis lease expiry and revision switches', async () => { + const f = fixture(); + await f.control.start(input); + const checkpoint = f.restores[0]; + f.registry.record = null; // Expired ephemeral lease; durable manifest survives. + await f.control.start(input); + expect(f.captures).toHaveLength(1); + expect(f.restores.at(-1)).toBe(checkpoint); + f.registry.record = null; + await expect(f.control.start({ ...input, spec: { ...appSpec, args: ['changed'] } })) + .rejects.toThrow('immutable'); + await f.control.start({ ...input, spec: { ...appSpec, revision: 'rev-2' } }); + await f.control.start(input); + expect(f.captures).toHaveLength(2); + expect(f.restores.at(-1)).toBe(checkpoint); +}); + +test('settles definite boot exhaustion during stop without replaying dead intents', async () => { + const f = fixture(); + f.registry.record = pendingRecord(); + f.runtime.launchError = new HostedAppMicrovmError('hosted_app_boot_failed', 'both attempts dead', true); + expect((await f.control.stop(input.hostedAppRuntimeId, input, input.signal)).state).toBe('stopped'); + expect(f.registry.record!.state).toBe('TERMINATED'); + await f.control.stop(input.hostedAppRuntimeId, input, input.signal); + expect(f.runtime.launches).toHaveLength(1); +}); + +test('keeps a healthy VM through preview and control credential outages', async () => { + const f = fixture(); + await f.control.start(input); + const original = structuredClone(f.registry.record); + f.runtime.previewToken = async () => { throw new HostedAppMicrovmError('hosted_app_auth_failed', 'throttled', true); }; + await expect(f.control.start(input)).rejects.toThrow('throttled'); + expect(f.runtime.terminations).toHaveLength(0); + expect(f.runtime.launches).toHaveLength(1); + expect(f.registry.record).toEqual(original); + f.runtime.healthError = new HostedAppMicrovmError('hosted_app_auth_failed', 'control auth unavailable', true); + await expect(f.control.start(input)).rejects.toThrow('control auth unavailable'); + expect(f.runtime.terminations).toHaveLength(0); +}); + +test('does not report an expired pending intent as starting', () => { + expect(hostedAppPublicStatus({ ...pendingRecord(), hard_deadline_at: 100 }, 101).state) + .toBe('failed'); +}); + +function pendingRecord(): RuntimeSessionRecord { + const generation = hostedAppLaunchGenerationSeed(runtimeConfig); + return { + runtime_session_id: input.hostedAppRuntimeId, + tenant_id: input.tenantId, + canonical_user_id: input.canonicalUserId, + state: 'PENDING', + generation, + launched_at: 1_799_999_000_000, + hard_deadline_at: 1_800_010_000_000, + last_seen_at: 1_799_999_000_000, + image_arn: runtimeConfig.imageArn, + image_version: runtimeConfig.imageVersion, + port: runtimeConfig.previewPort, + launch_fingerprint: hostedAppLaunchFingerprint(runtimeConfig), + launch_request_fingerprint: hostedAppLaunchRequestFingerprint(runtimeConfig), + launch_client_token: hostedAppLaunchClientToken(input.hostedAppRuntimeId, generation), + hosted_app: { + source_runtime_session_id: input.sourceRuntimeSessionId, + app_id: appSpec.app_id, + revision: appSpec.revision, + spec_fingerprint: hostedAppSpecFingerprint(appSpec), + spec: appSpec, + checkpoint_key: 'rtsx-checkpoints/rt_source/exact.tar.gz', + }, + }; +} + +describe('HostedAppControlPlane', () => { + test('does not advertise an expired AWS lease as a running preview', () => { + const expired = { + ...pendingRecord(), + state: 'RUNNING' as const, + microvm_id: 'vm-expired', + endpoint: 'https://vm-expired.test', + hard_deadline_at: 99, + }; + expect(hostedAppPublicStatus(expired, 100).state).toBe('stopped'); + }); + + test('does not expose provider details persisted in an internal failure record', () => { + const failed = { + ...pendingRecord(), + state: 'TERMINATED' as const, + last_error: 'AccessDenied for arn:aws:iam::123456789012:role/private', + }; + const status = hostedAppPublicStatus(failed); + expect(status.state).toBe('failed'); + expect(status.error).toBe('Hosted app operation failed'); + expect(JSON.stringify(status)).not.toContain('123456789012'); + }); + + test('checkpoints, launches, restores, starts, and persists only a sealed preview credential', async () => { + const f = fixture(); + + const status = await f.control.start(input); + + expect(status).toMatchObject({ state: 'running', preview_id: 'happ_123', revision: 'rev-1' }); + expect(f.captures).toEqual(['rt_source']); + expect(f.restores).toEqual(['rtsx-checkpoints/rt_source/0001.tar.gz']); + expect(f.runtime.starts).toHaveLength(1); + expect(f.registry.writes.map(record => record.state)).toEqual(['PENDING', 'PENDING', 'RUNNING']); + expect(JSON.stringify(f.registry.record)).not.toContain('secret-token-1'); + const sealed = f.registry.record?.hosted_app?.preview_credential as string; + expect(openHostedAppCredential('happ_123', sealed, f.credentialKey).token).toBe('secret-token-1'); + }); + + test('derives the advertised lease deadline from the provider start time', async () => { + const runtime = new FakeRuntime(); + runtime.startedAtMs = 1_800_000_000_500; + const f = fixture({ runtime }); + + await f.control.start(input); + + expect(f.registry.record?.launched_at).toBe(runtime.startedAtMs); + expect(f.registry.record?.hard_deadline_at).toBe( + runtime.startedAtMs + runtimeConfig.maximumDurationSeconds * 1_000 - 60_000, + ); + }); + + test('replays an exact pending launch intent without taking a different checkpoint', async () => { + const registry = new MemoryRegistry(); + registry.record = pendingRecord(); + const f = fixture({ registry }); + + await f.control.start(input); + + expect(f.captures).toEqual([]); + expect(registry.allocations).toBe(0); + expect(f.runtime.launches).toEqual([pendingRecord().launch_client_token as string]); + expect(f.restores).toEqual(['rtsx-checkpoints/rt_source/exact.tar.gz']); + }); + + test('rejects changed launch settings under an immutable revision', async () => { + const registry = new MemoryRegistry(); + registry.record = pendingRecord(); + const f = fixture({ registry }); + const changed = { ...input, spec: { ...appSpec, args: ['--changed'] } }; + + const error = await f.control.start(changed).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppControlPlaneError); + expect(error.code).toBe('hosted_app_revision_conflict'); + expect(f.runtime.launches).toEqual([]); + }); + + test('does not overwrite an ambiguous pending provider launch with a new revision', async () => { + const registry = new MemoryRegistry(); + registry.record = pendingRecord(); + const f = fixture({ registry }); + + const error = await f.control.start({ + ...input, + spec: { ...appSpec, revision: 'rev-2' }, + }).catch(value => value); + + expect(error.code).toBe('hosted_app_launch_in_progress'); + expect(f.captures).toEqual([]); + expect(registry.allocations).toBe(0); + expect(f.runtime.launches).toEqual([]); + }); + + test('reasserts an exact running revision and rotates its preview credential', async () => { + const registry = new MemoryRegistry(); + registry.record = { + ...pendingRecord(), + state: 'RUNNING', + microvm_id: 'vm-existing', + endpoint: 'https://vm-existing.test', + }; + const f = fixture({ registry }); + + await f.control.start(input); + + expect(f.captures).toEqual([]); + expect(f.runtime.launches).toEqual([]); + expect(f.runtime.healthChecks).toEqual(['vm-existing']); + expect(f.runtime.starts).toHaveLength(1); + expect(f.runtime.previewMints).toBe(1); + }); + + test('recycles a dead app VM and restores the exact immutable revision checkpoint', async () => { + const registry = new MemoryRegistry(); + registry.record = { + ...pendingRecord(), + state: 'RUNNING', + microvm_id: 'vm-dead', + endpoint: 'https://vm-dead.test', + }; + const runtime = new FakeRuntime(); + runtime.healthError = new HostedAppMicrovmError( + 'hosted_app_unhealthy', + 'endpoint is gone', + true, + ); + const f = fixture({ registry, runtime }); + + await f.control.start(input); + + expect(f.captures).toEqual([]); + expect(runtime.terminations).toEqual(['vm-dead']); + expect(runtime.launches).toHaveLength(1); + expect(f.restores).toEqual(['rtsx-checkpoints/rt_source/exact.tar.gz']); + }); + + test('records replacement cleanup before terminating the prior revision', async () => { + const registry = new MemoryRegistry(); + registry.record = { + ...pendingRecord(), + state: 'RUNNING', + microvm_id: 'vm-old-revision', + endpoint: 'https://vm-old-revision.test', + }; + registry.allocateGeneration = async () => { + throw new Error('generation store unavailable'); + }; + const runtime = new FakeRuntime(); + const f = fixture({ registry, runtime }); + + await f.control.start({ + ...input, + spec: { ...appSpec, revision: 'rev-2' }, + }).catch(() => undefined); + + expect(runtime.terminations).toEqual(['vm-old-revision']); + expect(registry.record).toMatchObject({ + state: 'TERMINATING', + microvm_id: 'vm-old-revision', + endpoint: 'https://vm-old-revision.test', + }); + }); + + test('terminates and retires a VM whose exact checkpoint cannot be restored', async () => { + const f = fixture({ restore: 'fetch_failed' }); + + const error = await f.control.start(input).catch(value => value); + + expect(error.code).toBe('hosted_app_restore_failed'); + expect(f.runtime.terminations).toEqual(['vm-app-1']); + expect(f.registry.record).toMatchObject({ state: 'TERMINATED', microvm_id: undefined }); + }); + + test('retains a failed launch VM id until termination can be confirmed', async () => { + const runtime = new FakeRuntime(); + runtime.terminateSucceeds = false; + const f = fixture({ restore: 'push_failed', runtime }); + + await f.control.start(input).catch(() => undefined); + + expect(f.registry.record).toMatchObject({ + state: 'TERMINATING', + microvm_id: 'vm-app-1', + endpoint: 'https://vm-app-1.test', + }); + }); + + test('a failed stop keeps the possibly-live app running and retryable', async () => { + const registry = new MemoryRegistry(); + registry.record = { + ...pendingRecord(), + state: 'RUNNING', + microvm_id: 'vm-existing', + endpoint: 'https://vm-existing.test', + }; + const runtime = new FakeRuntime(); + runtime.terminateSucceeds = false; + const f = fixture({ registry, runtime }); + + const error = await f.control.stop( + input.hostedAppRuntimeId, + input, + input.signal, + ).catch(value => value); + + expect(error.code).toBe('hosted_app_stop_failed'); + expect(registry.record).toMatchObject({ + state: 'RUNNING', + microvm_id: 'vm-existing', + last_error: 'Could not terminate the hosted app', + }); + }); + + test('stop replays an ambiguous pending launch before terminating it', async () => { + const registry = new MemoryRegistry(); + registry.record = pendingRecord(); + const f = fixture({ registry }); + + const status = await f.control.stop(input.hostedAppRuntimeId, input, input.signal); + + expect(status.state).toBe('stopped'); + expect(f.runtime.launches).toEqual([pendingRecord().launch_client_token as string]); + expect(f.runtime.terminations).toEqual(['vm-app-1']); + expect(registry.writes.map(record => record.state)).toEqual([ + 'PENDING', + 'TERMINATING', + 'TERMINATED', + ]); + expect(registry.record).toMatchObject({ + state: 'TERMINATED', + microvm_id: undefined, + endpoint: undefined, + }); + }); + + test('stop preserves an ambiguous pending intent when recovery fails', async () => { + const registry = new MemoryRegistry(); + registry.record = pendingRecord(); + const runtime = new FakeRuntime(); + runtime.launchError = new HostedAppMicrovmError( + 'hosted_app_launch_failed', + 'connection reset after provider accepted the request', + true, + ); + const f = fixture({ registry, runtime }); + + const error = await f.control.stop(input.hostedAppRuntimeId, input, input.signal) + .catch(value => value); + + expect(error.code).toBe('hosted_app_launch_failed'); + expect(registry.writes).toEqual([]); + expect(registry.record).toEqual(pendingRecord()); + }); + + test('stop does not overwrite a pending intent that current config cannot replay', async () => { + const registry = new MemoryRegistry(); + registry.record = { ...pendingRecord(), launch_fingerprint: 'different-image' }; + const f = fixture({ registry }); + + const error = await f.control.stop(input.hostedAppRuntimeId, input, input.signal) + .catch(value => value); + + expect(error.code).toBe('hosted_app_stop_pending'); + expect(error.transient).toBe(true); + expect(f.runtime.launches).toEqual([]); + expect(registry.writes).toEqual([]); + }); + + test('a failed cleanup never promotes a partial launch to running', async () => { + const registry = new MemoryRegistry(); + registry.record = { + ...pendingRecord(), + state: 'TERMINATING', + microvm_id: 'vm-partial', + endpoint: 'https://vm-partial.test', + }; + const runtime = new FakeRuntime(); + runtime.terminateSucceeds = false; + const f = fixture({ registry, runtime }); + + await f.control.stop(input.hostedAppRuntimeId, input, input.signal) + .catch(() => undefined); + + expect(registry.record).toMatchObject({ + state: 'TERMINATING', + microvm_id: 'vm-partial', + }); + }); + + test('retires definite boot exhaustion but preserves an ambiguous launch intent', async () => { + const definiteRuntime = new FakeRuntime(); + definiteRuntime.launchError = new HostedAppMicrovmError( + 'hosted_app_boot_failed', + 'both attempts terminated', + true, + ); + const definite = fixture({ runtime: definiteRuntime }); + await definite.control.start(input).catch(() => undefined); + expect(definite.registry.record).toMatchObject({ state: 'TERMINATED' }); + + const ambiguousRuntime = new FakeRuntime(); + ambiguousRuntime.launchError = new HostedAppMicrovmError( + 'hosted_app_launch_failed', + 'connection reset after write', + true, + ); + const ambiguous = fixture({ runtime: ambiguousRuntime }); + await ambiguous.control.start(input).catch(() => undefined); + expect(ambiguous.registry.record?.state).toBe('PENDING'); + expect(ambiguous.registry.record?.microvm_id).toBeUndefined(); + }); + + test('does not refresh a preview after its advertised hard deadline', async () => { + const registry = new MemoryRegistry(); + registry.record = { + ...pendingRecord(), + state: 'RUNNING', + microvm_id: 'vm-existing', + endpoint: 'https://vm-existing.test', + hard_deadline_at: 1_800_000_000_000, + }; + const f = fixture({ registry }); + + const error = await f.control.refreshPreview(input.hostedAppRuntimeId, input, input.signal) + .catch(value => value); + + expect(error.code).toBe('hosted_app_not_running'); + expect(f.runtime.previewMints).toBe(0); + }); + + test('rejects an already-expired credential instead of publishing it', async () => { + const runtime = new FakeRuntime(); + runtime.previewExpiresAt = 1_800_000_000_000; + const f = fixture({ runtime }); + + const error = await f.control.start(input).catch(value => value); + + expect(error.code).toBe('hosted_app_preview_unavailable'); + expect(runtime.terminations).toEqual(['vm-app-1']); + expect(f.registry.record).toMatchObject({ state: 'TERMINATED' }); + }); +}); diff --git a/service/src/hosted-app/control-plane.ts b/service/src/hosted-app/control-plane.ts new file mode 100644 index 00000000..728f4aaf --- /dev/null +++ b/service/src/hosted-app/control-plane.ts @@ -0,0 +1,748 @@ +import type { CheckpointConfig } from '../runtime-session/checkpoint'; +import type { CheckpointStore } from '../runtime-session/checkpoint-store'; +import type { MicrovmDescription } from '../runtime-session/lambda-client'; +import type { RuntimeSessionRecord } from '../runtime-session/registry'; +import type { LockHeartbeat } from '../runtime-session/lock-heartbeat'; +import { sealHostedAppCredential } from './credential'; +import { + hostedAppLaunchClientToken, + hostedAppLaunchFingerprint, + hostedAppLaunchGenerationSeed, + hostedAppLaunchRequestFingerprint, + HostedAppMicrovmError, + type HostedAppMicrovmRuntime, +} from './microvm-runtime'; +import type { HostedAppPublicStatus, HostedAppRevision } from './record'; +import { + hostedAppSpecFingerprint, + type ResidentHostedAppSpec, +} from './spec'; + +const HOSTED_APP_DEADLINE_HEADROOM_MS = 60_000; + +export class HostedAppControlPlaneError extends Error { + constructor( + readonly code: string, + message: string, + readonly status: number, + readonly transient = false, + readonly cause?: unknown, + ) { + super(message); + this.name = 'HostedAppControlPlaneError'; + } +} + +export interface HostedAppOwner { + tenantId: string; + canonicalUserId: string; +} + +export interface HostedAppStartInput extends HostedAppOwner { + hostedAppRuntimeId: string; + sourceRuntimeSessionId: string; + spec: ResidentHostedAppSpec; + signal: AbortSignal; +} + +export interface HostedAppRegistry { + waitForLock(runtimeId: string, args: { + waitMs: number; + ttlMs: number; + signal: AbortSignal; + }): Promise; + renewLock(runtimeId: string, token: string, ttlMs: number, args: { + signal: AbortSignal; + onLateLost: () => void; + }): Promise<'held' | 'lost' | 'error'>; + releaseLock(runtimeId: string, token: string): Promise; + read(runtimeId: string, args: { signal?: AbortSignal }): Promise; + write(record: RuntimeSessionRecord, token: string, args: { + signal?: AbortSignal; + }): Promise; + allocateGeneration(runtimeId: string, seed: number, args: { + signal: AbortSignal; + }): Promise; +} + +export interface HostedAppControlPlaneDeps { + registry: HostedAppRegistry; + runtime: HostedAppMicrovmRuntime; + checkpointStore: CheckpointStore; + checkpointConfig: CheckpointConfig; + credentialKey: Buffer; + lockWaitMs: number; + lockTtlMs: number; + captureCheckpoint( + runtimeSessionId: string, + owner: HostedAppOwner, + signal: AbortSignal, + ): Promise; + restoreCheckpoint(args: { + runtimeSessionId: string; + checkpointKey: string; + vm: MicrovmDescription; + store: CheckpointStore; + config: CheckpointConfig; + signal: AbortSignal; + }): Promise<'restored' | 'absent' | 'fetch_failed' | 'push_failed'>; + startHeartbeat(args: { + renew: () => Promise<'held' | 'lost' | 'error'>; + fence: AbortController; + ttlMs: number; + }): LockHeartbeat; + now?: () => number; + readRevision(runtimeId: string, revision: string): Promise; + retainRevision(runtimeId: string, revision: HostedAppRevision): Promise; +} + +function publicState( + record: RuntimeSessionRecord, + now: number, +): HostedAppPublicStatus['state'] { + if ( + record.state === 'RUNNING' + && record.hard_deadline_at != null + && record.hard_deadline_at <= now + ) return 'stopped'; + if (record.state === 'RUNNING') return 'running'; + if (record.state === 'PENDING') { + return record.hard_deadline_at != null && record.hard_deadline_at <= now + ? 'failed' : 'starting'; + } + if (record.state === 'TERMINATED') { + return record.last_error ? 'failed' : 'stopped'; + } + if (record.state === 'TERMINATING') return 'stopping'; + return 'starting'; +} + +export function hostedAppPublicStatus( + record: RuntimeSessionRecord, + now = Date.now(), +): HostedAppPublicStatus { + const app = record.hosted_app; + if (!app) { + throw new HostedAppControlPlaneError( + 'hosted_app_record_invalid', + 'Hosted app registry record is missing app metadata', + 503, + true, + ); + } + return { + app_id: app.app_id, + revision: app.revision, + state: publicState(record, now), + preview_id: record.runtime_session_id, + hard_deadline_at: record.hard_deadline_at, + updated_at: record.last_seen_at, + ...(record.last_error ? { + /* Provider, endpoint, and checkpoint errors stay in the internal record + * and worker logs. Status is a public API and must not replay them. */ + error: record.state === 'TERMINATING' + ? 'Hosted app cleanup is pending' + : record.state === 'RUNNING' + ? 'Hosted app could not be stopped' + : 'Hosted app operation failed', + } : {}), + }; +} + +export function assertHostedAppOwned( + record: RuntimeSessionRecord, + owner: HostedAppOwner, + sourceRuntimeSessionId?: string, +): void { + if ( + record.tenant_id !== owner.tenantId + || record.canonical_user_id !== owner.canonicalUserId + || (sourceRuntimeSessionId != null + && record.hosted_app?.source_runtime_session_id !== sourceRuntimeSessionId) + ) { + /* Do not disclose whether another owner's opaque id exists. */ + throw new HostedAppControlPlaneError('hosted_app_not_found', 'Hosted app not found', 404); + } +} + +export class HostedAppControlPlane { + private readonly now: () => number; + + constructor(private readonly deps: HostedAppControlPlaneDeps) { + this.now = deps.now ?? Date.now; + } + + async status(runtimeId: string, owner: HostedAppOwner, callerSignal: AbortSignal): Promise { + return this.withLease(runtimeId, callerSignal, async signal => { + const record = await this.deps.registry.read(runtimeId, { signal }); + if (!record?.hosted_app) { + throw new HostedAppControlPlaneError('hosted_app_not_found', 'Hosted app not found', 404); + } + assertHostedAppOwned(record, owner); + const status = hostedAppPublicStatus(record, this.now()); + if (status.state !== 'running') return status; + const state = await this.deps.runtime.residentAppState(this.recordedVm(record), + record.hosted_app.source_runtime_session_id, record.hosted_app.spec, signal); + signal.throwIfAborted(); + const current = hostedAppPublicStatus(record, this.now()); + if (current.state !== 'running') return current; + // Process failure is not VM termination: keep its durable identity so stop + // can clean it up and start can reassert the resident process. + return { ...current, state, ...(state === 'failed' ? { error: 'Hosted app process failed' } : {}) }; + }); + } + + async start(input: HostedAppStartInput): Promise { + return this.withLease(input.hostedAppRuntimeId, input.signal, async (signal, lockToken) => { + const fingerprint = hostedAppSpecFingerprint(input.spec); + let revision = await this.deps.readRevision(input.hostedAppRuntimeId, input.spec.revision); + const assertRevision = (value: HostedAppRevision): void => { + if (value.tenantId !== input.tenantId || value.canonicalUserId !== input.canonicalUserId + || value.sourceRuntimeSessionId !== input.sourceRuntimeSessionId) { + throw new HostedAppControlPlaneError('hosted_app_not_found', 'Hosted app not found', 404); + } + if (value.revision !== input.spec.revision || value.specFingerprint !== fingerprint) { + throw new HostedAppControlPlaneError('hosted_app_revision_conflict', + 'An app revision is immutable; use a new revision for changed launch settings', 409); + } + }; + if (revision) assertRevision(revision); + let prior = await this.deps.registry.read(input.hostedAppRuntimeId, { signal }); + if (prior) { + assertHostedAppOwned(prior, input, input.sourceRuntimeSessionId); + if ( + prior.hosted_app?.revision === input.spec.revision + && prior.hosted_app.spec_fingerprint !== fingerprint + ) { + throw new HostedAppControlPlaneError( + 'hosted_app_revision_conflict', + 'An app revision is immutable; use a new revision for changed launch settings', + 409, + ); + } + } + + const exactRevision = prior?.hosted_app?.revision === input.spec.revision + && prior.hosted_app.spec_fingerprint === fingerprint; + // Migrate existing experimental records before the running fast path. + if (!revision && exactRevision && prior?.hosted_app) { + revision = await this.deps.retainRevision(input.hostedAppRuntimeId, { + tenantId: input.tenantId, canonicalUserId: input.canonicalUserId, + sourceRuntimeSessionId: input.sourceRuntimeSessionId, revision: input.spec.revision, + specFingerprint: fingerprint, checkpointKey: prior.hosted_app.checkpoint_key, + }); + assertRevision(revision); + } + if ( + exactRevision + && prior?.state === 'RUNNING' + && prior.launch_fingerprint === hostedAppLaunchFingerprint(this.deps.runtime.config) + && prior.microvm_id + && prior.endpoint + && (prior.hard_deadline_at == null + || prior.hard_deadline_at > this.now() + HOSTED_APP_DEADLINE_HEADROOM_MS) + ) { + /* Reassert both the runner and credential. This resumes a suspended VM, + * heals a dead resident process, and never trusts an expired token. */ + const vm = this.recordedVm(prior); + try { + await this.deps.runtime.waitForControlReady(vm, signal); + await this.deps.runtime.startResidentApp( + vm, + input.sourceRuntimeSessionId, + input.spec, + signal, + ); + } catch (error) { + if (!(error instanceof HostedAppMicrovmError) || !error.transient + || !['hosted_app_unhealthy', 'hosted_app_start_unavailable', 'hosted_app_start_failed'].includes(error.code)) throw error; + const terminating: RuntimeSessionRecord = { + ...prior, + state: 'TERMINATING', + last_seen_at: this.now(), + }; + /* Record the destructive transition before acting on AWS. If the + * subsequent generation allocation or Redis write fails, callers see + * a recoverable cleanup state rather than a stale RUNNING endpoint + * for a VM we already terminated. */ + await this.writeOrFence(terminating, lockToken, signal); + const terminated = await this.deps.runtime.terminate(vm.microvmId); + if (!terminated) throw error; + prior = { + ...terminating, + microvm_id: undefined, + endpoint: undefined, + state: 'TERMINATED', + last_seen_at: this.now(), + last_error: error.message, + }; + } + if (prior.state === 'RUNNING') { + // Credential/control-plane failures do not prove the VM is unhealthy. + return hostedAppPublicStatus( + await this.persistPreviewCredential(prior, vm, lockToken, signal), this.now(), + ); + } + } + + const replayPending = Boolean( + exactRevision + && prior?.state === 'PENDING' + && !prior.microvm_id + && prior.launch_client_token + && prior.hosted_app?.checkpoint_key + && prior.launch_fingerprint === hostedAppLaunchFingerprint(this.deps.runtime.config) + && prior.launch_request_fingerprint + === hostedAppLaunchRequestFingerprint(this.deps.runtime.config) + && prior.hard_deadline_at != null + && prior.hard_deadline_at > this.now() + HOSTED_APP_DEADLINE_HEADROOM_MS + ); + const pendingProviderCouldStillBeLive = Boolean( + prior?.state === 'PENDING' + && !prior.microvm_id + && ( + prior.hard_deadline_at == null + || prior.hard_deadline_at + + HOSTED_APP_DEADLINE_HEADROOM_MS + + this.deps.runtime.config.launchTimeoutMs > this.now() + ) + ); + if ( + pendingProviderCouldStillBeLive + && !replayPending + ) { + /* A provider call may have succeeded before its response was lost. We + * can recover only by replaying the exact revision/config token; never + * overwrite that intent with a different revision and orphan a VM. */ + throw new HostedAppControlPlaneError( + 'hosted_app_launch_in_progress', + 'The prior hosted app launch must be recovered before it can be replaced', + 409, + true, + ); + } + /* An exact revision always reuses its immutable source snapshot. A dead + * app VM must not silently pick up later workspace edits under the same + * revision; changed bytes require a new revision. */ + if (!revision) { + const checkpointKey = await this.deps.captureCheckpoint(input.sourceRuntimeSessionId, input, signal); + revision = await this.deps.retainRevision(input.hostedAppRuntimeId, { + tenantId: input.tenantId, canonicalUserId: input.canonicalUserId, + sourceRuntimeSessionId: input.sourceRuntimeSessionId, revision: input.spec.revision, + specFingerprint: fingerprint, checkpointKey, + }); + assertRevision(revision); + } + const checkpointKey = revision.checkpointKey; + signal.throwIfAborted(); + + if (prior?.microvm_id) { + const terminating: RuntimeSessionRecord = { + ...prior, + state: 'TERMINATING', + last_seen_at: this.now(), + }; + await this.writeOrFence(terminating, lockToken, signal); + const terminated = await this.deps.runtime.terminate(prior.microvm_id); + if (!terminated) { + throw new HostedAppControlPlaneError( + 'hosted_app_replace_failed', + 'Could not terminate the previous hosted app revision', + 503, + true, + ); + } + prior = terminating; + } + + const generation = replayPending && prior + ? prior.generation + : await this.deps.registry.allocateGeneration( + input.hostedAppRuntimeId, + hostedAppLaunchGenerationSeed(this.deps.runtime.config), + { signal }, + ); + const clientToken = replayPending && prior?.launch_client_token + ? prior.launch_client_token + : hostedAppLaunchClientToken(input.hostedAppRuntimeId, generation); + const launchedAt = replayPending && prior?.launched_at + ? prior.launched_at + : this.now(); + const hardDeadlineAt = replayPending && prior?.hard_deadline_at + ? prior.hard_deadline_at + : launchedAt + + this.deps.runtime.config.maximumDurationSeconds * 1_000 + - HOSTED_APP_DEADLINE_HEADROOM_MS; + let launchIntent: RuntimeSessionRecord = { + runtime_session_id: input.hostedAppRuntimeId, + tenant_id: input.tenantId, + canonical_user_id: input.canonicalUserId, + port: this.deps.runtime.config.previewPort, + image_arn: this.deps.runtime.config.imageArn, + image_version: this.deps.runtime.config.imageVersion, + launch_fingerprint: hostedAppLaunchFingerprint(this.deps.runtime.config), + launch_client_token: clientToken, + launch_request_fingerprint: hostedAppLaunchRequestFingerprint(this.deps.runtime.config), + state: 'PENDING', + generation, + launched_at: launchedAt, + last_seen_at: this.now(), + hard_deadline_at: hardDeadlineAt, + hosted_app: { + source_runtime_session_id: input.sourceRuntimeSessionId, + app_id: input.spec.app_id, + revision: input.spec.revision, + spec_fingerprint: fingerprint, + spec: input.spec, + checkpoint_key: checkpointKey, + }, + }; + await this.writeOrFence(launchIntent, lockToken, signal); + + let vm: MicrovmDescription | undefined; + try { + const launched = await this.deps.runtime.launch(clientToken, signal); + vm = launched.vm; + const providerStartedAt = Number.isFinite(vm.startedAtMs) + ? vm.startedAtMs + : undefined; + launchIntent = { + ...launchIntent, + launch_client_token: launched.clientToken, + microvm_id: vm.microvmId, + endpoint: vm.endpoint, + image_arn: vm.imageArn ?? launchIntent.image_arn, + image_version: vm.imageVersion ?? launchIntent.image_version, + launched_at: providerStartedAt ?? launchIntent.launched_at, + hard_deadline_at: providerStartedAt == null + ? launchIntent.hard_deadline_at + : providerStartedAt + + this.deps.runtime.config.maximumDurationSeconds * 1_000 + - HOSTED_APP_DEADLINE_HEADROOM_MS, + last_seen_at: this.now(), + }; + await this.writeOrFence(launchIntent, lockToken, signal); + await this.deps.runtime.waitForControlReady(vm, signal); + const restored = await this.deps.restoreCheckpoint({ + runtimeSessionId: input.sourceRuntimeSessionId, + checkpointKey, + vm, + store: this.deps.checkpointStore, + config: this.deps.checkpointConfig, + signal, + }); + if (restored !== 'restored') { + throw new HostedAppControlPlaneError( + 'hosted_app_restore_failed', + `Could not restore the exact app workspace revision (${restored})`, + 503, + true, + ); + } + await this.deps.runtime.startResidentApp( + vm, + input.sourceRuntimeSessionId, + input.spec, + signal, + ); + const running: RuntimeSessionRecord = { + ...launchIntent, + state: 'RUNNING', + last_seen_at: this.now(), + }; + return hostedAppPublicStatus( + await this.persistPreviewCredential(running, vm, lockToken, signal), + this.now(), + ); + } catch (error) { + const terminated = vm ? await this.deps.runtime.terminate(vm.microvmId) : false; + /* Preserve a no-id PENDING intent after an ambiguous provider failure: + * the successor replays the same token. Once a VM id is known, it was + * terminated above and the intent must be retired. */ + if (vm) { + const failed: RuntimeSessionRecord = { + ...launchIntent, + microvm_id: terminated ? undefined : vm.microvmId, + endpoint: terminated ? undefined : vm.endpoint, + state: terminated ? 'TERMINATED' : 'TERMINATING', + last_seen_at: this.now(), + last_error: terminated + ? (error instanceof Error ? error.message : 'Hosted app launch failed') + : 'Hosted app launch failed and its MicroVM still requires termination', + }; + await this.deps.registry.write(failed, lockToken, { signal }) + .catch(() => false); + } else if ( + error instanceof HostedAppMicrovmError + && error.code === 'hosted_app_boot_failed' + ) { + /* No VM id escaped launch(), and both boot attempts reached a + * terminal state. A rejected replay alone would not prove an earlier + * request was never admitted. Let the next request allocate a + * fresh generation instead of replaying dead tokens forever. */ + await this.deps.registry.write({ + ...launchIntent, + state: 'TERMINATED', + last_seen_at: this.now(), + last_error: error.message, + }, lockToken, { signal }).catch(() => false); + } + throw error; + } + }); + } + + async stop( + hostedAppRuntimeId: string, + owner: HostedAppOwner, + signal: AbortSignal, + ): Promise { + return this.withLease(hostedAppRuntimeId, signal, async (leaseSignal, lockToken) => { + let record = await this.deps.registry.read(hostedAppRuntimeId, { signal: leaseSignal }); + if (!record?.hosted_app) { + throw new HostedAppControlPlaneError('hosted_app_not_found', 'Hosted app not found', 404); + } + assertHostedAppOwned(record, owner); + if ( + record.state === 'PENDING' + && !record.microvm_id + && ( + record.hard_deadline_at == null + || record.hard_deadline_at + + HOSTED_APP_DEADLINE_HEADROOM_MS + + this.deps.runtime.config.launchTimeoutMs > this.now() + ) + ) { + const replayable = Boolean( + record.launch_client_token + && record.launch_fingerprint === hostedAppLaunchFingerprint(this.deps.runtime.config) + && record.launch_request_fingerprint + === hostedAppLaunchRequestFingerprint(this.deps.runtime.config) + ); + if (!replayable) { + /* The provider may still have accepted this request. Retain the + * intent until its maximum provider lifetime passes rather than + * claiming it was stopped and losing the only safe recovery key. */ + throw new HostedAppControlPlaneError( + 'hosted_app_stop_pending', + 'The pending hosted app launch must be recovered before it can be stopped', + 409, + true, + ); + } + const launched = await this.deps.runtime.launch( + record.launch_client_token as string, + leaseSignal, + ).catch(error => { + if (error instanceof HostedAppMicrovmError && error.code === 'hosted_app_boot_failed') return undefined; + throw error; // Ambiguous outcomes must keep the pending intent. + }); + if (launched) { + const recovered: RuntimeSessionRecord = { + ...record, + launch_client_token: launched.clientToken, + microvm_id: launched.vm.microvmId, + endpoint: launched.vm.endpoint, + image_arn: launched.vm.imageArn ?? record.image_arn, + image_version: launched.vm.imageVersion ?? record.image_version, + launched_at: Number.isFinite(launched.vm.startedAtMs) + ? launched.vm.startedAtMs + : record.launched_at, + hard_deadline_at: Number.isFinite(launched.vm.startedAtMs) + ? (launched.vm.startedAtMs as number) + + this.deps.runtime.config.maximumDurationSeconds * 1_000 + - HOSTED_APP_DEADLINE_HEADROOM_MS + : record.hard_deadline_at, + last_seen_at: this.now(), + }; + try { + await this.writeOrFence(recovered, lockToken, leaseSignal); + } catch (error) { + /* A recovered VM must never escape merely because we lost the Redis + * fence while recording its id. */ + await this.deps.runtime.terminate(launched.vm.microvmId).catch(() => false); + throw error; + } + record = recovered; + } + } + if (record.microvm_id) { + const terminating: RuntimeSessionRecord = { + ...record, + state: 'TERMINATING', + last_seen_at: this.now(), + }; + await this.writeOrFence(terminating, lockToken, leaseSignal); + if (!await this.deps.runtime.terminate(record.microvm_id)) { + await this.writeOrFence({ + ...record, + state: record.state === 'RUNNING' ? 'RUNNING' : 'TERMINATING', + last_seen_at: this.now(), + last_error: 'Could not terminate the hosted app', + }, lockToken, leaseSignal); + throw new HostedAppControlPlaneError( + 'hosted_app_stop_failed', + 'Could not terminate the hosted app', + 503, + true, + ); + } + } + const stopped: RuntimeSessionRecord = { + ...record, + microvm_id: undefined, + endpoint: undefined, + state: 'TERMINATED', + last_seen_at: this.now(), + last_error: undefined, + hosted_app: { + ...(record.hosted_app as NonNullable), + preview_credential: undefined, + preview_credential_expires_at: undefined, + }, + }; + await this.writeOrFence(stopped, lockToken, leaseSignal); + return hostedAppPublicStatus(stopped, this.now()); + }); + } + + async refreshPreview( + hostedAppRuntimeId: string, + owner: HostedAppOwner, + signal: AbortSignal, + ): Promise { + return this.withLease(hostedAppRuntimeId, signal, async (leaseSignal, lockToken) => { + const record = await this.deps.registry.read(hostedAppRuntimeId, { signal: leaseSignal }); + if (!record?.hosted_app) { + throw new HostedAppControlPlaneError( + 'hosted_app_not_running', + 'Hosted app is not running', + 409, + true, + ); + } + assertHostedAppOwned(record, owner); + if ( + record.state !== 'RUNNING' + || !record.microvm_id + || !record.endpoint + || record.hard_deadline_at == null + || record.hard_deadline_at <= this.now() + ) { + throw new HostedAppControlPlaneError( + 'hosted_app_not_running', + 'Hosted app is not running', + 409, + true, + ); + } + return hostedAppPublicStatus( + await this.persistPreviewCredential( + record, + this.recordedVm(record), + lockToken, + leaseSignal, + ), + this.now(), + ); + }); + } + + private recordedVm(record: RuntimeSessionRecord): MicrovmDescription { + return { + microvmId: record.microvm_id as string, + endpoint: record.endpoint, + state: 'RUNNING', + imageArn: record.image_arn, + imageVersion: record.image_version, + }; + } + + private async persistPreviewCredential( + record: RuntimeSessionRecord, + vm: MicrovmDescription, + lockToken: string, + signal: AbortSignal, + ): Promise { + const credential = await this.deps.runtime.previewToken(vm.microvmId, signal); + if (credential.expiresAtMs <= this.now()) { + throw new HostedAppControlPlaneError( + 'hosted_app_preview_unavailable', + 'Hosted app preview credential is unavailable', + 503, + true, + ); + } + const running: RuntimeSessionRecord = { + ...record, + state: 'RUNNING', + last_seen_at: this.now(), + last_error: undefined, + hosted_app: { + ...(record.hosted_app as NonNullable), + preview_credential: sealHostedAppCredential( + record.runtime_session_id, + credential, + this.deps.credentialKey, + ), + preview_credential_expires_at: credential.expiresAtMs, + }, + }; + await this.writeOrFence(running, lockToken, signal); + return running; + } + + private async writeOrFence( + record: RuntimeSessionRecord, + lockToken: string, + signal: AbortSignal, + ): Promise { + if (!await this.deps.registry.write(record, lockToken, { signal })) { + signal.throwIfAborted(); + throw new HostedAppControlPlaneError( + 'hosted_app_fenced', + 'Lost the hosted app lease while updating it', + 409, + true, + ); + } + } + + private async withLease( + runtimeId: string, + callerSignal: AbortSignal, + operation: (signal: AbortSignal, lockToken: string) => Promise, + ): Promise { + const lockToken = await this.deps.registry.waitForLock(runtimeId, { + waitMs: this.deps.lockWaitMs, + ttlMs: this.deps.lockTtlMs, + signal: callerSignal, + }); + if (!lockToken) { + throw new HostedAppControlPlaneError( + 'hosted_app_busy', + 'Another hosted app transition is in progress', + 409, + true, + ); + } + const fence = new AbortController(); + const signal = AbortSignal.any([callerSignal, fence.signal]); + const heartbeat = this.deps.startHeartbeat({ + renew: () => this.deps.registry.renewLock( + runtimeId, + lockToken, + this.deps.lockTtlMs, + { signal: callerSignal, onLateLost: () => fence.abort() }, + ), + fence, + ttlMs: this.deps.lockTtlMs, + }); + try { + return await operation(signal, lockToken); + } finally { + heartbeat.stop(); + await this.deps.registry.releaseLock(runtimeId, lockToken); + } + } +} diff --git a/service/src/hosted-app/credential.test.ts b/service/src/hosted-app/credential.test.ts new file mode 100644 index 00000000..e14236b7 --- /dev/null +++ b/service/src/hosted-app/credential.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from 'bun:test'; +import { + openHostedAppCredential, + parseHostedAppCredentialKey, + sealHostedAppCredential, +} from './credential'; + +const key = Buffer.alloc(32, 7); +const credential = { + headerName: 'X-aws-proxy-auth', + token: 'secret-jwe', + expiresAtMs: 1_787_300_000_000, +}; + +describe('hosted-app credential envelope', () => { + test('round-trips a preview token without emitting plaintext', () => { + const sealed = sealHostedAppCredential('happ_owner_a', credential, key); + expect(sealed).not.toContain(credential.token); + expect(openHostedAppCredential('happ_owner_a', sealed, key)).toEqual(credential); + }); + + test('binds ciphertext to one hosted-app identity', () => { + const sealed = sealHostedAppCredential('happ_owner_a', credential, key); + expect(() => openHostedAppCredential('happ_owner_b', sealed, key)).toThrow( + 'could not be authenticated', + ); + }); + + test('rejects tampering and malformed key material', () => { + const sealed = sealHostedAppCredential('happ_owner_a', credential, key); + const parts = sealed.split('.'); + const tag = Buffer.from(parts[2] as string, 'base64url'); + tag[0] ^= 1; + parts[2] = tag.toString('base64url'); + const tampered = parts.join('.'); + expect(() => openHostedAppCredential('happ_owner_a', tampered, key)).toThrow( + 'could not be authenticated', + ); + expect(() => openHostedAppCredential( + 'happ_owner_a', + `${sealed.slice(0, -1)}${sealed.endsWith('A') ? 'B' : 'A'}`, + key, + )).toThrow('could not be authenticated'); + expect(() => parseHostedAppCredentialKey(Buffer.alloc(31).toString('base64'))) + .toThrow('exactly 32 bytes'); + expect(parseHostedAppCredentialKey(key.toString('base64'))).toEqual(key); + }); +}); diff --git a/service/src/hosted-app/credential.ts b/service/src/hosted-app/credential.ts new file mode 100644 index 00000000..836fc0fa --- /dev/null +++ b/service/src/hosted-app/credential.ts @@ -0,0 +1,98 @@ +import { + createCipheriv, + createDecipheriv, + randomBytes, +} from 'node:crypto'; +import type { MicrovmAuthToken } from '../runtime-session/lambda-client'; + +const TOKEN_FORMAT = 'v1'; +const IV_BYTES = 12; +const TAG_BYTES = 16; + +export class HostedAppCredentialError extends Error {} + +export function parseHostedAppCredentialKey(raw: string): Buffer { + let key: Buffer; + try { + key = Buffer.from(raw, 'base64'); + } catch { + throw new HostedAppCredentialError('CODEAPI_HOSTED_APP_CREDENTIAL_KEY must be base64'); + } + if (key.length !== 32 || key.toString('base64').replace(/=+$/, '') !== raw.trim().replace(/=+$/, '')) { + throw new HostedAppCredentialError( + 'CODEAPI_HOSTED_APP_CREDENTIAL_KEY must encode exactly 32 bytes', + ); + } + return key; +} + +export function sealHostedAppCredential( + hostedAppRuntimeId: string, + credential: MicrovmAuthToken, + key: Buffer, +): string { + if (key.length !== 32) throw new HostedAppCredentialError('credential key must be 32 bytes'); + const iv = randomBytes(IV_BYTES); + const cipher = createCipheriv('aes-256-gcm', key, iv); + cipher.setAAD(Buffer.from(hostedAppRuntimeId, 'utf8')); + const plaintext = Buffer.from(JSON.stringify(credential), 'utf8'); + const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]); + const tag = cipher.getAuthTag(); + return [ + TOKEN_FORMAT, + iv.toString('base64url'), + tag.toString('base64url'), + ciphertext.toString('base64url'), + ].join('.'); +} + +export function openHostedAppCredential( + hostedAppRuntimeId: string, + sealed: string, + key: Buffer, +): MicrovmAuthToken { + if (key.length !== 32) throw new HostedAppCredentialError('credential key must be 32 bytes'); + const [version, ivRaw, tagRaw, ciphertextRaw, extra] = sealed.split('.'); + if (version !== TOKEN_FORMAT || !ivRaw || !tagRaw || !ciphertextRaw || extra !== undefined) { + throw new HostedAppCredentialError('hosted-app credential is malformed'); + } + try { + const decode = (raw: string, expectedBytes?: number): Buffer => { + if (!/^[A-Za-z0-9_-]+$/.test(raw)) throw new Error('credential encoding is malformed'); + const decoded = Buffer.from(raw, 'base64url'); + if ( + decoded.toString('base64url') !== raw + || (expectedBytes != null && decoded.length !== expectedBytes) + ) { + throw new Error('credential encoding is malformed'); + } + return decoded; + }; + const iv = decode(ivRaw, IV_BYTES); + const tag = decode(tagRaw, TAG_BYTES); + const ciphertext = decode(ciphertextRaw); + const decipher = createDecipheriv('aes-256-gcm', key, iv); + decipher.setAAD(Buffer.from(hostedAppRuntimeId, 'utf8')); + decipher.setAuthTag(tag); + const plaintext = Buffer.concat([ + decipher.update(ciphertext), + decipher.final(), + ]).toString('utf8'); + const parsed = JSON.parse(plaintext) as Partial; + if ( + typeof parsed.headerName !== 'string' + || parsed.headerName.length === 0 + || typeof parsed.token !== 'string' + || parsed.token.length === 0 + || !Number.isSafeInteger(parsed.expiresAtMs) + || (parsed.expiresAtMs as number) <= 0 + ) { + throw new Error('credential payload is invalid'); + } + return parsed as MicrovmAuthToken; + } catch (error) { + throw new HostedAppCredentialError( + `hosted-app credential could not be authenticated: ${(error as Error).message}`, + ); + } +} diff --git a/service/src/hosted-app/factory.ts b/service/src/hosted-app/factory.ts new file mode 100644 index 00000000..472365e3 --- /dev/null +++ b/service/src/hosted-app/factory.ts @@ -0,0 +1,145 @@ +import { env } from '../config'; +import { checkpointSession, restoreSession } from '../runtime-session/checkpoint'; +import { MinioCheckpointStore } from '../runtime-session/checkpoint-store'; +import { AwsLambdaMicrovmClient } from '../runtime-session/lambda-client-aws'; +import { startRuntimeSessionLockHeartbeat } from '../runtime-session/lock-heartbeat'; +import { + allocateRuntimeSessionGeneration, + RUNTIME_SESSION_LOCK_TTL_MS, + readRuntimeSessionRecord, + releaseRuntimeSessionLock, + renewRuntimeSessionLock, + waitForRuntimeSessionLock, + writeRuntimeSessionRecord, +} from '../runtime-session/registry'; +import { + HostedAppControlPlane, + HostedAppControlPlaneError, +} from './control-plane'; +import { parseHostedAppCredentialKey } from './credential'; +import { + HostedAppMicrovmRuntime, + normalizeHostedAppMicrovmEndpoint, +} from './microvm-runtime'; +import { captureHostedAppSourceCheckpoint } from './source-checkpoint'; + +let controlPlane: HostedAppControlPlane | undefined; + +export function getHostedAppControlPlane(): HostedAppControlPlane { + if (controlPlane) return controlPlane; + if (!env.HOSTED_APPS_ENABLED) { + throw new HostedAppControlPlaneError( + 'hosted_apps_disabled', + 'Hosted apps are not enabled on this execution profile', + 404, + ); + } + + const client = new AwsLambdaMicrovmClient({ region: env.LAMBDA_MICROVM_REGION }); + const runtime = new HostedAppMicrovmRuntime(client, { + imageArn: env.HOSTED_APP_IMAGE_ARN, + imageVersion: env.HOSTED_APP_IMAGE_VERSION as string, + executionRoleArn: env.LAMBDA_MICROVM_EXECUTION_ROLE_ARN, + logGroup: env.LAMBDA_MICROVM_LOG_GROUP, + ingressConnectorArns: env.LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS, + controlPort: env.HOSTED_APP_CONTROL_PORT, + previewPort: env.HOSTED_APP_PREVIEW_PORT, + maximumDurationSeconds: env.HOSTED_APP_MAX_DURATION_SECONDS, + idleSeconds: env.HOSTED_APP_IDLE_SECONDS, + suspendedSeconds: env.HOSTED_APP_SUSPEND_SECONDS, + authTokenTtlSeconds: env.LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS, + launchTimeoutMs: env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS, + healthTimeoutMs: env.LAMBDA_MICROVM_HEALTH_TIMEOUT_MS, + appStartTimeoutMs: env.HOSTED_APP_START_TIMEOUT_MS, + launchTps: env.LAMBDA_MICROVM_LAUNCH_TPS, + tokenTps: env.LAMBDA_MICROVM_TOKEN_TPS, + }); + const store = new MinioCheckpointStore(); + const checkpointConfig = { + port: env.HOSTED_APP_CONTROL_PORT, + authTokenTtlSeconds: env.LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS, + maxBytes: env.CHECKPOINT_MAX_BYTES, + timeoutMs: env.CHECKPOINT_TIMEOUT_MS, + }; + + controlPlane = new HostedAppControlPlane({ + registry: { + waitForLock: (runtimeId, args) => waitForRuntimeSessionLock(runtimeId, args), + renewLock: (runtimeId, token, ttlMs, args) => renewRuntimeSessionLock( + runtimeId, + token, + ttlMs, + args, + ), + releaseLock: releaseRuntimeSessionLock, + read: readRuntimeSessionRecord, + write: (record, token, args) => writeRuntimeSessionRecord( + record, + token, + undefined, + args, + ), + allocateGeneration: allocateRuntimeSessionGeneration, + }, + runtime, + checkpointStore: store, + readRevision: (runtimeId, revision) => store.readHostedAppRevision(runtimeId, revision), + retainRevision: (runtimeId, revision) => store.retainHostedAppRevision(runtimeId, revision), + checkpointConfig, + credentialKey: parseHostedAppCredentialKey(env.HOSTED_APP_CREDENTIAL_KEY), + lockWaitMs: env.RUNTIME_SESSION_LOCK_WAIT_MS, + lockTtlMs: RUNTIME_SESSION_LOCK_TTL_MS, + startHeartbeat: startRuntimeSessionLockHeartbeat, + captureCheckpoint: (runtimeSessionId, owner, signal) => ( + captureHostedAppSourceCheckpoint({ + runtimeSessionId, + owner, + signal, + lockWaitMs: env.RUNTIME_SESSION_LOCK_WAIT_MS, + deps: { + waitForLock: (sourceId, args) => waitForRuntimeSessionLock(sourceId, args), + releaseLock: releaseRuntimeSessionLock, + retain: (sourceId, key) => store.retainForHostedApp(sourceId, key), + read: readRuntimeSessionRecord, + checkpoint: ({ runtimeSessionId: sourceId, lockToken, signal: sourceSignal }) => ( + checkpointSession({ + mintToken: microvmId => runtime.mintToken( + microvmId, + env.LAMBDA_MICROVM_PORT, + sourceSignal, + ), + store, + runtimeSessionId: sourceId, + config: { + ...checkpointConfig, + port: env.LAMBDA_MICROVM_PORT, + }, + normalizeEndpoint: normalizeHostedAppMicrovmEndpoint, + lockToken, + signal: sourceSignal, + }) + ), + }, + }) + ), + restoreCheckpoint: args => restoreSession({ + mintToken: microvmId => runtime.mintToken( + microvmId, + env.HOSTED_APP_CONTROL_PORT, + args.signal, + ), + store: args.store, + runtimeSessionId: args.runtimeSessionId, + microvmId: args.vm.microvmId, + endpointBase: normalizeHostedAppMicrovmEndpoint(args.vm.endpoint ?? ''), + config: args.config, + signal: args.signal, + checkpointKey: args.checkpointKey, + }), + }); + return controlPlane; +} + +export function resetHostedAppControlPlaneForTests(): void { + controlPlane = undefined; +} diff --git a/service/src/hosted-app/jobs.ts b/service/src/hosted-app/jobs.ts new file mode 100644 index 00000000..2a035e16 --- /dev/null +++ b/service/src/hosted-app/jobs.ts @@ -0,0 +1,32 @@ +import type { Job } from 'bullmq'; +import type { HostedAppPublicStatus } from './record'; +import type { ResidentHostedAppSpec } from './spec'; + +export const HOSTED_APP_JOBS = [ + 'hosted-app:start', + 'hosted-app:stop', + 'hosted-app:refresh-preview', + 'hosted-app:status', +] as const; +export type HostedAppJobName = (typeof HOSTED_APP_JOBS)[number]; + +interface HostedAppJobBase { + hostedAppRuntimeId: string; + tenantId: string; + canonicalUserId: string; + _otel?: Record; +} + +export interface HostedAppStartJobData extends HostedAppJobBase { + operation: 'start'; + sourceRuntimeSessionId: string; + spec: ResidentHostedAppSpec; +} + +export interface HostedAppStopJobData extends HostedAppJobBase { + operation: 'stop' | 'refresh-preview' | 'status'; +} + +export type HostedAppJobData = HostedAppStartJobData | HostedAppStopJobData; +export type HostedAppJobResult = HostedAppPublicStatus; +export type HostedAppJob = Job; diff --git a/service/src/hosted-app/microvm-runtime.test.ts b/service/src/hosted-app/microvm-runtime.test.ts new file mode 100644 index 00000000..eae7ad38 --- /dev/null +++ b/service/src/hosted-app/microvm-runtime.test.ts @@ -0,0 +1,309 @@ +import { describe, expect, test } from 'bun:test'; +import { FakeLambdaMicrovmClient } from '../runtime-session/lambda-client-fake'; +import { LambdaMicrovmApiError } from '../runtime-session/lambda-client'; +import { MicrovmOpThrottledError } from '../runtime-session/throttle'; +import { + hostedAppLaunchFingerprint, + hostedAppLaunchGenerationSeed, + hostedAppLaunchRequestFingerprint, + HostedAppMicrovmError, + HostedAppMicrovmRuntime, + type HostedAppMicrovmConfig, +} from './microvm-runtime'; +import type { ResidentHostedAppSpec } from './spec'; + +function config(): HostedAppMicrovmConfig { + return { + imageArn: 'arn:aws:lambda:us-east-2:1:microvm-image:app-host', + imageVersion: '7', + executionRoleArn: 'arn:aws:iam::1:role/app-host', + logGroup: '/aws/lambda-microvm/codeapi-app-host', + ingressConnectorArns: ['arn:ingress/private'], + controlPort: 8080, + previewPort: 3000, + maximumDurationSeconds: 28_800, + idleSeconds: 300, + suspendedSeconds: 900, + authTokenTtlSeconds: 3_600, + launchTimeoutMs: 5_000, + healthTimeoutMs: 500, + appStartTimeoutMs: 2_000, + launchTps: 4, + tokenTps: 8, + }; +} + +const spec: ResidentHostedAppSpec = { + adapter: 'resident', + app_id: 'demo', + revision: 'rev-1', + language: 'node', + version: '>=22', + entrypoint: 'server.js', + cwd: '.', + args: [], + env: {}, +}; + +function runtime( + fake: FakeLambdaMicrovmClient, + fetchImpl: (input: string | URL | Request, init?: RequestInit) => Promise + = async () => new Response('{}', { status: 200 }), +) { + const reservations: string[] = []; + const poisons: string[] = []; + return { + reservations, + poisons, + runtime: new HostedAppMicrovmRuntime(fake, config(), { + reserveOp: async op => { reservations.push(op); }, + poisonOp: async op => { poisons.push(op); }, + fetch: fetchImpl, + sleep: async () => {}, + }), + }; +} + +describe('HostedAppMicrovmRuntime', () => { + test('keeps suspended recovery ambiguous when the shared resume budget is exhausted', async () => { + const fake = new FakeLambdaMicrovmClient(); + const signal = new AbortController().signal; + const { vm } = await runtime(fake).runtime.launch('resume-budget', signal); + await fake.suspendMicrovm(vm.microvmId); + const limited = new HostedAppMicrovmRuntime(fake, config(), { + reserveOp: async op => { if (op === 'resume') throw new MicrovmOpThrottledError('resume', 1); }, + }); + const failure = await limited.launch('resume-budget', signal).catch(error => error); + expect(failure.transient).toBe(true); + expect(fake.callsFor('resumeMicrovm')).toHaveLength(0); + expect(fake.vms.size).toBe(1); + }); + test('reserves and poisons the distributed resume budget on provider throttling', async () => { + const fake = new FakeLambdaMicrovmClient(); + const f = runtime(fake); + const signal = new AbortController().signal; + const { vm } = await f.runtime.launch('resume-throttle', signal); + await fake.suspendMicrovm(vm.microvmId); + fake.failNext('resumeMicrovm', new LambdaMicrovmApiError('throttled', 'ResumeMicrovm', 'throttled')); + await expect(f.runtime.launch('resume-throttle', signal)).rejects.toThrow('throttled'); + expect(f.reservations).toEqual(['run', 'run', 'resume']); + expect(f.poisons).toEqual(['resume']); + }); + + test('does not turn control-token failure into evidence of an unhealthy VM', async () => { + const fake = new FakeLambdaMicrovmClient(); + const f = runtime(fake); + const signal = new AbortController().signal; + const { vm } = await f.runtime.launch('health-auth', signal); + fake.failNext('createMicrovmAuthToken', new LambdaMicrovmApiError('throttled', 'CreateMicrovmAuthToken', 'throttled')); + const failure = await f.runtime.waitForControlReady(vm, signal).catch(error => error); + expect(failure.code).toBe('hosted_app_auth_failed'); + }); + test('cancels unsuccessful and successful health response bodies before continuing', async () => { + const fake = new FakeLambdaMicrovmClient(); + let probes = 0; + let canceled = 0; + const f = runtime(fake, async () => { + expect(canceled).toBe(probes); + probes++; + return new Response(new ReadableStream({ cancel() { canceled++; } }), { + status: probes === 1 ? 503 : 200, + }); + }); + const signal = new AbortController().signal; + const { vm } = await f.runtime.launch('health-disposal', signal); + await f.runtime.waitForControlReady(vm, signal); + expect(canceled).toBe(2); + }); + + test('reads session-bound resident status and rejects mismatched revision', async () => { + const fake = new FakeLambdaMicrovmClient(); + let revision = spec.revision; + const f = runtime(fake, async (url, init) => { + expect(String(url)).toEndWith('/api/v2/hosted-app/status'); + expect(new Headers(init?.headers).get('X-Runtime-Session-Id')).toBe('source'); + return Response.json({ app_id: spec.app_id, revision, state: 'failed' }); + }); + const signal = new AbortController().signal; + const { vm } = await f.runtime.launch('status', signal); + expect(await f.runtime.residentAppState(vm, 'source', spec, signal)).toBe('failed'); + revision = 'wrong-revision'; + await expect(f.runtime.residentAppState(vm, 'source', spec, signal)).rejects.toThrow('does not match'); + }); + test('keeps a second-attempt cancellation ambiguous and replayable', async () => { + const fake = new FakeLambdaMicrovmClient(); + fake.terminateNextLaunch(); + const caller = new AbortController(); + let attempts = 0; + const run = fake.runMicrovm.bind(fake); + fake.runMicrovm = async (...args) => { + const vm = await run(...args); + if (++attempts === 2) { + caller.abort(new Error('caller left after provider acceptance')); + throw caller.signal.reason; + } + return vm; + }; + const error = await runtime(fake).runtime.launch('retry-abort', caller.signal).catch(e => e); + expect(error.transient).toBe(true); + expect(fake.callsFor('runMicrovm').map(c => (c.args as { clientToken: string }).clientToken)) + .toEqual(['retry-abort', 'retry-abort-r1']); + const recovered = await runtime(fake).runtime.launch('retry-abort', new AbortController().signal); + expect(recovered.clientToken).toBe('retry-abort-r1'); + expect(fake.vms.size).toBe(2); + }); + + test('classifies a resident-start network reset as transient', async () => { + const fake = new FakeLambdaMicrovmClient(); + const f = runtime(fake, async () => { throw new TypeError('fetch failed'); }); + const signal = new AbortController().signal; + const { vm } = await f.runtime.launch('resident-reset', signal); + const error = await f.runtime.startResidentApp(vm, 'source', spec, signal).catch(e => e); + expect(error).toBeInstanceOf(HostedAppMicrovmError); + expect(error.transient).toBe(true); + }); + test('seeds idempotency from exact wire inputs while keeping semantic matching order-independent', () => { + const first = { ...config(), ingressConnectorArns: ['arn:b', 'arn:a'] }; + const reordered = { ...config(), ingressConnectorArns: ['arn:a', 'arn:b'] }; + expect(hostedAppLaunchFingerprint(first)).toBe(hostedAppLaunchFingerprint(reordered)); + expect(hostedAppLaunchRequestFingerprint(first)).not.toBe( + hostedAppLaunchRequestFingerprint(reordered), + ); + expect(hostedAppLaunchGenerationSeed(first)).not.toBe(hostedAppLaunchGenerationSeed(reordered)); + }); + + test('launches the dedicated image with bounded idle policy and no egress connector', async () => { + const fake = new FakeLambdaMicrovmClient(); + const fixture = runtime(fake); + + const launched = await fixture.runtime.launch('sess-happ-1', new AbortController().signal); + + expect(launched.clientToken).toBe('sess-happ-1'); + expect(launched.vm.state).toBe('RUNNING'); + expect(fixture.reservations).toEqual(['run']); + const args = fake.callsFor('runMicrovm')[0].args as Record; + expect(args).toMatchObject({ + imageIdentifier: config().imageArn, + imageVersion: '7', + maximumDurationSeconds: 28_800, + idlePolicy: { + maxIdleSeconds: 300, + suspendedSeconds: 900, + autoResume: true, + }, + }); + expect(args.egressConnectorArns).toBeUndefined(); + }); + + test('retries a definite boot-time death once under a distinct token', async () => { + const fake = new FakeLambdaMicrovmClient(); + fake.terminateNextLaunch(); + const fixture = runtime(fake); + + const launched = await fixture.runtime.launch('sess-happ-2', new AbortController().signal); + + expect(launched.clientToken).toBe('sess-happ-2-r1'); + expect(fake.callsFor('runMicrovm').map(call => ( + call.args as { clientToken?: string } + ).clientToken)).toEqual(['sess-happ-2', 'sess-happ-2-r1']); + }); + + test('resumes a suspended same-token launch instead of provisioning a second VM', async () => { + const fake = new FakeLambdaMicrovmClient(); + const fixture = runtime(fake); + const first = await fixture.runtime.launch('sess-happ-recovered', new AbortController().signal); + await fake.suspendMicrovm(first.vm.microvmId); + + const recovered = await fixture.runtime.launch( + 'sess-happ-recovered', + new AbortController().signal, + ); + + expect(recovered.vm.microvmId).toBe(first.vm.microvmId); + expect(recovered.clientToken).toBe('sess-happ-recovered'); + expect(fake.vms.size).toBe(1); + expect(fake.callsFor('resumeMicrovm')).toHaveLength(1); + expect(fake.callsFor('runMicrovm').map(call => ( + call.args as { clientToken?: string } + ).clientToken)).toEqual(['sess-happ-recovered', 'sess-happ-recovered']); + }); + + test('does not rotate the idempotency token after an ambiguous provider failure', async () => { + const fake = new FakeLambdaMicrovmClient(); + fake.failNext('runMicrovm', new LambdaMicrovmApiError( + 'other', + 'RunMicrovm', + 'connection reset after request write', + )); + const fixture = runtime(fake); + + const error = await fixture.runtime.launch( + 'sess-happ-3', + new AbortController().signal, + ).catch(value => value); + + expect(error.code).toBe('hosted_app_launch_failed'); + expect(fake.callsFor('runMicrovm')).toHaveLength(1); + }); + + test('reuses one control credential while polling health', async () => { + const fake = new FakeLambdaMicrovmClient({ endpointProvider: () => 'http://app-host.test' }); + let probes = 0; + const fixture = runtime(fake, async () => { + probes += 1; + return new Response('{}', { status: probes < 3 ? 503 : 200 }); + }); + const { vm } = await fixture.runtime.launch('sess-happ-4', new AbortController().signal); + + await fixture.runtime.waitForControlReady(vm, new AbortController().signal); + + expect(probes).toBe(3); + expect(fake.callsFor('createMicrovmAuthToken')).toHaveLength(1); + }); + + test('starts the resident app through the control port with its session binding', async () => { + const fake = new FakeLambdaMicrovmClient({ endpointProvider: () => 'http://app-host.test' }); + let captured: { url: string; init?: RequestInit } | undefined; + const fixture = runtime(fake, async (input, init) => { + captured = { url: String(input), init }; + return new Response('{}', { status: 200 }); + }); + const { vm } = await fixture.runtime.launch('sess-happ-5', new AbortController().signal); + + await fixture.runtime.startResidentApp( + vm, + 'rt_source_session', + spec, + new AbortController().signal, + ); + + expect(captured?.url).toBe('http://app-host.test/api/v2/hosted-app/start'); + expect(captured?.init?.method).toBe('POST'); + expect(captured?.init?.headers).toMatchObject({ + 'X-aws-proxy-auth': expect.any(String), + 'X-Runtime-Session-Id': 'rt_source_session', + 'Content-Type': 'application/json', + }); + expect(JSON.parse(String(captured?.init?.body))).toEqual(spec); + }); + + test('preserves a runner validation status as a non-retryable typed failure', async () => { + const fake = new FakeLambdaMicrovmClient({ endpointProvider: () => 'http://app-host.test' }); + const fixture = runtime(fake, async () => new Response(JSON.stringify({ + error: 'hosted_app_runtime_not_found', + message: 'runtime node@99 is not installed', + }), { status: 400 })); + const { vm } = await fixture.runtime.launch('sess-happ-6', new AbortController().signal); + + const error = await fixture.runtime.startResidentApp( + vm, + 'rt_source_session', + spec, + new AbortController().signal, + ).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppMicrovmError); + expect(error.httpStatus).toBe(400); + expect(error.transient).toBe(false); + }); +}); diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts new file mode 100644 index 00000000..8b594f37 --- /dev/null +++ b/service/src/hosted-app/microvm-runtime.ts @@ -0,0 +1,496 @@ +import { + LambdaMicrovmApiError, + microvmPortHeaders, + type LambdaMicrovmClient, + type MicrovmAuthToken, + type MicrovmDescription, +} from '../runtime-session/lambda-client'; +import { + MicrovmOpThrottledError, + acquireOpBudget, + poisonOpBucket, + type ThrottledOp, +} from '../runtime-session/throttle'; +import type { ResidentHostedAppSpec } from './spec'; +import { createHash } from 'node:crypto'; + +export interface HostedAppMicrovmConfig { + imageArn: string; + imageVersion: string; + executionRoleArn?: string; + logGroup?: string; + ingressConnectorArns?: string[]; + controlPort: number; + previewPort: number; + maximumDurationSeconds: number; + idleSeconds: number; + suspendedSeconds: number; + authTokenTtlSeconds: number; + launchTimeoutMs: number; + healthTimeoutMs: number; + appStartTimeoutMs: number; + launchTps: number; + tokenTps: number; +} + +/** Order-independent identity for every immutable app-host launch input. */ +export function hostedAppLaunchFingerprint(config: HostedAppMicrovmConfig): string { + return JSON.stringify({ + imageArn: config.imageArn, + imageVersion: config.imageVersion, + executionRoleArn: config.executionRoleArn ?? '', + logGroup: config.logGroup ?? '', + ingressConnectorArns: [...(config.ingressConnectorArns ?? [])].sort(), + controlPort: config.controlPort, + previewPort: config.previewPort, + maximumDurationSeconds: config.maximumDurationSeconds, + idlePolicy: { + maxIdleSeconds: config.idleSeconds, + suspendedSeconds: config.suspendedSeconds, + autoResume: true, + }, + }); +} + +/** Exact RunMicrovm request identity. Preserve connector order because AWS + * idempotency compares the submitted request, not our semantic policy. */ +export function hostedAppLaunchRequestFingerprint(config: HostedAppMicrovmConfig): string { + return JSON.stringify({ + launchFingerprint: hostedAppLaunchFingerprint(config), + runMicrovm: { + imageIdentifier: config.imageArn, + imageVersion: config.imageVersion, + executionRoleArn: config.executionRoleArn, + logGroup: config.logGroup, + ingressConnectorArns: config.ingressConnectorArns, + maximumDurationSeconds: config.maximumDurationSeconds, + idlePolicy: { + maxIdleSeconds: config.idleSeconds, + suspendedSeconds: config.suspendedSeconds, + autoResume: true, + }, + }, + }); +} + +/** Keep reset Redis counters in an image/config-specific safe-integer range. */ +export function hostedAppLaunchGenerationSeed(config: HostedAppMicrovmConfig): number { + const offset = Number.parseInt( + createHash('sha256') + .update(hostedAppLaunchRequestFingerprint(config), 'utf8') + .digest('hex') + .slice(0, 13), + 16, + ); + return 1_000_000_000_000_000 + offset; +} + +export function hostedAppLaunchClientToken(runtimeId: string, generation: number): string { + if (!Number.isSafeInteger(generation) || generation < 1) { + throw new Error('Hosted app generation must be a positive safe integer'); + } + const token = `happ-${runtimeId}-${generation}`; + /* launch() reserves a single `-r1` suffix after a definite boot failure. */ + if (token.length > 125) { + throw new Error('Hosted app clientToken exceeds the AWS length limit'); + } + return token; +} + +export class HostedAppMicrovmError extends Error { + constructor( + readonly code: string, + message: string, + readonly transient: boolean, + readonly cause?: unknown, + readonly httpStatus = 503, + ) { + super(message); + this.name = 'HostedAppMicrovmError'; + } +} + +interface HostedAppMicrovmDeps { + reserveOp: ( + op: ThrottledOp, + args: { limitPerSecond: number; deadlineAtMs: number; signal: AbortSignal }, + ) => Promise; + poisonOp: (op: ThrottledOp, deadlineAtMs: number, signal: AbortSignal) => Promise; + fetch: (input: string | URL | Request, init?: RequestInit) => Promise; + sleep: (ms: number, signal: AbortSignal) => Promise; +} + +const abortableSleep = (ms: number, signal: AbortSignal): Promise => new Promise( + (resolve, reject) => { + if (signal.aborted) { + reject(signal.reason instanceof Error ? signal.reason : new Error('operation aborted')); + return; + } + const finish = (): void => { + signal.removeEventListener('abort', onAbort); + resolve(); + }; + const timer = setTimeout(finish, ms); + const onAbort = (): void => { + clearTimeout(timer); + reject(signal.reason instanceof Error ? signal.reason : new Error('operation aborted')); + }; + signal.addEventListener('abort', onAbort, { once: true }); + }, +); + +export function normalizeHostedAppMicrovmEndpoint(endpoint: string): string { + if (endpoint.startsWith('http://') || endpoint.startsWith('https://')) { + return endpoint.replace(/\/+$/, ''); + } + return `https://${endpoint.replace(/\/+$/, '')}`; +} + +function launchFailure(error: unknown): HostedAppMicrovmError { + if (error instanceof HostedAppMicrovmError) return error; + if (error instanceof LambdaMicrovmApiError) { + // A rejected replay does not prove an earlier same-token request was never + // admitted. Only observed terminal VMs establish definite boot exhaustion. + const transient = true; + return new HostedAppMicrovmError( + error.kind === 'throttled' ? 'hosted_app_launch_throttled' : 'hosted_app_launch_failed', + error.message, + transient, + error, + ); + } + return new HostedAppMicrovmError( + 'hosted_app_launch_failed', + error instanceof Error ? error.message : 'Hosted app MicroVM launch failed', + true, + error, + ); +} + +export class HostedAppMicrovmRuntime { + private readonly deps: HostedAppMicrovmDeps; + + constructor( + private readonly client: LambdaMicrovmClient, + readonly config: HostedAppMicrovmConfig, + deps: Partial = {}, + ) { + this.deps = { + reserveOp: (op, args) => acquireOpBudget(op, { + limitPerSecond: args.limitPerSecond, + budgetMs: Math.max(1, args.deadlineAtMs - Date.now()), + deadlineAtMs: args.deadlineAtMs, + signal: args.signal, + }), + poisonOp: (op, deadlineAtMs, signal) => poisonOpBucket(op, undefined, { + deadlineAtMs, + signal, + }), + fetch, + sleep: abortableSleep, + ...deps, + }; + } + + async launch(clientToken: string, callerSignal: AbortSignal): Promise<{ + vm: MicrovmDescription; + clientToken: string; + }> { + const deadlineAtMs = Date.now() + this.config.launchTimeoutMs; + const deadlineSignal = AbortSignal.timeout(this.config.launchTimeoutMs); + const signal = AbortSignal.any([callerSignal, deadlineSignal]); + try { + const first = await this.launchOnce(clientToken, deadlineAtMs, signal, deadlineSignal); + return { vm: first, clientToken }; + } catch (error) { + const failure = signal.aborted + ? new HostedAppMicrovmError( + 'hosted_app_launch_timeout', + `Hosted app MicroVM did not reach RUNNING within ${this.config.launchTimeoutMs}ms`, + true, + error, + ) + : launchFailure(error); + /* Only a definite boot-time death is safe to retry with a new token. + * Ambiguous RunMicrovm failures retain the original token so a successor + * can replay and recover the provider resource. */ + if (failure.code !== 'hosted_app_boot_failed' || callerSignal.aborted) throw failure; + const retryToken = `${clientToken}-r1`; + const vm = await this.launchOnce(retryToken, deadlineAtMs, signal, deadlineSignal) + .catch(second => { + if (signal.aborted) { + throw new HostedAppMicrovmError('hosted_app_launch_timeout', + 'Hosted app launch interrupted; replay the persisted intent', true, second); + } + throw launchFailure(second); + }); + return { vm, clientToken: retryToken }; + } + } + + private async launchOnce( + clientToken: string, + deadlineAtMs: number, + signal: AbortSignal, + reconcileSignal: AbortSignal, + ): Promise { + try { + await this.deps.reserveOp('run', { + limitPerSecond: this.config.launchTps, + deadlineAtMs, + signal, + }); + } catch (error) { + if (error instanceof MicrovmOpThrottledError) { + throw new HostedAppMicrovmError( + 'hosted_app_launch_throttled', + error.message, + true, + error, + ); + } + throw error; + } + + let vm: MicrovmDescription; + try { + vm = await this.client.runMicrovm({ + imageIdentifier: this.config.imageArn, + imageVersion: this.config.imageVersion, + executionRoleArn: this.config.executionRoleArn, + logGroup: this.config.logGroup, + ingressConnectorArns: this.config.ingressConnectorArns, + /* No egress connector is passed. The app-host runner additionally + * blocks all new OUTPUT traffic from the untrusted app UID. */ + maximumDurationSeconds: this.config.maximumDurationSeconds, + idlePolicy: { + maxIdleSeconds: this.config.idleSeconds, + suspendedSeconds: this.config.suspendedSeconds, + autoResume: true, + }, + clientToken, + }, signal, reconcileSignal); + } catch (error) { + if (error instanceof LambdaMicrovmApiError && error.kind === 'throttled') { + await this.deps.poisonOp('run', deadlineAtMs, signal).catch(() => {}); + } + throw error; + } + + let current = vm; + let resumeRequested = false; + for (;;) { + if (Date.now() >= deadlineAtMs) { + throw new HostedAppMicrovmError( + 'hosted_app_launch_timeout', + 'Hosted app MicroVM launch timed out', + true, + ); + } + if (current.state === 'RUNNING' && current.endpoint) return current; + if (current.state === 'TERMINATING' || current.state === 'TERMINATED') { + throw new HostedAppMicrovmError( + 'hosted_app_boot_failed', + `Hosted app MicroVM entered ${current.state} before becoming ready`, + true, + ); + } + /* Same-token recovery can find an older accepted launch after its idle + * policy suspended it. That is the resource we must recover, not evidence + * of a failed boot: rotating the token here would provision a second VM + * and abandon the first one until its hard deadline. Resume it once, then + * keep polling the same id while AWS completes the transition. */ + if (current.state === 'SUSPENDED' && !resumeRequested) { + resumeRequested = true; + try { + await this.deps.reserveOp('resume', { limitPerSecond: 5, deadlineAtMs, signal }); + current = await this.client.resumeMicrovm(current.microvmId, signal); + continue; + } catch (error) { + if (error instanceof LambdaMicrovmApiError && error.kind === 'throttled') { + await this.deps.poisonOp('resume', deadlineAtMs, signal).catch(() => {}); + } + if (error instanceof LambdaMicrovmApiError && error.kind === 'not_found') { + throw new HostedAppMicrovmError( + 'hosted_app_boot_failed', + 'Recovered hosted app MicroVM no longer exists', + true, + error, + ); + } + /* A conflicting resume commonly means auto-resume or another request + * won the state transition. Poll the known id instead of discarding + * it or rotating the launch token. */ + if (!(error instanceof LambdaMicrovmApiError) || error.kind !== 'conflict') { + throw error; + } + } + } + await this.deps.sleep(Math.min(250, Math.max(1, deadlineAtMs - Date.now())), signal); + current = await this.client.getMicrovm(current.microvmId, signal); + } + } + + async mintToken( + microvmId: string, + port: number, + callerSignal: AbortSignal, + ): Promise { + const deadlineAtMs = Date.now() + this.config.launchTimeoutMs; + const signal = AbortSignal.any([ + callerSignal, + AbortSignal.timeout(this.config.launchTimeoutMs), + ]); + try { + await this.deps.reserveOp('token', { + limitPerSecond: this.config.tokenTps, + deadlineAtMs, + signal, + }); + return await this.client.createMicrovmAuthToken({ + microvmId, + port, + ttlSeconds: this.config.authTokenTtlSeconds, + }, signal); + } catch (error) { + if (error instanceof LambdaMicrovmApiError && error.kind === 'throttled') { + await this.deps.poisonOp('token', deadlineAtMs, signal).catch(() => {}); + } + throw new HostedAppMicrovmError( + 'hosted_app_auth_failed', + error instanceof Error ? error.message : 'Could not authorize hosted app endpoint', + true, + error, + ); + } + } + + async waitForControlReady(vm: MicrovmDescription, callerSignal: AbortSignal): Promise { + const endpoint = normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? ''); + const deadlineAtMs = Date.now() + this.config.launchTimeoutMs; + let lastError: unknown; + let token: MicrovmAuthToken | undefined; + while (Date.now() < deadlineAtMs) { + callerSignal.throwIfAborted(); + try { + if (token == null || token.expiresAtMs <= Date.now() + this.config.healthTimeoutMs) { + token = await this.mintToken(vm.microvmId, this.config.controlPort, callerSignal); + } + const response = await this.deps.fetch(`${endpoint}/api/v2/health`, { + headers: { + [token.headerName]: token.token, + ...microvmPortHeaders(this.config.controlPort), + }, + signal: AbortSignal.any([ + callerSignal, + AbortSignal.timeout(this.config.healthTimeoutMs), + ]), + }); + await response.body?.cancel(); + if (response.ok) return; + lastError = new Error(`health returned ${response.status}`); + } catch (error) { + if (error instanceof HostedAppMicrovmError && error.code === 'hosted_app_auth_failed') throw error; + lastError = error; + } + await this.deps.sleep(250, callerSignal); + } + throw new HostedAppMicrovmError( + 'hosted_app_unhealthy', + `Hosted app control listener did not become ready: ${lastError instanceof Error ? lastError.message : 'unknown error'}`, + true, + lastError, + ); + } + + async startResidentApp( + vm: MicrovmDescription, + runtimeSessionId: string, + spec: ResidentHostedAppSpec, + callerSignal: AbortSignal, + ): Promise { + const token = await this.mintToken(vm.microvmId, this.config.controlPort, callerSignal); + const response = await this.deps.fetch( + `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/start`, + { + method: 'POST', + headers: { + [token.headerName]: token.token, + ...microvmPortHeaders(this.config.controlPort), + 'X-Runtime-Session-Id': runtimeSessionId, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(spec), + signal: AbortSignal.any([ + callerSignal, + AbortSignal.timeout(this.config.appStartTimeoutMs), + ]), + }, + ).catch(error => { + callerSignal.throwIfAborted(); + throw new HostedAppMicrovmError('hosted_app_start_unavailable', + 'Hosted app runner transport failed', true, error); + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new HostedAppMicrovmError( + 'hosted_app_start_failed', + `Hosted app runner rejected start (${response.status}): ${body.slice(0, 1_024)}`, + response.status >= 500, + undefined, + response.status, + ); + } + await response.body?.cancel(); + } + + async residentAppState( + vm: MicrovmDescription, + runtimeSessionId: string, + spec: ResidentHostedAppSpec, + signal: AbortSignal, + ): Promise<'starting' | 'running' | 'stopping' | 'stopped' | 'failed'> { + const token = await this.mintToken(vm.microvmId, this.config.controlPort, signal); + const response = await this.deps.fetch( + `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/status`, + { + headers: { + [token.headerName]: token.token, + ...microvmPortHeaders(this.config.controlPort), + 'X-Runtime-Session-Id': runtimeSessionId, + }, + signal: AbortSignal.any([signal, AbortSignal.timeout(this.config.healthTimeoutMs)]), + }, + ); + if (!response.ok) { + await response.body?.cancel(); + if (response.status === 404) return 'failed'; + throw new HostedAppMicrovmError('hosted_app_status_unavailable', + 'Hosted app runner status is unavailable', true); + } + const body = await response.json() as Record; + if (body.app_id !== spec.app_id || body.revision !== spec.revision + || !['starting', 'running', 'stopping', 'stopped', 'failed'].includes(String(body.state))) { + throw new HostedAppMicrovmError('hosted_app_status_invalid', + 'Hosted app runner status does not match the revision', true); + } + return body.state as 'starting' | 'running' | 'stopping' | 'stopped' | 'failed'; + } + + previewToken(microvmId: string, signal: AbortSignal): Promise { + return this.mintToken(microvmId, this.config.previewPort, signal); + } + + async terminate(microvmId: string): Promise { + try { + await this.client.terminateMicrovm( + microvmId, + AbortSignal.timeout(this.config.launchTimeoutMs), + ); + return true; + } catch (error) { + if (error instanceof LambdaMicrovmApiError && error.kind === 'not_found') return true; + return false; + } + } +} diff --git a/service/src/hosted-app/preview-access.test.ts b/service/src/hosted-app/preview-access.test.ts new file mode 100644 index 00000000..1f383bf9 --- /dev/null +++ b/service/src/hosted-app/preview-access.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from 'bun:test'; +import { + hostedAppPreviewAuthorizeUrl, + hostedAppPreviewHostname, + hostedAppPreviewOwnerBinding, + hostedAppRuntimeIdFromHostname, + signHostedAppPreviewAccess, + verifyHostedAppPreviewAccess, +} from './preview-access'; + +const key = Buffer.alloc(32, 9); +const claims = { + hostedAppRuntimeId: `happ_${'a'.repeat(40)}`, + revision: 'rev-1', + ownerBinding: hostedAppPreviewOwnerBinding({ + tenantId: 'tenant-1', + canonicalUserId: 'user-1', + }, key), + expiresAt: 2_000_000, +}; + +describe('hosted app preview access', () => { + test('round-trips owner-bound claims and rejects tampering or expiry', () => { + const token = signHostedAppPreviewAccess(claims, key); + expect(verifyHostedAppPreviewAccess(token, key, 1_000_000)).toEqual(claims); + const parts = token.split('.'); + const signature = Buffer.from(parts[2] as string, 'base64url'); + signature[0] ^= 1; + parts[2] = signature.toString('base64url'); + expect(() => verifyHostedAppPreviewAccess(parts.join('.'), key, 1_000_000)).toThrow('invalid'); + expect(() => verifyHostedAppPreviewAccess(`${token}=`, key, 1_000_000)).toThrow('malformed'); + expect(() => verifyHostedAppPreviewAccess(token, key, claims.expiresAt)).toThrow('expired'); + }); + + test('binds the capability to one immutable app revision', () => { + const token = signHostedAppPreviewAccess(claims, key); + expect(verifyHostedAppPreviewAccess(token, key, 1_000_000).revision).toBe('rev-1'); + expect(() => signHostedAppPreviewAccess({ + ...claims, + revision: '../rev-2', + }, key)).toThrow('claims are invalid'); + }); + + test('blinds tenant and user identities into a stable keyed owner binding', () => { + const first = hostedAppPreviewOwnerBinding({ + tenantId: 'tenant-1', canonicalUserId: 'user-1', + }, key); + const second = hostedAppPreviewOwnerBinding({ + tenantId: 'tenant-1', canonicalUserId: 'user-2', + }, key); + expect(first).not.toContain('tenant-1'); + expect(first).not.toContain('user-1'); + expect(first).not.toBe(second); + }); + + test('maps the opaque runtime id to one wildcard host and clean exchange URL', () => { + const host = hostedAppPreviewHostname(claims.hostedAppRuntimeId, 'https://apps.example.test'); + expect(host).toBe(`happ-${'a'.repeat(40)}.apps.example.test`); + expect(hostedAppRuntimeIdFromHostname(host, 'https://apps.example.test')).toBe( + claims.hostedAppRuntimeId, + ); + expect(hostedAppRuntimeIdFromHostname('apps.example.test', 'https://apps.example.test')).toBeUndefined(); + const url = new URL(hostedAppPreviewAuthorizeUrl( + claims.hostedAppRuntimeId, + 'https://apps.example.test', + 'signed-token', + )); + expect(url.hostname).toBe(host); + expect(url.pathname).toBe('/__codeapi/authorize'); + expect(url.searchParams.get('token')).toBe('signed-token'); + }); +}); diff --git a/service/src/hosted-app/preview-access.ts b/service/src/hosted-app/preview-access.ts new file mode 100644 index 00000000..374b34ea --- /dev/null +++ b/service/src/hosted-app/preview-access.ts @@ -0,0 +1,144 @@ +import { createHmac, timingSafeEqual } from 'node:crypto'; +import type { HostedAppOwner } from './control-plane'; +import { HOSTED_APP_REVISION_PATTERN } from './spec'; + +const PREVIEW_TOKEN_VERSION = 'v1'; +const RUNTIME_ID_PATTERN = /^happ_[0-9a-f]{40}$/; + +export interface HostedAppPreviewClaims { + hostedAppRuntimeId: string; + revision: string; + ownerBinding: string; + expiresAt: number; +} + +export class HostedAppPreviewAccessError extends Error {} + +export function hostedAppPreviewOwnerBinding(owner: HostedAppOwner, key: Buffer): string { + if (!owner.tenantId || !owner.canonicalUserId) { + throw new HostedAppPreviewAccessError('preview owner is invalid'); + } + return createHmac('sha256', key) + .update('hosted-app-preview-owner-v1\0') + .update(owner.tenantId, 'utf8') + .update('\0') + .update(owner.canonicalUserId, 'utf8') + .digest('base64url'); +} + +function signature(payload: string, key: Buffer): Buffer { + if (key.length !== 32) throw new HostedAppPreviewAccessError('preview signing key must be 32 bytes'); + return createHmac('sha256', key).update(PREVIEW_TOKEN_VERSION).update('.').update(payload).digest(); +} + +function decodeBase64url(raw: string, maxBytes: number): Buffer { + if (!/^[A-Za-z0-9_-]+$/.test(raw)) { + throw new HostedAppPreviewAccessError('preview access token is malformed'); + } + const decoded = Buffer.from(raw, 'base64url'); + if (decoded.length > maxBytes || decoded.toString('base64url') !== raw) { + throw new HostedAppPreviewAccessError('preview access token is malformed'); + } + return decoded; +} + +export function signHostedAppPreviewAccess( + claims: HostedAppPreviewClaims, + key: Buffer, +): string { + if (!RUNTIME_ID_PATTERN.test(claims.hostedAppRuntimeId)) { + throw new HostedAppPreviewAccessError('hosted app runtime id is malformed'); + } + if ( + !Number.isSafeInteger(claims.expiresAt) + || claims.expiresAt <= 0 + || !HOSTED_APP_REVISION_PATTERN.test(claims.revision) + || !/^[A-Za-z0-9_-]{43}$/.test(claims.ownerBinding) + ) { + throw new HostedAppPreviewAccessError('preview claims are invalid'); + } + const payload = Buffer.from(JSON.stringify({ + r: claims.hostedAppRuntimeId, + v: claims.revision, + o: claims.ownerBinding, + e: claims.expiresAt, + }), 'utf8').toString('base64url'); + return `${PREVIEW_TOKEN_VERSION}.${payload}.${signature(payload, key).toString('base64url')}`; +} + +export function verifyHostedAppPreviewAccess( + token: string, + key: Buffer, + now = Date.now(), +): HostedAppPreviewClaims { + if (token.length > 1_024) { + throw new HostedAppPreviewAccessError('preview access token is malformed'); + } + const [version, payload, signatureRaw, extra] = token.split('.'); + if (version !== PREVIEW_TOKEN_VERSION || !payload || !signatureRaw || extra !== undefined) { + throw new HostedAppPreviewAccessError('preview access token is malformed'); + } + const received = decodeBase64url(signatureRaw, 32); + const expected = signature(payload, key); + if (received.length !== expected.length || !timingSafeEqual(received, expected)) { + throw new HostedAppPreviewAccessError('preview access token is invalid'); + } + try { + const parsed = JSON.parse(decodeBase64url(payload, 512).toString('utf8')) as { + r?: unknown; v?: unknown; o?: unknown; e?: unknown; + }; + if ( + typeof parsed.r !== 'string' + || !RUNTIME_ID_PATTERN.test(parsed.r) + || typeof parsed.v !== 'string' + || !HOSTED_APP_REVISION_PATTERN.test(parsed.v) + || typeof parsed.o !== 'string' + || !/^[A-Za-z0-9_-]{43}$/.test(parsed.o) + || !Number.isSafeInteger(parsed.e) + || (parsed.e as number) <= now + ) { + throw new Error('claims invalid or expired'); + } + return { + hostedAppRuntimeId: parsed.r, + revision: parsed.v, + ownerBinding: parsed.o, + expiresAt: parsed.e as number, + }; + } catch (error) { + throw new HostedAppPreviewAccessError( + `preview access token claims are invalid: ${(error as Error).message}`, + ); + } +} + +export function hostedAppPreviewHostname(runtimeId: string, previewOrigin: string): string { + if (!RUNTIME_ID_PATTERN.test(runtimeId)) { + throw new HostedAppPreviewAccessError('hosted app runtime id is malformed'); + } + return `${runtimeId.replace('_', '-')}.${new URL(previewOrigin).hostname}`; +} + +export function hostedAppRuntimeIdFromHostname( + hostname: string, + previewOrigin: string, +): string | undefined { + const suffix = new URL(previewOrigin).hostname.toLowerCase(); + const lower = hostname.toLowerCase().replace(/\.$/, ''); + if (!lower.endsWith(`.${suffix}`)) return undefined; + const label = lower.slice(0, -(suffix.length + 1)); + if (!/^happ-[0-9a-f]{40}$/.test(label)) return undefined; + return label.replace('-', '_'); +} + +export function hostedAppPreviewAuthorizeUrl( + runtimeId: string, + previewOrigin: string, + token: string, +): string { + const origin = new URL(previewOrigin); + origin.hostname = hostedAppPreviewHostname(runtimeId, previewOrigin); + origin.pathname = '/__codeapi/authorize'; + origin.searchParams.set('token', token); + return origin.toString(); +} diff --git a/service/src/hosted-app/preview-gateway.ts b/service/src/hosted-app/preview-gateway.ts new file mode 100644 index 00000000..4ce6ed5d --- /dev/null +++ b/service/src/hosted-app/preview-gateway.ts @@ -0,0 +1,165 @@ +import type { NextFunction, Request, Response } from 'express'; +import { env } from '../config'; +import type { AuthenticatedRequest } from '../types'; +import { readRuntimeSessionRecord } from '../runtime-session/registry'; +import { HostedAppControlPlaneError } from './control-plane'; +import { + hostedAppRuntimeIdFromHostname, + HostedAppPreviewAccessError, + hostedAppPreviewOwnerBinding, + signHostedAppPreviewAccess, + verifyHostedAppPreviewAccess, +} from './preview-access'; +import { proxyHostedAppPreview } from './preview-proxy'; +import { applyHostedAppPreviewSecurityHeaders } from './proxy-policy'; + +const COOKIE_NAME = '__Host-codeapi-app'; +const PREVIEW_COOKIE_TTL_MS = 60 * 60_000; + +function rawHostname(req: Request): string | undefined { + const host = req.headers.host; + if (!host || /[\s/@\\]/.test(host)) return undefined; + try { + return new URL(`http://${host}`).hostname; + } catch { + return undefined; + } +} + +function cookie(req: Request, name: string): string | undefined { + for (const item of (req.headers.cookie ?? '').split(';')) { + const separator = item.indexOf('='); + if (separator < 0 || item.slice(0, separator).trim() !== name) continue; + try { + return decodeURIComponent(item.slice(separator + 1).trim()); + } catch { + return undefined; + } + } + return undefined; +} + +function previewKey(): Buffer { + return Buffer.from(env.HOSTED_APP_PREVIEW_SIGNING_KEY, 'base64'); +} + +function reject(res: Response, status: number, message: string): Response { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Referrer-Policy', 'no-referrer'); + return res.status(status).type('text/plain').send(message); +} + +export async function hostedAppPreviewGateway( + req: Request, + res: Response, + next: NextFunction, +): Promise { + if (!env.HOSTED_APPS_ENABLED || !env.HOSTED_APP_PREVIEW_ORIGIN) return next(); + const hostname = rawHostname(req); + const runtimeId = hostname + ? hostedAppRuntimeIdFromHostname(hostname, env.HOSTED_APP_PREVIEW_ORIGIN) + : undefined; + if (!runtimeId) return next(); + + /* A wildcard app host is a separate, unprivileged origin. Never fall through + * from it into CodeAPI routes, even when authentication fails. */ + /* App routes are arbitrary user data. Collapse them before the outer metrics + * middleware records its completion event so Prometheus labels stay bounded. */ + res.locals.codeapiMetricPath = '/hosted-app-preview/*'; + applyHostedAppPreviewSecurityHeaders(res); + try { + if (req.path === '/__codeapi/authorize') { + if (req.method !== 'GET' || typeof req.query.token !== 'string') { + reject(res, 400, 'Invalid preview authorization request'); + return; + } + const linkClaims = verifyHostedAppPreviewAccess(req.query.token, previewKey()); + if (linkClaims.hostedAppRuntimeId !== runtimeId) { + reject(res, 403, 'Preview authorization does not match this app'); + return; + } + const record = await readRuntimeSessionRecord(runtimeId); + if ( + !record?.hosted_app + || record.state !== 'RUNNING' + || record.hosted_app.revision !== linkClaims.revision + || !record.microvm_id + || !record.endpoint + || record.hard_deadline_at == null + || record.hard_deadline_at <= Date.now() + ) { + reject(res, 409, 'Hosted app is not running'); + return; + } + if (hostedAppPreviewOwnerBinding({ + tenantId: record.tenant_id, + canonicalUserId: record.canonical_user_id, + }, previewKey()) !== linkClaims.ownerBinding) { + reject(res, 403, 'Preview authorization does not match this owner'); + return; + } + const expiresAt = Math.min( + record.hard_deadline_at ?? Date.now() + PREVIEW_COOKIE_TTL_MS, + Date.now() + PREVIEW_COOKIE_TTL_MS, + ); + if (expiresAt <= Date.now()) { + reject(res, 409, 'Hosted app lease has expired'); + return; + } + const sessionToken = signHostedAppPreviewAccess({ + ...linkClaims, + expiresAt, + }, previewKey()); + const maxAge = Math.max(1, Math.floor((expiresAt - Date.now()) / 1_000)); + res.setHeader('Set-Cookie', [ + `${COOKIE_NAME}=${encodeURIComponent(sessionToken)}`, + 'Path=/', + 'HttpOnly', + 'Secure', + 'SameSite=Strict', + `Max-Age=${maxAge}`, + ].join('; ')); + res.setHeader('Cache-Control', 'no-store'); + res.redirect(303, '/'); + return; + } + + const sessionToken = cookie(req, COOKIE_NAME); + if (!sessionToken) { + reject(res, 401, 'Preview authorization required'); + return; + } + const claims = verifyHostedAppPreviewAccess(sessionToken, previewKey()); + if (claims.hostedAppRuntimeId !== runtimeId) { + reject(res, 403, 'Preview authorization does not match this app'); + return; + } + const publicOrigin = new URL(env.HOSTED_APP_PREVIEW_ORIGIN); + publicOrigin.hostname = hostname as string; + await proxyHostedAppPreview( + req as AuthenticatedRequest, + res, + { + hostedAppRuntimeId: runtimeId, + revision: claims.revision, + ownerBinding: claims.ownerBinding, + publicHost: publicOrigin.host, + }, + req.path, + ); + } catch (error) { + if (res.headersSent) { + res.destroy(error instanceof Error ? error : undefined); + return; + } + if (error instanceof HostedAppPreviewAccessError) { + reject(res, 401, 'Preview authorization failed'); + return; + } + if (error instanceof HostedAppControlPlaneError) { + reject(res, error.status, error.message); + return; + } + reject(res, 502, 'Hosted app preview is unavailable'); + } +} diff --git a/service/src/hosted-app/preview-proxy.test.ts b/service/src/hosted-app/preview-proxy.test.ts new file mode 100644 index 00000000..c096e22f --- /dev/null +++ b/service/src/hosted-app/preview-proxy.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, test } from 'bun:test'; +import type { RuntimeSessionRecord } from '../runtime-session/registry'; +import { + hostedAppPreviewCredentialUsable, + hostedAppForwardedQuery, + hostedAppUpstreamUrl, + rewriteHostedAppLocation, + hostedAppRequestFailure, + previewRecord as resolvePreviewRecord, +} from './preview-proxy'; +import { HostedAppControlPlaneError } from './control-plane'; + +test('refresh failure falls back only after rereading and reauthorizing a valid credential', async () => { + const record = previewRecord(); + record.hard_deadline_at = Date.now() + 60_000; + record.hosted_app!.preview_credential_expires_at = Date.now() + 30_000; + const target = { + hostedAppRuntimeId: record.runtime_session_id, + revision: 'rev-2', + identity: { tenantId: record.tenant_id, canonicalUserId: record.canonical_user_id }, + }; + let reads = 0; + const deps = { + read: async () => { reads++; return record; }, + refresh: async () => { throw new Error('worker unavailable'); }, + }; + expect(await resolvePreviewRecord(target, new AbortController().signal, deps) === record).toBe(true); + expect(reads).toBe(2); + reads = 0; + deps.read = async () => { + reads++; + return reads === 1 ? record : { ...record, canonical_user_id: 'another-owner' }; + }; + await expect(resolvePreviewRecord(target, new AbortController().signal, deps)) + .rejects.toThrow('Hosted app not found'); +}); + +test('preserves the request limit classification through fetch error wrapping', () => { + const limit = new HostedAppControlPlaneError('hosted_app_request_too_large', 'too large', 413); + expect(hostedAppRequestFailure(new TypeError('fetch failed', { cause: limit }))).toBe(limit); +}); + +test('bounds queued refresh well before credential expiry, then falls back', async () => { + const record = previewRecord(); + record.hard_deadline_at = Date.now() + 60_000; + record.hosted_app!.preview_credential_expires_at = Date.now() + 1_000; + let budget = 0; + let reads = 0; + const result = await resolvePreviewRecord({ + hostedAppRuntimeId: record.runtime_session_id, + identity: { tenantId: record.tenant_id, canonicalUserId: record.canonical_user_id }, + }, new AbortController().signal, { + read: async () => { reads++; return record; }, + refresh: async (_name, _data, _id, waitMs) => { + budget = waitMs!; + return new Promise(() => {}); // No worker / stalled admission. + }, + }); + expect(budget).toBeGreaterThan(0); + expect(budget).toBeLessThanOrEqual(500); + expect(reads).toBe(2); + expect(hostedAppPreviewCredentialUsable(result, { hostedAppRuntimeId: record.runtime_session_id })).toBe(true); +}); + +test('accepts only the exact public app origin and rejects network-path redirects', () => { + const upstream = 'https://vm.aws.example/app'; + const origin = 'https://app.apps.example.test'; + expect(rewriteHostedAppLocation(`${origin}/login`, upstream, origin)).toBe('/login'); + expect(rewriteHostedAppLocation('https://other.apps.example.test/login', upstream, origin)) + .toBeUndefined(); + expect(rewriteHostedAppLocation(`${origin}//attacker.test`, upstream, origin)).toBeUndefined(); +}); + +function previewRecord(): RuntimeSessionRecord { + return { + runtime_session_id: `happ_${'a'.repeat(40)}`, + tenant_id: 'tenant-1', + canonical_user_id: 'user-1', + state: 'RUNNING', + generation: 1, + launched_at: 1_000, + last_seen_at: 1_000, + hard_deadline_at: 20_000, + microvm_id: 'vm-1', + endpoint: 'https://vm.aws.example', + hosted_app: { + source_runtime_session_id: 'rt-source', + app_id: 'demo', + revision: 'rev-2', + spec_fingerprint: 'fingerprint', + spec: { + adapter: 'resident', + app_id: 'demo', + revision: 'rev-2', + language: 'node', + version: '22', + entrypoint: 'server.js', + cwd: '.', + args: [], + env: {}, + }, + checkpoint_key: 'checkpoint', + preview_credential: 'sealed', + preview_credential_expires_at: 15_000, + }, + }; +} + +describe('hosted app preview upstream URL', () => { + test('keeps protocol-relative and ordinary request paths on the AWS endpoint origin', () => { + const endpoint = 'https://vm.aws.example/'; + expect(hostedAppUpstreamUrl(endpoint, '/assets/app.js').toString()) + .toBe('https://vm.aws.example/assets/app.js'); + expect(hostedAppUpstreamUrl(endpoint, '//attacker.example/steal').origin) + .toBe('https://vm.aws.example'); + expect(hostedAppUpstreamUrl(endpoint, '//attacker.example/steal').pathname) + .toBe('//attacker.example/steal'); + }); + + test('rejects non-HTTPS and credential-bearing endpoints', () => { + expect(() => hostedAppUpstreamUrl('http://vm.aws.example', '/')).toThrow( + 'Hosted app endpoint is invalid', + ); + expect(() => hostedAppUpstreamUrl('https://user@vm.aws.example', '/')).toThrow( + 'Hosted app endpoint is invalid', + ); + }); + + test('resolves relative redirects against the current app route', () => { + const current = 'https://vm.aws.example/projects/demo/start?from=preview'; + expect(rewriteHostedAppLocation('../login?next=demo', current)) + .toBe('/projects/login?next=demo'); + expect(rewriteHostedAppLocation('?ready=true', current)) + .toBe('/projects/demo/start?ready=true'); + expect(rewriteHostedAppLocation('https://attacker.example/steal', current)).toBeUndefined(); + }); + + test('preserves signed and repeated query bytes without Express re-encoding', () => { + expect(hostedAppForwardedQuery('/download?sig=a%2Fb+c&tag=one&tag=two')) + .toBe('?sig=a%2Fb+c&tag=one&tag=two'); + expect(hostedAppForwardedQuery('/download')).toBe(''); + }); + + test('rejects stale revisions, expired leases, and expired preview credentials', () => { + const record = previewRecord(); + const target = { + hostedAppRuntimeId: record.runtime_session_id, + revision: 'rev-2', + ownerBinding: 'owner', + }; + expect(hostedAppPreviewCredentialUsable(record, target, 10_000)).toBe(true); + expect(hostedAppPreviewCredentialUsable(record, { ...target, revision: 'rev-1' }, 10_000)) + .toBe(false); + expect(hostedAppPreviewCredentialUsable({ + ...record, + hard_deadline_at: 10_000, + }, target, 10_000)).toBe(false); + expect(hostedAppPreviewCredentialUsable({ + ...record, + hosted_app: { + ...record.hosted_app!, + preview_credential_expires_at: 10_000, + }, + }, target, 10_000)).toBe(false); + }); +}); diff --git a/service/src/hosted-app/preview-proxy.ts b/service/src/hosted-app/preview-proxy.ts new file mode 100644 index 00000000..18d1a2e0 --- /dev/null +++ b/service/src/hosted-app/preview-proxy.ts @@ -0,0 +1,343 @@ +import type { Response } from 'express'; +import { Readable, Transform } from 'node:stream'; +import { env } from '../config'; +import { readRuntimeSessionRecord } from '../runtime-session/registry'; +import type { RuntimeSessionRecord } from '../runtime-session/registry'; +import { captureTraceCarrier } from '../telemetry'; +import type { AuthenticatedRequest } from '../types'; +import { assertHostedAppOwned, HostedAppControlPlaneError } from './control-plane'; +import { openHostedAppCredential, parseHostedAppCredentialKey } from './credential'; +import { + hostedAppProxyRequestHeaders, + hostedAppProxyResponseHeaders, +} from './proxy-policy'; +import { submitHostedAppJob } from './queue'; +import { hostedAppPreviewOwnerBinding } from './preview-access'; + +const PREVIEW_REFRESH_SKEW_MS = 60_000; + +/** Bound admission as well as the queue result wait (Redis may be reconnecting). */ +async function waitForPreviewRefresh(work: Promise, waitMs: number, signal: AbortSignal): Promise { + signal.throwIfAborted(); + let timer: ReturnType | undefined; + let onAbort: () => void = () => {}; + try { + await Promise.race([ + work.catch(() => undefined), + new Promise(resolve => { + onAbort = resolve; + timer = setTimeout(resolve, waitMs); + signal.addEventListener('abort', onAbort, { once: true }); + }), + ]); + signal.throwIfAborted(); + } finally { + clearTimeout(timer); + signal.removeEventListener('abort', onAbort); + } +} + +export interface HostedAppPreviewTarget { + hostedAppRuntimeId: string; + revision?: string; + sourceRuntimeSessionId?: string; + identity?: { tenantId: string; canonicalUserId: string }; + ownerBinding?: string; + publicHost?: string; +} + +export function hostedAppPreviewRecordUsable( + record: RuntimeSessionRecord | null | undefined, + resolved: HostedAppPreviewTarget, + now = Date.now(), +): record is RuntimeSessionRecord & { + hosted_app: NonNullable; + microvm_id: string; + endpoint: string; +} { + return Boolean( + record?.hosted_app + && record.state === 'RUNNING' + && record.microvm_id + && record.endpoint + && record.hard_deadline_at != null + && record.hard_deadline_at > now + && (resolved.revision == null || record.hosted_app.revision === resolved.revision) + ); +} + +export function hostedAppPreviewCredentialUsable( + record: RuntimeSessionRecord | null | undefined, + resolved: HostedAppPreviewTarget, + now = Date.now(), + minimumTtlMs = 0, +): record is RuntimeSessionRecord & { + hosted_app: NonNullable & { + preview_credential: string; + preview_credential_expires_at: number; + }; + microvm_id: string; + endpoint: string; +} { + return hostedAppPreviewRecordUsable(record, resolved, now) + && Boolean( + record.hosted_app.preview_credential + && record.hosted_app.preview_credential_expires_at != null + && record.hosted_app.preview_credential_expires_at > now + minimumTtlMs + ); +} + +export async function previewRecord( + resolved: HostedAppPreviewTarget, + signal: AbortSignal, + deps = { read: readRuntimeSessionRecord, refresh: submitHostedAppJob }, +) { + let record = await deps.read(resolved.hostedAppRuntimeId, { signal }); + if (!hostedAppPreviewRecordUsable(record, resolved)) { + throw new HostedAppControlPlaneError( + 'hosted_app_not_running', + 'Hosted app is not running', + 409, + true, + ); + } + assertPreviewRecordAuthorized(record, resolved); + if (!hostedAppPreviewCredentialUsable(record, resolved, Date.now(), PREVIEW_REFRESH_SKEW_MS)) { + // Leave time to use the existing credential when no worker can refresh it. + const remaining = Math.min(record.hard_deadline_at!, + record.hosted_app.preview_credential_expires_at ?? Date.now()) - Date.now(); + const waitMs = remaining > 0 ? Math.max(1, Math.min(2_000, remaining / 2)) : 2_000; + await waitForPreviewRefresh(deps.refresh('hosted-app:refresh-preview', { + operation: 'refresh-preview', + hostedAppRuntimeId: resolved.hostedAppRuntimeId, + tenantId: record.tenant_id, + canonicalUserId: record.canonical_user_id, + _otel: captureTraceCarrier(), + }, `happ-refresh-${resolved.hostedAppRuntimeId}-${Math.floor(Date.now() / 30_000)}`, waitMs), waitMs, signal); + record = await deps.read(resolved.hostedAppRuntimeId, { signal }); + } + if (!hostedAppPreviewCredentialUsable(record, resolved)) { + throw new HostedAppControlPlaneError( + 'hosted_app_preview_unavailable', + 'Hosted app preview credential is unavailable', + 503, + true, + ); + } + /* Refresh waits on another worker and then rereads durable state. Recheck + * ownership on that second snapshot instead of relying on the pre-await + * authorization decision. */ + assertPreviewRecordAuthorized(record, resolved); + return record; +} + +function assertPreviewRecordAuthorized( + record: RuntimeSessionRecord, + resolved: HostedAppPreviewTarget, +): void { + if (resolved.identity) { + assertHostedAppOwned(record, resolved.identity, resolved.sourceRuntimeSessionId); + } else { + const key = Buffer.from(env.HOSTED_APP_PREVIEW_SIGNING_KEY, 'base64'); + const expected = hostedAppPreviewOwnerBinding({ + tenantId: record.tenant_id, + canonicalUserId: record.canonical_user_id, + }, key); + if (!resolved.ownerBinding || resolved.ownerBinding !== expected) { + throw new HostedAppControlPlaneError('hosted_app_not_found', 'Hosted app not found', 404); + } + } +} + +function requestBody(req: AuthenticatedRequest): BodyInit | undefined { + if (req.method === 'GET' || req.method === 'HEAD') return undefined; + const declaredLength = Number(req.headers['content-length']); + if (Number.isFinite(declaredLength) && declaredLength > env.MAX_FILE_SIZE) { + throw new HostedAppControlPlaneError( + 'hosted_app_request_too_large', + `Hosted app request exceeds ${env.MAX_FILE_SIZE} bytes`, + 413, + ); + } + if (req.body != null) { + const body = Buffer.isBuffer(req.body) || typeof req.body === 'string' + ? req.body + : JSON.stringify(req.body); + if (Buffer.byteLength(body) > env.MAX_FILE_SIZE) { + throw new HostedAppControlPlaneError( + 'hosted_app_request_too_large', + `Hosted app request exceeds ${env.MAX_FILE_SIZE} bytes`, + 413, + ); + } + return body as unknown as BodyInit; + } + let bytes = 0; + const limiter = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + bytes += chunk.length; + callback( + bytes > env.MAX_FILE_SIZE + ? new HostedAppControlPlaneError( + 'hosted_app_request_too_large', + `Hosted app request exceeds ${env.MAX_FILE_SIZE} bytes`, + 413, + ) + : null, + chunk, + ); + }, + }); + return req.pipe(limiter) as unknown as BodyInit; +} + +export function rewriteHostedAppLocation( + location: string, + currentUpstreamUrl: string, + publicOrigin?: string, +): string | undefined { + try { + const upstreamOrigin = new URL(currentUpstreamUrl); + const destination = new URL(location, upstreamOrigin); + if (destination.username || destination.password) return undefined; + if (destination.origin !== upstreamOrigin.origin + && destination.origin !== publicOrigin) return undefined; + // A relative Location beginning with // would redirect to another host. + if (destination.pathname.startsWith('//')) return undefined; + return `${destination.pathname}${destination.search}${destination.hash}`; + } catch { + return undefined; + } +} + +/** Preserve the AWS endpoint origin even when a hostile request path begins + * with `//` (which URL resolution would otherwise treat as a new host). */ +export function hostedAppUpstreamUrl(endpoint: string, upstreamPath: string): URL { + let upstream: URL; + try { + upstream = new URL(endpoint); + } catch { + throw new HostedAppControlPlaneError( + 'hosted_app_endpoint_invalid', + 'Hosted app endpoint is invalid', + 502, + true, + ); + } + if ( + upstream.protocol !== 'https:' + || upstream.username + || upstream.password + || !upstream.hostname + ) { + throw new HostedAppControlPlaneError( + 'hosted_app_endpoint_invalid', + 'Hosted app endpoint is invalid', + 502, + true, + ); + } + upstream.pathname = upstreamPath.startsWith('/') ? upstreamPath : `/${upstreamPath}`; + upstream.search = ''; + upstream.hash = ''; + return upstream; +} + +export function hostedAppForwardedQuery(originalUrl: string): string { + const queryStart = originalUrl.indexOf('?'); + return queryStart < 0 ? '' : originalUrl.slice(queryStart); +} + +export async function proxyHostedAppPreview( + req: AuthenticatedRequest, + res: Response, + resolved: HostedAppPreviewTarget, + upstreamPath: string, +): Promise { + const controller = new AbortController(); + const abort = (): void => controller.abort(new Error('Preview client disconnected')); + req.once('aborted', abort); + res.once('close', abort); + try { + const record = await previewRecord(resolved, controller.signal); + const key = parseHostedAppCredentialKey(env.HOSTED_APP_CREDENTIAL_KEY); + const token = openHostedAppCredential( + resolved.hostedAppRuntimeId, + record.hosted_app?.preview_credential as string, + key, + ); + if (token.expiresAtMs <= Date.now()) { + throw new HostedAppControlPlaneError( + 'hosted_app_preview_unavailable', + 'Hosted app preview credential is unavailable', + 503, + true, + ); + } + const endpoint = `${record.endpoint?.replace(/\/+$/, '')}/`; + const upstream = hostedAppUpstreamUrl(endpoint, upstreamPath); + /* A reverse proxy must not parse and rebuild signed/repeated query strings: + * decoding and re-encoding changes their byte representation, while nested + * values can disappear entirely through Express's query parser. Preserve + * the original query bytes and constrain only the upstream origin/path. */ + upstream.search = hostedAppForwardedQuery(req.originalUrl); + const init: RequestInit & { duplex?: 'half' } = { + method: req.method, + headers: hostedAppProxyRequestHeaders( + req.headers, + token, + env.HOSTED_APP_PREVIEW_PORT, + resolved.publicHost ? { + host: resolved.publicHost, + protocol: new URL(env.HOSTED_APP_PREVIEW_ORIGIN).protocol === 'http:' ? 'http' : 'https', + } : undefined, + ), + body: requestBody(req), + redirect: 'manual', + signal: controller.signal, + }; + if (init.body != null && !Buffer.isBuffer(init.body) && typeof init.body !== 'string') { + init.duplex = 'half'; + } + const response = await fetch(upstream, init).catch(error => { + throw hostedAppRequestFailure(error); + }); + res.status(response.status); + hostedAppProxyResponseHeaders(response.headers).forEach((value, name) => { + res.setHeader(name, value); + }); + const location = response.headers.get('location'); + /* Resolve relative redirects against the request URL, not the AWS endpoint + * root. Framework redirects such as `Location: ../login` depend on the + * current route while the same-origin check still strips the AWS origin. */ + const safeLocation = location + ? rewriteHostedAppLocation(location, upstream.toString(), + resolved.publicHost ? `https://${resolved.publicHost}` : undefined) + : undefined; + if (location && !safeLocation) { + await response.body?.cancel().catch(() => {}); + return res.status(502).type('text/plain').send('Hosted app returned an unsafe redirect'); + } + if (safeLocation) res.setHeader('Location', safeLocation); + if (req.method === 'HEAD' || response.body == null) return res.end(); + const body = Readable.fromWeb(response.body as never); + body.once('error', error => res.destroy(error)); + body.pipe(res); + } finally { + req.removeListener('aborted', abort); + if (res.writableEnded) res.removeListener('close', abort); + } +} + +/** Node fetch wraps errors raised by a streaming request body in TypeError. */ +export function hostedAppRequestFailure(error: unknown): unknown { + let current = error; + const seen = new Set(); + while (current instanceof Error && !seen.has(current)) { + if (current instanceof HostedAppControlPlaneError + && current.code === 'hosted_app_request_too_large') return current; + seen.add(current); + current = (current as Error & { cause?: unknown }).cause; + } + return error; +} diff --git a/service/src/hosted-app/proxy-policy.test.ts b/service/src/hosted-app/proxy-policy.test.ts new file mode 100644 index 00000000..0553362f --- /dev/null +++ b/service/src/hosted-app/proxy-policy.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, test } from 'bun:test'; +import { + applyHostedAppPreviewSecurityHeaders, + hostedAppProxyRequestHeaders, + hostedAppProxyResponseHeaders, +} from './proxy-policy'; + +describe('hosted app preview proxy policy', () => { + test('constrains browser fetches and disables persistent workers', () => { + const headers = new Map(); + applyHostedAppPreviewSecurityHeaders({ + setHeader(name, value) { + headers.set(name.toLowerCase(), value); + }, + }); + + expect(headers.get('content-security-policy')).toContain("connect-src 'self'"); + expect(headers.get('content-security-policy')).toContain("worker-src 'none'"); + expect(headers.get('content-security-policy')).toContain("form-action 'self'"); + expect(headers.get('permissions-policy')).toContain('camera=()'); + expect(headers.get('x-dns-prefetch-control')).toBe('off'); + expect(headers.get('cross-origin-opener-policy')).toBe('same-origin'); + expect(headers.get('cache-control')).toBe('private, no-store'); + }); + + test('keeps CodeAPI identity and caller-supplied AWS headers out of the app', () => { + const headers = hostedAppProxyRequestHeaders({ + authorization: 'Bearer codeapi-secret', + cookie: 'librechat=session-secret', + 'x-api-key': 'api-secret', + 'x-forwarded-user': 'user-1', + 'x-aws-proxy-auth': 'attacker-token', + accept: 'text/event-stream', + 'x-app-action': 'move', + }, { + headerName: 'X-aws-proxy-auth', + token: 'worker-minted-token', + expiresAtMs: Date.now() + 60_000, + }, 3000, { + host: 'happ-safe.apps.example.test', + protocol: 'https', + }); + + expect(Object.fromEntries(headers)).toEqual({ + accept: 'text/event-stream', + 'x-app-action': 'move', + 'x-aws-proxy-auth': 'worker-minted-token', + 'x-aws-proxy-port': '3000', + 'x-forwarded-host': 'happ-safe.apps.example.test', + 'x-forwarded-proto': 'https', + }); + }); + + test('does not let a hosted app set cookies, caching, redirects, or security policy', () => { + const headers = hostedAppProxyResponseHeaders(new Headers({ + 'content-type': 'text/html', + 'cache-control': 'public, max-age=31536000', + expires: 'Wed, 21 Oct 2037 07:28:00 GMT', + 'set-cookie': 'session=owned', + location: 'https://internal-microvm.example/secret', + 'content-security-policy': "default-src *", + 'access-control-allow-origin': '*', + })); + + expect(Object.fromEntries(headers)).toEqual({ 'content-type': 'text/html' }); + }); +}); diff --git a/service/src/hosted-app/proxy-policy.ts b/service/src/hosted-app/proxy-policy.ts new file mode 100644 index 00000000..8f9bbe91 --- /dev/null +++ b/service/src/hosted-app/proxy-policy.ts @@ -0,0 +1,105 @@ +import type { IncomingHttpHeaders } from 'node:http'; +import { microvmPortHeaders, type MicrovmAuthToken } from '../runtime-session/lambda-client'; + +export interface HostedAppPreviewHeaderWriter { + setHeader(name: string, value: string): unknown; +} + +/** Browser-enforced guardrails owned by the trusted gateway, not user code. */ +export function applyHostedAppPreviewSecurityHeaders( + response: HostedAppPreviewHeaderWriter, +): void { + response.setHeader('Referrer-Policy', 'no-referrer'); + response.setHeader('X-Content-Type-Options', 'nosniff'); + response.setHeader('X-DNS-Prefetch-Control', 'off'); + response.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + /* The app origin and browser cache outlive an individual capability and app + * revision. User-controlled caching could otherwise replay old HTML/JS after + * the revision-bound cookie has expired or a replacement has landed. */ + response.setHeader('Cache-Control', 'private, no-store'); + /* User code has no server-side egress and should not regain it through the + * owner's browser. Disabling workers also prevents a service worker from one + * revision persisting on this stable app origin into a later revision. */ + response.setHeader('Content-Security-Policy', [ + "default-src 'self' data: blob:", + "connect-src 'self'", + "script-src 'self' 'unsafe-inline' 'unsafe-eval'", + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data: blob:", + "media-src 'self' data: blob:", + "font-src 'self' data:", + "worker-src 'none'", + "child-src 'none'", + "frame-src 'none'", + "object-src 'none'", + "base-uri 'none'", + "form-action 'self'", + "frame-ancestors 'self'", + ].join('; ')); + response.setHeader( + 'Permissions-Policy', + 'camera=(), microphone=(), geolocation=(), payment=(), usb=()', + ); +} + +const SAFE_REQUEST_HEADERS = new Set([ + 'accept', + 'accept-language', + 'cache-control', + 'content-type', + 'if-match', + 'if-modified-since', + 'if-none-match', + 'if-range', + 'if-unmodified-since', + 'range', + 'user-agent', +]); + +const SAFE_RESPONSE_HEADERS = new Set([ + 'accept-ranges', + 'content-disposition', + 'content-language', + 'content-range', + 'content-type', + 'etag', + 'last-modified', + 'vary', +]); + +/** Build a capability-minimal upstream request. In particular, never expose + * LibreChat/CodeAPI auth cookies, API keys, forwarded identity, or an + * attacker-supplied AWS proxy credential to the untrusted hosted app. */ +export function hostedAppProxyRequestHeaders( + source: IncomingHttpHeaders, + token: MicrovmAuthToken, + previewPort: number, + publicOrigin?: { host: string; protocol: 'https' | 'http' }, +): Headers { + const headers = new Headers({ + [token.headerName]: token.token, + ...microvmPortHeaders(previewPort), + }); + for (const [name, value] of Object.entries(source)) { + const lower = name.toLowerCase(); + if (!SAFE_REQUEST_HEADERS.has(lower) && !lower.startsWith('x-app-')) continue; + if (value == null) continue; + headers.set(name, Array.isArray(value) ? value.join(', ') : value); + } + headers.delete('content-length'); + if (publicOrigin) { + headers.set('X-Forwarded-Host', publicOrigin.host); + headers.set('X-Forwarded-Proto', publicOrigin.protocol); + } + return headers; +} +/** Cookies, redirects, CORS, and security-policy headers from user code must + * not mutate the CodeAPI/LibreChat origin. The narrow representation headers + * below are sufficient for HTML, assets, ranges, and SSE. */ +export function hostedAppProxyResponseHeaders(source: Headers): Headers { + const headers = new Headers(); + source.forEach((value, name) => { + if (SAFE_RESPONSE_HEADERS.has(name.toLowerCase())) headers.set(name, value); + }); + return headers; +} diff --git a/service/src/hosted-app/queue-deadline.test.ts b/service/src/hosted-app/queue-deadline.test.ts new file mode 100644 index 00000000..d93b9e46 --- /dev/null +++ b/service/src/hosted-app/queue-deadline.test.ts @@ -0,0 +1,15 @@ +import { expect, test } from 'bun:test'; +import { withHostedAppQueueDeadline } from './queue-deadline'; + +test('bounds unavailable Redis admission, not only an admitted job result', async () => { + const start = Date.now(); + await expect(withHostedAppQueueDeadline(() => new Promise(() => {}), 25)) + .rejects.toThrow('queue wait timed out'); + expect(Date.now() - start).toBeLessThan(500); +}); + +test('preserves completed results and failures within the queue deadline', async () => { + expect(await withHostedAppQueueDeadline(async () => 'ready', 1_000)).toBe('ready'); + await expect(withHostedAppQueueDeadline(async () => { throw new Error('admission failed'); }, 1_000)) + .rejects.toThrow('admission failed'); +}); diff --git a/service/src/hosted-app/queue-deadline.ts b/service/src/hosted-app/queue-deadline.ts new file mode 100644 index 00000000..007fe7b3 --- /dev/null +++ b/service/src/hosted-app/queue-deadline.ts @@ -0,0 +1,15 @@ +/** BullMQ starts its result timer after Redis readiness; bound admission too. + * A timed-out submission may still be admitted and must remain idempotent. */ +export async function withHostedAppQueueDeadline(work: () => Promise, waitMs: number): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + work(), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('Hosted app queue wait timed out')), waitMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} diff --git a/service/src/hosted-app/queue.ts b/service/src/hosted-app/queue.ts new file mode 100644 index 00000000..1c0c6904 --- /dev/null +++ b/service/src/hosted-app/queue.ts @@ -0,0 +1,78 @@ +import { Queue, QueueEvents } from 'bullmq'; +import { setMaxListeners } from 'node:events'; +import { nanoid } from 'nanoid'; +import { connection } from '../queue'; +import { hostedAppOperationTimeoutMs } from '../config'; +import logger from '../logger'; +import { withHostedAppQueueDeadline } from './queue-deadline'; +import type { HostedAppJobData, HostedAppJobName, HostedAppJobResult } from './jobs'; + +/** Hosted apps are stateful-only. A fixed isolated queue prevents an ordinary + * stateless worker from ever receiving an AWS lifecycle job. */ +export const HOSTED_APP_QUEUE_NAME = 'stateful-hosted-app-queue'; + +type HostedAppQueue = Queue< + HostedAppJobData, + HostedAppJobResult, + HostedAppJobName +>; + +let hostedAppQueue: HostedAppQueue | undefined; +let hostedAppQueueEvents: QueueEvents | undefined; + +function resources(): { queue: HostedAppQueue; events: QueueEvents } { + if (!hostedAppQueue || !hostedAppQueueEvents) { + hostedAppQueue = new Queue( + HOSTED_APP_QUEUE_NAME, + { connection }, + ); + hostedAppQueueEvents = new QueueEvents(HOSTED_APP_QUEUE_NAME, { connection }); + setMaxListeners(0, hostedAppQueue, hostedAppQueueEvents); + /* These resources are created after lifecycle startup, on first use, so the + * ordinary queue listener registration never sees them. An unhandled + * EventEmitter `error` would otherwise crash an API process during a Redis + * failover. */ + hostedAppQueue.on('error', error => { + logger.error('Hosted app queue error', { error }); + }); + hostedAppQueueEvents.on('error', error => { + logger.error('Hosted app queue events error', { error }); + }); + } + return { queue: hostedAppQueue, events: hostedAppQueueEvents }; +} + +/* A cold start can capture (pull + store), restore (load + push), launch, wait + * for control, start the process, and mint preview credentials. Budget every + * independently bounded leg so the queue waiter cannot abandon valid work. */ +const HOSTED_APP_OPERATION_WAIT_MS = hostedAppOperationTimeoutMs() + 15_000; + +export async function submitHostedAppJob( + name: HostedAppJobName, + data: HostedAppJobData, + jobId = `happ-${nanoid()}`, + waitMs = HOSTED_APP_OPERATION_WAIT_MS, +): Promise { + const { queue, events } = resources(); + return withHostedAppQueueDeadline(async () => { + const job = await queue.add(name, data, { + jobId, + removeOnComplete: { age: 3_600, count: 1_000 }, + removeOnFail: { age: 86_400, count: 1_000 }, + }); + return job.waitUntilFinished(events, waitMs); + }, waitMs); +} + +/** No-op unless this process submitted hosted-app work. Keeping construction + * lazy means disabled/default-profile services create no extra Redis clients. */ +export async function closeHostedAppQueueResources(): Promise { + const queue = hostedAppQueue; + const events = hostedAppQueueEvents; + hostedAppQueue = undefined; + hostedAppQueueEvents = undefined; + await Promise.all([ + ...(queue ? [queue.close()] : []), + ...(events ? [events.close()] : []), + ]); +} diff --git a/service/src/hosted-app/record.ts b/service/src/hosted-app/record.ts new file mode 100644 index 00000000..12b45e6c --- /dev/null +++ b/service/src/hosted-app/record.ts @@ -0,0 +1,38 @@ +import type { ResidentHostedAppSpec } from './spec'; + +/** Immutable identity retained independently of the ephemeral VM lease. */ +export interface HostedAppRevision { + tenantId: string; + canonicalUserId: string; + sourceRuntimeSessionId: string; + revision: string; + specFingerprint: string; + checkpointKey: string; +} + +/** Durable hosted-app fields carried by the existing fenced MicroVM registry. + * The registry's top-level runtime_session_id is the opaque `happ_*` lease id; + * `source_runtime_session_id` identifies the coding workspace checkpoint that + * was copied into this independent app-host VM. */ +export interface HostedAppRecordDetails { + source_runtime_session_id: string; + app_id: string; + revision: string; + spec_fingerprint: string; + spec: ResidentHostedAppSpec; + checkpoint_key: string; + preview_credential?: string; + preview_credential_expires_at?: number; +} + +export interface HostedAppPublicStatus { + app_id: string; + revision: string; + state: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed'; + preview_id: string; + /** Short-lived owner capability exchange URL on the isolated app origin. */ + preview_url?: string; + hard_deadline_at?: number; + updated_at: number; + error?: string; +} diff --git a/service/src/hosted-app/router.ts b/service/src/hosted-app/router.ts new file mode 100644 index 00000000..a6ded8ec --- /dev/null +++ b/service/src/hosted-app/router.ts @@ -0,0 +1,217 @@ +import { Router, type Response } from 'express'; +import type { AuthenticatedRequest } from '../types'; +import { env } from '../config'; +import { getExecutionIdentity } from '../execution-identity'; +import { checkServiceShutDown, checkServiceStartUp } from '../lifecycle'; +import { readRuntimeSessionRecord } from '../runtime-session/registry'; +import { + deriveRuntimeSessionId, + validateRuntimeSessionHint, + RuntimeSessionHintError, +} from '../runtime-session/id'; +import { captureTraceCarrier } from '../telemetry'; +import { executionLimiter } from '../middleware/limits'; +import { + assertHostedAppOwned, + HostedAppControlPlaneError, + hostedAppPublicStatus, +} from './control-plane'; +import { submitHostedAppJob } from './queue'; +import { + deriveHostedAppRuntimeId, + HostedAppSpecError, + parseHostedAppStartRequest, + validateHostedAppId, +} from './spec'; +import { + hostedAppPreviewAuthorizeUrl, + hostedAppPreviewOwnerBinding, + signHostedAppPreviewAccess, +} from './preview-access'; +import type { HostedAppPublicStatus } from './record'; +import type { HostedAppPreviewTarget } from './preview-proxy'; + +const router = Router(); +const PREVIEW_LINK_TTL_MS = 5 * 60_000; + +function presentStatus( + status: HostedAppPublicStatus, + owner: { tenantId: string; canonicalUserId: string }, +): HostedAppPublicStatus { + if (status.state !== 'running') return status; + const key = Buffer.from(env.HOSTED_APP_PREVIEW_SIGNING_KEY, 'base64'); + const token = signHostedAppPreviewAccess({ + hostedAppRuntimeId: status.preview_id, + revision: status.revision, + ownerBinding: hostedAppPreviewOwnerBinding(owner, key), + expiresAt: Date.now() + PREVIEW_LINK_TTL_MS, + }, key); + return { + ...status, + preview_url: hostedAppPreviewAuthorizeUrl( + status.preview_id, + env.HOSTED_APP_PREVIEW_ORIGIN, + token, + ), + }; +} + +function unavailable(res: Response): Response | undefined { + if (!env.HOSTED_APPS_ENABLED) { + return res.status(404).json({ error: 'hosted_apps_disabled', message: 'Not Found' }); + } + if (checkServiceShutDown()) { + return res.status(503).json({ error: 'service_shutting_down', message: 'Service is shutting down' }); + } + if (checkServiceStartUp()) { + return res.status(503).json({ error: 'service_starting', message: 'Service is starting up' }); + } + return undefined; +} + +function target( + req: AuthenticatedRequest, + rawAppId: unknown, + rawHint: unknown, +): HostedAppPreviewTarget & { + appId: string; + sourceRuntimeSessionId: string; + identity: { tenantId: string; canonicalUserId: string }; +} { + const appId = validateHostedAppId(rawAppId); + const hint = validateRuntimeSessionHint(rawHint); + if (!hint) throw new HostedAppSpecError('runtime_session_hint is required'); + const identity = getExecutionIdentity(req); + const sourceRuntimeSessionId = deriveRuntimeSessionId({ + storageNamespace: identity.storageNamespace, + canonicalUserId: identity.canonicalUserId, + hint, + }); + return { + appId, + identity, + sourceRuntimeSessionId, + hostedAppRuntimeId: deriveHostedAppRuntimeId(sourceRuntimeSessionId, appId), + }; +} + +function parseWorkerFailure(error: unknown): HostedAppControlPlaneError | undefined { + const message = error instanceof Error ? error.message : String(error); + const jsonStart = message.indexOf('{'); + if (jsonStart < 0) return undefined; + try { + const parsed = JSON.parse(message.slice(jsonStart)) as Record; + if ( + typeof parsed.code === 'string' + && typeof parsed.message === 'string' + && typeof parsed.status === 'number' + ) { + return new HostedAppControlPlaneError( + parsed.code, + parsed.message, + parsed.status, + parsed.transient === true, + ); + } + } catch { + // Fall through to the generic failure below. + } + return undefined; +} + +function sendFailure(error: unknown, res: Response): Response { + if ( + error instanceof HostedAppSpecError + || error instanceof RuntimeSessionHintError + ) { + return res.status(error.status).json({ error: 'invalid_hosted_app_request', message: error.message }); + } + const known = error instanceof HostedAppControlPlaneError + ? error + : parseWorkerFailure(error); + if (known) { + return res.status(known.status).json({ + error: known.code, + message: known.message, + retryable: known.transient, + }); + } + return res.status(503).json({ + error: 'hosted_app_operation_failed', + message: 'Hosted app operation failed', + retryable: true, + }); +} + +router.post('/', executionLimiter, async (req: AuthenticatedRequest, res) => { + if (unavailable(res)) return; + try { + const parsed = parseHostedAppStartRequest(req.body); + const resolved = target(req, parsed.spec.app_id, parsed.runtimeSessionHint); + const result = await submitHostedAppJob('hosted-app:start', { + operation: 'start', + hostedAppRuntimeId: resolved.hostedAppRuntimeId, + sourceRuntimeSessionId: resolved.sourceRuntimeSessionId, + tenantId: resolved.identity.tenantId, + canonicalUserId: resolved.identity.canonicalUserId, + spec: parsed.spec, + _otel: captureTraceCarrier(), + }); + return res.status(200).json(presentStatus(result, { + tenantId: resolved.identity.tenantId, + canonicalUserId: resolved.identity.canonicalUserId, + })); + } catch (error) { + return sendFailure(error, res); + } +}); + +router.get('/:appId', executionLimiter, async (req: AuthenticatedRequest, res) => { + if (unavailable(res)) return; + try { + const resolved = target(req, req.params.appId, req.query.runtime_session_hint); + const record = await readRuntimeSessionRecord(resolved.hostedAppRuntimeId); + if (!record?.hosted_app) { + throw new HostedAppControlPlaneError('hosted_app_not_found', 'Hosted app not found', 404); + } + assertHostedAppOwned(record, { + tenantId: resolved.identity.tenantId, + canonicalUserId: resolved.identity.canonicalUserId, + }, resolved.sourceRuntimeSessionId); + const cached = hostedAppPublicStatus(record); + const status = cached.state === 'running' + ? await submitHostedAppJob('hosted-app:status', { + operation: 'status', + hostedAppRuntimeId: resolved.hostedAppRuntimeId, + tenantId: resolved.identity.tenantId, + canonicalUserId: resolved.identity.canonicalUserId, + _otel: captureTraceCarrier(), + }, undefined, 15_000) + : cached; + return res.status(200).json(presentStatus(status, { + tenantId: resolved.identity.tenantId, + canonicalUserId: resolved.identity.canonicalUserId, + })); + } catch (error) { + return sendFailure(error, res); + } +}); + +router.delete('/:appId', executionLimiter, async (req: AuthenticatedRequest, res) => { + if (unavailable(res)) return; + try { + const resolved = target(req, req.params.appId, req.query.runtime_session_hint); + const result = await submitHostedAppJob('hosted-app:stop', { + operation: 'stop', + hostedAppRuntimeId: resolved.hostedAppRuntimeId, + tenantId: resolved.identity.tenantId, + canonicalUserId: resolved.identity.canonicalUserId, + _otel: captureTraceCarrier(), + }); + return res.status(200).json(result); + } catch (error) { + return sendFailure(error, res); + } +}); + +export default router; diff --git a/service/src/hosted-app/source-checkpoint.test.ts b/service/src/hosted-app/source-checkpoint.test.ts new file mode 100644 index 00000000..9971ded0 --- /dev/null +++ b/service/src/hosted-app/source-checkpoint.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from 'bun:test'; +import type { RuntimeSessionRecord } from '../runtime-session/registry'; +import { captureHostedAppSourceCheckpoint } from './source-checkpoint'; + +const owner = { tenantId: 'tenant-1', canonicalUserId: 'user-1' }; + +function record(state: RuntimeSessionRecord['state']): RuntimeSessionRecord { + return { + runtime_session_id: 'rt_source', + tenant_id: owner.tenantId, + canonical_user_id: owner.canonicalUserId, + state, + generation: 1, + launched_at: 1, + last_seen_at: 1, + workspace_checkpoint: 'rtsx-checkpoints/rt_source/0001.tar.gz', + ...(state === 'RUNNING' + ? { microvm_id: 'vm-source', endpoint: 'https://vm-source.test' } + : {}), + }; +} + +function fixture(source: RuntimeSessionRecord | null) { + let current = source; + const calls: string[] = []; + return { + calls, + deps: { + waitForLock: async () => { calls.push('lock'); return 'source-lock'; }, + releaseLock: async () => { calls.push('release'); }, + retain: async (_id: string, key: string) => { calls.push('retain'); return key; }, + read: async () => { calls.push('read'); return current; }, + checkpoint: async () => { + calls.push('checkpoint'); + current = current ? { + ...current, + workspace_checkpoint: 'rtsx-checkpoints/rt_source/0002.tar.gz', + } : null; + return 'stored' as const; + }, + }, + }; +} + +describe('hosted app source checkpoint capture', () => { + test('holds one lock across a fresh live checkpoint and its committed pointer read', async () => { + const f = fixture(record('RUNNING')); + const key = await captureHostedAppSourceCheckpoint({ + runtimeSessionId: 'rt_source', owner, signal: new AbortController().signal, + lockWaitMs: 100, deps: f.deps, + }); + expect(key).toBe('rtsx-checkpoints/rt_source/0002.tar.gz'); + expect(f.calls).toEqual(['lock', 'read', 'checkpoint', 'read', 'retain', 'release']); + }); + + test('reuses a stopped workspace checkpoint without requiring a live source VM', async () => { + const f = fixture(record('TERMINATED')); + const key = await captureHostedAppSourceCheckpoint({ + runtimeSessionId: 'rt_source', owner, signal: new AbortController().signal, + lockWaitMs: 100, deps: f.deps, + }); + expect(key).toBe('rtsx-checkpoints/rt_source/0001.tar.gz'); + expect(f.calls).toEqual(['lock', 'read', 'retain', 'release']); + }); + + test('fails closed on owner mismatch and still releases the source lock', async () => { + const f = fixture({ ...record('TERMINATED'), canonical_user_id: 'user-2' }); + const error = await captureHostedAppSourceCheckpoint({ + runtimeSessionId: 'rt_source', owner, signal: new AbortController().signal, + lockWaitMs: 100, deps: f.deps, + }).catch(value => value); + expect(error.code).toBe('hosted_app_source_not_found'); + expect(f.calls).toEqual(['lock', 'read', 'release']); + }); +}); diff --git a/service/src/hosted-app/source-checkpoint.ts b/service/src/hosted-app/source-checkpoint.ts new file mode 100644 index 00000000..80141b69 --- /dev/null +++ b/service/src/hosted-app/source-checkpoint.ts @@ -0,0 +1,86 @@ +import type { RuntimeSessionRecord } from '../runtime-session/registry'; +import { HostedAppControlPlaneError, type HostedAppOwner } from './control-plane'; + +export interface HostedAppSourceCheckpointDeps { + waitForLock( + runtimeSessionId: string, + args: { waitMs: number; signal: AbortSignal }, + ): Promise; + releaseLock(runtimeSessionId: string, lockToken: string): Promise; + retain(runtimeSessionId: string, checkpointKey: string): Promise; + read( + runtimeSessionId: string, + args: { signal: AbortSignal }, + ): Promise; + checkpoint(args: { + runtimeSessionId: string; + lockToken: string; + signal: AbortSignal; + }): Promise<'stored' | 'skipped_busy' | 'skipped_state' | 'failed'>; +} + +/** Freeze one exact source workspace revision for an app. A live VM gets a + * fresh checkpoint; a stopped VM can reuse its last committed immutable + * checkpoint. The source lock spans commit and pointer read. */ +export async function captureHostedAppSourceCheckpoint(args: { + runtimeSessionId: string; + owner: HostedAppOwner; + signal: AbortSignal; + lockWaitMs: number; + deps: HostedAppSourceCheckpointDeps; +}): Promise { + const lockToken = await args.deps.waitForLock(args.runtimeSessionId, { + waitMs: args.lockWaitMs, + signal: args.signal, + }); + if (!lockToken) { + throw new HostedAppControlPlaneError( + 'hosted_app_source_busy', + 'The stateful workspace is busy; retry after its execution completes', + 409, + true, + ); + } + try { + let source = await args.deps.read(args.runtimeSessionId, { signal: args.signal }); + if ( + !source + || source.tenant_id !== args.owner.tenantId + || source.canonical_user_id !== args.owner.canonicalUserId + ) { + throw new HostedAppControlPlaneError( + 'hosted_app_source_not_found', + 'Stateful source workspace not found', + 404, + ); + } + if (source.state === 'RUNNING' && source.microvm_id && source.endpoint) { + const result = await args.deps.checkpoint({ + runtimeSessionId: args.runtimeSessionId, + lockToken, + signal: args.signal, + }); + if (result !== 'stored') { + throw new HostedAppControlPlaneError( + 'hosted_app_checkpoint_failed', + 'Could not capture the current stateful workspace revision', + 503, + true, + ); + } + source = await args.deps.read(args.runtimeSessionId, { signal: args.signal }); + } + if (!source?.workspace_checkpoint) { + throw new HostedAppControlPlaneError( + 'hosted_app_checkpoint_missing', + 'The stateful workspace has no durable checkpoint to host', + 503, + true, + ); + } + args.signal.throwIfAborted(); + return await args.deps.retain(args.runtimeSessionId, source.workspace_checkpoint); + } finally { + await args.deps.releaseLock(args.runtimeSessionId, lockToken); + } +} diff --git a/service/src/hosted-app/spec.test.ts b/service/src/hosted-app/spec.test.ts new file mode 100644 index 00000000..26417c7a --- /dev/null +++ b/service/src/hosted-app/spec.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from 'bun:test'; +import { + deriveHostedAppRuntimeId, + hostedAppSpecFingerprint, + parseHostedAppStartRequest, +} from './spec'; + +const valid = () => ({ + runtime_session_hint: 'conversation-1', + app_id: 'demo-app', + revision: 'rev-1', + language: 'node', + version: '>=22', + entrypoint: 'src/server.js', +}); + +describe('hosted app spec', () => { + test('normalizes the resident adapter defaults', () => { + expect(parseHostedAppStartRequest(valid())).toEqual({ + runtimeSessionHint: 'conversation-1', + spec: { + adapter: 'resident', + app_id: 'demo-app', + revision: 'rev-1', + language: 'node', + version: '>=22', + entrypoint: 'src/server.js', + cwd: '.', + args: [], + env: {}, + }, + }); + }); + + test('requires the stateful session hint and rejects unsupported adapters', () => { + expect(() => parseHostedAppStartRequest({ ...valid(), runtime_session_hint: '' })) + .toThrow('runtime_session_hint is required'); + expect(() => parseHostedAppStartRequest({ ...valid(), adapter: 'static' })) + .toThrow('adapter must be "resident"'); + }); + + test('rejects traversal, non-canonical paths, and runner-owned networking env', () => { + expect(() => parseHostedAppStartRequest({ ...valid(), entrypoint: '../server.js' })) + .toThrow('canonical relative path'); + expect(() => parseHostedAppStartRequest({ ...valid(), cwd: 'src/../src' })) + .toThrow('canonical relative path'); + expect(() => parseHostedAppStartRequest({ ...valid(), env: { port: '9999' } })) + .toThrow('runner-controlled'); + expect(() => parseHostedAppStartRequest({ ...valid(), env: { BASH_ENV: 'bootstrap.sh' } })) + .toThrow('runner-controlled'); + expect(() => parseHostedAppStartRequest({ ...valid(), env: { LD_PRELOAD: './evil.so' } })) + .toThrow('runner-controlled'); + }); + + test('fingerprints equivalent env maps identically and changed launch settings differently', () => { + const a = parseHostedAppStartRequest({ ...valid(), env: { B: '2', A: '1' } }).spec; + const b = parseHostedAppStartRequest({ ...valid(), env: { A: '1', B: '2' } }).spec; + const changed = { ...b, args: ['--changed'] }; + expect(hostedAppSpecFingerprint(a)).toBe(hostedAppSpecFingerprint(b)); + expect(hostedAppSpecFingerprint(a)).not.toBe(hostedAppSpecFingerprint(changed)); + }); + + test('derives a stable owner-scoped opaque runtime id', () => { + const first = deriveHostedAppRuntimeId('rt_owner_a', 'demo'); + expect(first).toMatch(/^happ_[0-9a-f]{40}$/); + expect(deriveHostedAppRuntimeId('rt_owner_a', 'demo')).toBe(first); + expect(deriveHostedAppRuntimeId('rt_owner_b', 'demo')).not.toBe(first); + expect(deriveHostedAppRuntimeId('rt_owner_a', 'other')).not.toBe(first); + }); +}); diff --git a/service/src/hosted-app/spec.ts b/service/src/hosted-app/spec.ts new file mode 100644 index 00000000..6ee2201f --- /dev/null +++ b/service/src/hosted-app/spec.ts @@ -0,0 +1,228 @@ +import { createHash } from 'node:crypto'; +import * as path from 'node:path'; +import { validateRuntimeSessionHint } from '../runtime-session/id'; + +const APP_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +export const HOSTED_APP_REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const MAX_ARGS = 64; +const MAX_ARG_BYTES = 4_096; +const MAX_ENV_VARS = 64; +const MAX_ENV_VALUE_BYTES = 4_096; +const MAX_ENV_BYTES = 32 * 1_024; +const MAX_PATH_LENGTH = 256; +const MAX_PATH_DEPTH = 10; +/* Mirror the app-host runner's root-launch filter. Rejecting here makes the + * immutable control-plane spec match what the runner actually applies. */ +const RESERVED_ENV_KEYS = new Set([ + 'OPENBLAS_NUM_THREADS', + 'MKL_NUM_THREADS', + 'OMP_NUM_THREADS', + 'SANDBOX_LANGUAGE', + 'HOME', + 'PATH', + 'TOOL_CALL_SOCKET', + 'PYTHONPATH', + 'PYTHONSTARTUP', + 'PYTHONHOME', + 'PYTHONEXECUTABLE', + 'PYTHONIOENCODING', + 'NODE_OPTIONS', + 'NODE_PATH', + 'BASH_ENV', + 'ENV', + 'PROMPT_COMMAND', + 'IFS', + 'SHELLOPTS', + 'BASHOPTS', + 'GLIBC_TUNABLES', + 'PTC_HISTORY_PATH', + 'PORT', + 'HOST', +]); +const RESERVED_ENV_PREFIXES = ['LD_', 'DYLD_', 'PTC_']; + +export interface ResidentHostedAppSpec { + adapter: 'resident'; + app_id: string; + revision: string; + language: string; + version: string; + entrypoint: string; + cwd: string; + args: string[]; + env: Record; +} + +export interface HostedAppStartRequest extends Omit { + adapter?: 'resident'; + runtime_session_hint: string; + cwd?: string; + args?: string[]; + env?: Record; +} + +export class HostedAppSpecError extends Error { + readonly status = 400; + + constructor(message: string) { + super(message); + this.name = 'HostedAppSpecError'; + } +} + +export function validateHostedAppId(value: unknown): string { + if (typeof value !== 'string' || !APP_ID_PATTERN.test(value)) { + throw new HostedAppSpecError('app_id is malformed'); + } + return value; +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const proto = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +function boundedString( + source: Record, + key: string, + maxBytes: number, +): string { + const value = source[key]; + if ( + typeof value !== 'string' + || value.length === 0 + || value.includes('\0') + || Buffer.byteLength(value, 'utf8') > maxBytes + ) { + throw new HostedAppSpecError(`${key} must be a non-empty bounded string`); + } + return value; +} + +function canonicalRelativePath(value: string, field: string, allowDot: boolean): string { + if ( + value.includes('\0') + || path.posix.isAbsolute(value) + || path.posix.normalize(value) !== value + || value.endsWith('/') + || (!allowDot && value === '.') + || value === '' + || value.length > MAX_PATH_LENGTH + || value.split('/').filter(Boolean).length > MAX_PATH_DEPTH + || value === '..' + || value.startsWith('../') + ) { + throw new HostedAppSpecError(`${field} must be a canonical relative path`); + } + return value; +} + +export function parseHostedAppStartRequest(raw: unknown): { + runtimeSessionHint: string; + spec: ResidentHostedAppSpec; +} { + if (!isPlainObject(raw)) throw new HostedAppSpecError('request body must be an object'); + if (raw.adapter !== undefined && raw.adapter !== 'resident') { + throw new HostedAppSpecError('adapter must be "resident"'); + } + + const runtimeSessionHint = validateRuntimeSessionHint(raw.runtime_session_hint); + if (!runtimeSessionHint) { + throw new HostedAppSpecError('runtime_session_hint is required'); + } + const appId = boundedString(raw, 'app_id', 64); + validateHostedAppId(appId); + const revision = boundedString(raw, 'revision', 128); + if (!HOSTED_APP_REVISION_PATTERN.test(revision)) { + throw new HostedAppSpecError('revision is malformed'); + } + const language = boundedString(raw, 'language', 64); + const version = boundedString(raw, 'version', 128); + const entrypoint = canonicalRelativePath( + boundedString(raw, 'entrypoint', MAX_PATH_LENGTH), + 'entrypoint', + false, + ); + const rawCwd = raw.cwd ?? '.'; + if (typeof rawCwd !== 'string') throw new HostedAppSpecError('cwd must be a string'); + const cwd = canonicalRelativePath(rawCwd, 'cwd', true); + + const rawArgs = raw.args ?? []; + if ( + !Array.isArray(rawArgs) + || rawArgs.length > MAX_ARGS + || rawArgs.some(value => ( + typeof value !== 'string' + || value.includes('\0') + || Buffer.byteLength(value, 'utf8') > MAX_ARG_BYTES + )) + ) { + throw new HostedAppSpecError(`args must contain at most ${MAX_ARGS} bounded strings`); + } + + const rawEnv = raw.env ?? {}; + if (!isPlainObject(rawEnv) || Object.keys(rawEnv).length > MAX_ENV_VARS) { + throw new HostedAppSpecError(`env must be an object with at most ${MAX_ENV_VARS} entries`); + } + const env: Record = {}; + let envBytes = 0; + for (const [key, value] of Object.entries(rawEnv)) { + if ( + !ENV_NAME_PATTERN.test(key) + || typeof value !== 'string' + || value.includes('\0') + || Buffer.byteLength(value, 'utf8') > MAX_ENV_VALUE_BYTES + ) { + throw new HostedAppSpecError(`env.${key} is invalid`); + } + const upperKey = key.toUpperCase(); + if ( + RESERVED_ENV_KEYS.has(upperKey) + || RESERVED_ENV_PREFIXES.some(prefix => upperKey.startsWith(prefix)) + ) { + throw new HostedAppSpecError(`env.${key} is runner-controlled`); + } + envBytes += Buffer.byteLength(key, 'utf8') + Buffer.byteLength(value, 'utf8'); + if (envBytes > MAX_ENV_BYTES) throw new HostedAppSpecError('env is too large'); + env[key] = value; + } + + return { + runtimeSessionHint, + spec: { + adapter: 'resident', + app_id: appId, + revision, + language, + version, + entrypoint, + cwd, + args: [...rawArgs] as string[], + env, + }, + }; +} + +function canonicalSpec(spec: ResidentHostedAppSpec): string { + return JSON.stringify({ + ...spec, + env: Object.fromEntries(Object.entries(spec.env).sort(([a], [b]) => a.localeCompare(b))), + }); +} + +export function hostedAppSpecFingerprint(spec: ResidentHostedAppSpec): string { + return createHash('sha256').update(canonicalSpec(spec), 'utf8').digest('hex'); +} + +/** Opaque, owner-scoped identity used by Redis and preview URLs. */ +export function deriveHostedAppRuntimeId(runtimeSessionId: string, appId: string): string { + const digest = createHash('sha256') + .update(runtimeSessionId, 'utf8') + .update('\0', 'utf8') + .update(appId, 'utf8') + .digest('hex') + .slice(0, 40); + return `happ_${digest}`; +} diff --git a/service/src/hosted-app/worker.test.ts b/service/src/hosted-app/worker.test.ts new file mode 100644 index 00000000..2b870122 --- /dev/null +++ b/service/src/hosted-app/worker.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from 'bun:test'; +import { HostedAppControlPlaneError } from './control-plane'; +import { HostedAppMicrovmError } from './microvm-runtime'; +import { serializedHostedAppFailure } from './worker'; + +function wire(error: unknown): Record { + return JSON.parse(serializedHostedAppFailure(error).message) as Record; +} + +describe('hosted app worker error boundary', () => { + test('preserves safe control-plane and runner validation classifications', () => { + expect(wire(new HostedAppControlPlaneError('busy', 'Try later', 409, true))) + .toEqual({ code: 'busy', status: 409, message: 'Try later', transient: true }); + expect(wire(new HostedAppMicrovmError( + 'hosted_app_start_failed', + 'runtime node@99 is not installed', + false, + undefined, + 400, + ))).toEqual({ + code: 'hosted_app_start_failed', + status: 400, + message: 'runtime node@99 is not installed', + transient: false, + }); + }); + + test('redacts provider details while preserving retryability', () => { + const result = wire(new HostedAppMicrovmError( + 'hosted_app_launch_failed', + 'AWS arn:secret leaked detail', + true, + )); + expect(result).toEqual({ + code: 'hosted_app_launch_failed', + status: 503, + message: 'Hosted app infrastructure operation failed', + transient: true, + }); + expect(JSON.stringify(result)).not.toContain('arn:secret'); + }); +}); diff --git a/service/src/hosted-app/worker.ts b/service/src/hosted-app/worker.ts new file mode 100644 index 00000000..4794d6c2 --- /dev/null +++ b/service/src/hosted-app/worker.ts @@ -0,0 +1,123 @@ +import { Worker } from 'bullmq'; +import { env, hostedAppOperationTimeoutMs } from '../config'; +import { connection } from '../queue'; +import { withSpan, withTraceContext } from '../telemetry'; +import { HostedAppControlPlaneError } from './control-plane'; +import { HostedAppMicrovmError } from './microvm-runtime'; +import type { HostedAppJob, HostedAppJobData, HostedAppJobName, HostedAppJobResult } from './jobs'; +import { HOSTED_APP_QUEUE_NAME } from './queue'; +import logger from '../logger'; +import { workerRunning } from '../metrics'; + +export function serializedHostedAppFailure(error: unknown): Error { + if (error instanceof HostedAppControlPlaneError) { + return new Error(JSON.stringify({ + code: error.code, + status: error.status, + message: error.message, + transient: error.transient, + })); + } + if (error instanceof HostedAppMicrovmError) { + const status = error.httpStatus >= 400 && error.httpStatus < 500 + ? error.httpStatus + : error.code === 'hosted_app_start_failed' && error.httpStatus === 504 + ? 504 + : 503; + return new Error(JSON.stringify({ + code: error.code, + status, + message: status < 500 + ? error.message + : status === 504 + ? 'Hosted app did not become ready before the startup deadline' + : 'Hosted app infrastructure operation failed', + transient: error.transient, + })); + } + return error instanceof Error ? error : new Error('Hosted app operation failed'); +} + +export async function processHostedAppJob(job: HostedAppJob): Promise { + return withTraceContext(job.data._otel, () => withSpan('codeapi.hosted_app.process', { + 'messaging.system': 'bullmq', + 'messaging.operation.name': job.name, + 'messaging.message.id': String(job.id ?? ''), + 'codeapi.hosted_app.id': job.data.hostedAppRuntimeId, + }, async () => { + const controller = new AbortController(); + const timeoutMs = job.name === 'hosted-app:status' ? 10_000 : hostedAppOperationTimeoutMs(); + const timer = setTimeout( + () => controller.abort(new Error(`Hosted app operation timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + timer.unref?.(); + try { + /* Keep AWS SDK and checkpoint-store construction out of default-profile + * workers; this import is reached only by an enabled hosted-app job. */ + const control = (await import('./factory')).getHostedAppControlPlane(); + if (job.name === 'hosted-app:status' && job.data.operation === 'status') { + return await control.status(job.data.hostedAppRuntimeId, job.data, controller.signal); + } + if (job.name === 'hosted-app:start' && job.data.operation === 'start') { + return await control.start({ + ...job.data, + signal: controller.signal, + }); + } + if (job.name === 'hosted-app:stop' && job.data.operation === 'stop') { + return await control.stop( + job.data.hostedAppRuntimeId, + job.data, + controller.signal, + ); + } + if ( + job.name === 'hosted-app:refresh-preview' + && job.data.operation === 'refresh-preview' + ) { + return await control.refreshPreview( + job.data.hostedAppRuntimeId, + job.data, + controller.signal, + ); + } + throw new HostedAppControlPlaneError( + 'hosted_app_job_invalid', + 'Hosted app job name and payload do not match', + 400, + ); + } catch (error) { + throw serializedHostedAppFailure(error); + } finally { + clearTimeout(timer); + } + }, 'CONSUMER')); +} + +export const hostedAppWorker: Worker< + HostedAppJobData, + HostedAppJobResult, + HostedAppJobName +> | undefined = env.HOSTED_APPS_ENABLED + ? new Worker(HOSTED_APP_QUEUE_NAME, processHostedAppJob, { + connection, + /* Lifecycle transitions are serialized again by their per-app Redis lock. + * This modest concurrency allows unrelated apps to launch in parallel while + * the fleet-wide AWS throttle remains authoritative. */ + concurrency: Math.max(1, Math.min(env.OTHER_CONCURRENCY, 4)), + }) + : undefined; + +if (hostedAppWorker) workerRunning.set({ worker_type: 'hosted-app' }, 1); + +hostedAppWorker?.on('failed', (job, error) => { + logger.error('Hosted app job failed', { jobId: job?.id, error }); +}); +hostedAppWorker?.on('error', error => { + logger.error('Hosted app worker error', { error }); + workerRunning.set({ worker_type: 'hosted-app' }, 0); +}); +hostedAppWorker?.on('closed', () => { + workerRunning.set({ worker_type: 'hosted-app' }, 0); +}); diff --git a/service/src/lifecycle.ts b/service/src/lifecycle.ts index 2aa47a69..f3747ceb 100644 --- a/service/src/lifecycle.ts +++ b/service/src/lifecycle.ts @@ -7,18 +7,20 @@ import { closeQueueConnections, } from './queue'; import { validateStartupAuthConfig } from './auth/startup'; -import { env } from './config'; +import { env, hostedAppOperationTimeoutMs } from './config'; import { validateApiBridgePolicy, validateApiHardenedConfig, validateApiSandboxBackendPolicy, validateExecutionProfilePolicy, + validateHostedAppsApiConfig, validateSandboxBackendPolicy, validateWorkerHardenedConfig, } from './secure-startup'; import logger from './logger'; import { shutdownTelemetry } from './telemetry'; import { configureExecutionProfileMetrics } from './metrics'; +import { closeHostedAppQueueResources } from './hosted-app/queue'; const { INSTANCE_ID } = env; let isShuttingDown = false; @@ -99,6 +101,7 @@ export async function startupApiOnly(): Promise { validateApiBridgePolicy(); validateExecutionProfilePolicy({ requireBackendMatch: false }); validateApiSandboxBackendPolicy(); + validateHostedAppsApiConfig(); /* No full validateSandboxBackendPolicy() here: an API-only pod authenticates and * enqueues jobs, it never constructs the Lambda backend or checkpoint store. * Bridge credentials are validated separately above because this process @@ -130,6 +133,9 @@ export async function startupWorkerOnly(): Promise { // Dynamically import workers to start them const { pyWorker, otherWorker } = await import('./workers'); + const hostedAppWorker = env.HOSTED_APPS_ENABLED + ? (await import('./hosted-app/worker')).hostedAppWorker + : undefined; registerWorkers(); @@ -145,6 +151,9 @@ export async function startupWorkerOnly(): Promise { throw new Error('Other worker is not running'); } logger.info('Workers health check passed'); + if (env.HOSTED_APPS_ENABLED && !hostedAppWorker?.isRunning()) { + throw new Error('Hosted app worker is not running'); + } }; checkWorkers(); @@ -161,6 +170,7 @@ async function gracefulStartup(): Promise { validateApiHardenedConfig(); validateWorkerHardenedConfig(); validateExecutionProfilePolicy(); + validateHostedAppsApiConfig(); validateSandboxBackendPolicy(); validateApiBridgePolicy(); await validateLifecycleAuthConfig(); @@ -171,6 +181,9 @@ async function gracefulStartup(): Promise { // Import workers (this starts them) const { pyWorker, otherWorker } = await import('./workers'); + const hostedAppWorker = env.HOSTED_APPS_ENABLED + ? (await import('./hosted-app/worker')).hostedAppWorker + : undefined; registerWorkers(); @@ -190,6 +203,9 @@ async function gracefulStartup(): Promise { throw new Error('Other worker is not running'); } logger.info('Workers health check passed'); + if (env.HOSTED_APPS_ENABLED && !hostedAppWorker?.isRunning()) { + throw new Error('Hosted app worker is not running'); + } }; checkWorkers(); @@ -225,18 +241,26 @@ export async function gracefulShutdown(): Promise { const shutdownTimeout = setTimeout(() => { logger.error('Shutdown timeout reached, forcing exit'); process.exit(1); - }, 30000); + }, hasWorkers && env.HOSTED_APPS_ENABLED + ? hostedAppOperationTimeoutMs() + env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS + 30_000 + : 30_000); try { if (hasWorkers) { // Worker shutdown: close workers gracefully const { pyWorker, otherWorker } = await import('./workers'); + const hostedAppWorker = env.HOSTED_APPS_ENABLED + ? (await import('./hosted-app/worker')).hostedAppWorker + : undefined; // Pause workers and wait for active jobs to complete // Note: We pause workers, NOT queues (queues are shared) // pause(false) = wait for active jobs to finish before resolving (doNotWaitActive=false) // pause(true) = return immediately without waiting for active jobs - const pauseAndDrain = async (worker: typeof pyWorker, name: string): Promise => { + const pauseAndDrain = async ( + worker: { pause(doNotWaitActive?: boolean): Promise }, + name: string, + ): Promise => { logger.info(`Pausing ${name} worker and waiting for active jobs to drain...`); try { // doNotWaitActive=false means wait for active jobs to complete @@ -249,19 +273,24 @@ export async function gracefulShutdown(): Promise { await Promise.all([ pauseAndDrain(pyWorker, 'Python'), - pauseAndDrain(otherWorker, 'Other') + pauseAndDrain(otherWorker, 'Other'), + ...(hostedAppWorker ? [pauseAndDrain(hostedAppWorker, 'Hosted app')] : []), ]); // Close workers await Promise.all([ pyWorker.close(), - otherWorker.close() + otherWorker.close(), + ...(hostedAppWorker ? [hostedAppWorker.close()] : []), ]); logger.info('Workers closed'); } // Close queue connections (both API and Worker need this) - await closeQueueConnections(); + await Promise.all([ + closeQueueConnections(), + closeHostedAppQueueResources(), + ]); logger.info('Queue connections closed'); // Only disconnect Redis if explicitly requested diff --git a/service/src/middleware/httpMetrics.test.ts b/service/src/middleware/httpMetrics.test.ts new file mode 100644 index 00000000..0a507a74 --- /dev/null +++ b/service/src/middleware/httpMetrics.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, test } from 'bun:test'; +import type { Request, Response } from 'express'; +import { httpMetricPath } from './httpMetrics'; + +describe('HTTP metric path overrides', () => { + test('collapses arbitrary hosted-app routes to one bounded label', () => { + const req = { path: '/generated/assets/nonce-123.js' } as Request; + const res = { + locals: { codeapiMetricPath: '/hosted-app-preview/*' }, + } as unknown as Response; + + expect(httpMetricPath(req, res)).toBe('/hosted-app-preview/*'); + expect(httpMetricPath(req, { locals: {} } as unknown as Response)).toBe(req.path); + }); +}); diff --git a/service/src/middleware/httpMetrics.ts b/service/src/middleware/httpMetrics.ts index 19a0d31f..a2024505 100644 --- a/service/src/middleware/httpMetrics.ts +++ b/service/src/middleware/httpMetrics.ts @@ -8,6 +8,11 @@ function expressRouteLabel(req: Request): string { return 'unmatched'; } +export function httpMetricPath(req: Request, res: Response): string { + const override = res.locals?.codeapiMetricPath; + return typeof override === 'string' && override.length > 0 ? override : req.path; +} + export function httpMetricsMiddleware(req: Request, res: Response, next: NextFunction): void { const start = httpLatencyStartMs(); let recorded = false; @@ -22,7 +27,7 @@ export function httpMetricsMiddleware(req: Request, res: Response, next: NextFun recordHttpRequest({ method: req.method, route: expressRouteLabel(req), - rawPath: req.path, + rawPath: httpMetricPath(req, res), statusCode, durationSeconds, }); diff --git a/service/src/runtime-session/checkpoint-store.test.ts b/service/src/runtime-session/checkpoint-store.test.ts index 918fec93..6107df2f 100644 --- a/service/src/runtime-session/checkpoint-store.test.ts +++ b/service/src/runtime-session/checkpoint-store.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test } from 'bun:test'; import * as fsp from 'fs/promises'; +import { Readable } from 'node:stream'; +import { env } from '../config'; import { MemoryCheckpointStore, MinioCheckpointStore, @@ -11,6 +13,83 @@ import { const BIG = 1_000_000; +test('adds retention tags only for hosted-enabled checkpoint writers', async () => { + const previous = env.HOSTED_APPS_ENABLED; + const tags: Array = []; + const store = new MinioCheckpointStore({ + async send(command: unknown) { + tags.push((command as { input: { Tagging?: string } }).input.Tagging); + return {}; + }, + }, { bucket: 'test' }); + try { + env.HOSTED_APPS_ENABLED = false; + await store.put('source', 1, Buffer.from('workspace')); + await store.commit('source', 1); + env.HOSTED_APPS_ENABLED = true; + await store.put('source', 2, Buffer.from('workspace')); + await store.commit('source', 2); + expect(tags).toEqual([undefined, undefined, 'codeapi-retention=rolling', 'codeapi-retention=rolling']); + } finally { env.HOSTED_APPS_ENABLED = previous; } +}); + +test('stores create-only durable revision manifests and fails closed on storage errors', async () => { + const objects = new Map(); + let outage = false; + const store = new MinioCheckpointStore({ + async send(command: unknown) { + const c = command as { constructor: { name: string }; input: any }; + if (outage) throw new Error('storage unavailable'); + if (c.constructor.name === 'PutObjectCommand') { + expect(c.input.IfNoneMatch).toBe('*'); + expect(c.input.Tagging).toBe('codeapi-retention=hosted'); + if (objects.has(c.input.Key)) throw { $metadata: { httpStatusCode: 412 } }; + objects.set(c.input.Key, c.input.Body); + return {}; + } + const data = objects.get(c.input.Key); + if (!data) throw { name: 'NoSuchKey' }; + return { Body: Readable.from([data]) }; + }, + }, { bucket: 'test' }); + const revision = { tenantId: 'tenant', canonicalUserId: 'user', sourceRuntimeSessionId: 'source', + revision: 'rev1', specFingerprint: 'fingerprint', checkpointKey: 'snapshot' }; + expect(await store.readHostedAppRevision('app', 'rev1')).toBeNull(); + await store.retainHostedAppRevision('app', revision); + expect(await store.retainHostedAppRevision('app', { ...revision, checkpointKey: 'later' })).toEqual(revision); + expect(await store.readHostedAppRevision('app', 'rev1')).toEqual(revision); + expect(await store.readHostedAppRevision('different-app', 'rev1')).toBeNull(); + outage = true; + await expect(store.readHostedAppRevision('app', 'rev1')).rejects.toThrow('storage unavailable'); +}); + +test('retained hosted snapshots survive subsequent source checkpoint pruning', async () => { + const objects = new Map(); + const source = checkpointObjectKey('rt_source', 1); + objects.set(source, 'immutable revision'); + const store = new MinioCheckpointStore({ + async send(command: unknown) { + const c = command as { constructor: { name: string }; input: any }; + if (c.constructor.name === 'CopyObjectCommand') { + expect(c.input.TaggingDirective).toBe('REPLACE'); + expect(c.input.Tagging).toBe('codeapi-retention=hosted'); + objects.set(c.input.Key, objects.get(decodeURIComponent(c.input.CopySource).slice(5))!); + } else if (c.constructor.name === 'ListObjectsV2Command') { + return { Contents: [...objects.keys()].map(Key => ({ Key })) }; + } else if (c.constructor.name === 'DeleteObjectsCommand') { + for (const { Key } of c.input.Delete.Objects) objects.delete(Key); + } + return {}; + }, + }, { bucket: 'test' }); + const retained = await store.retainForHostedApp('rt_source', source); + objects.set(checkpointObjectKey('rt_source', 2), 'later workspace'); + await store.pruneOlderThan('rt_source', 2); + expect(objects.has(source)).toBe(false); + expect(objects.get(retained)).toBe('immutable revision'); + await expect(store.retainForHostedApp('another-source', source)).rejects.toThrow('outside'); +}); + async function readStored( store: MemoryCheckpointStore, runtimeSessionId: string, diff --git a/service/src/runtime-session/checkpoint-store.ts b/service/src/runtime-session/checkpoint-store.ts index d971018a..e2ba326b 100644 --- a/service/src/runtime-session/checkpoint-store.ts +++ b/service/src/runtime-session/checkpoint-store.ts @@ -1,4 +1,5 @@ import { + CopyObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, @@ -10,9 +11,11 @@ import * as fs from 'fs'; import * as fsp from 'fs/promises'; import * as os from 'os'; import * as path from 'path'; +import { createHash } from 'node:crypto'; import { Readable, Transform } from 'stream'; import { pipeline } from 'stream/promises'; import { env } from '../config'; +import type { HostedAppRevision } from '../hosted-app/record'; interface S3SendClient { send( @@ -249,6 +252,7 @@ export class MinioCheckpointStore implements CheckpointStore { Body: source, ContentLength: size, ContentType: 'application/x-gtar', + Tagging: env.HOSTED_APPS_ENABLED ? 'codeapi-retention=rolling' : undefined, }), { abortSignal }); } finally { if (!Buffer.isBuffer(source)) source.destroy(); @@ -264,12 +268,85 @@ export class MinioCheckpointStore implements CheckpointStore { Body: marker, ContentLength: marker.length, ContentType: 'text/plain', + Tagging: env.HOSTED_APPS_ENABLED ? 'codeapi-retention=rolling' : undefined, })); } + /** Copy while the source lease is held, outside rolling checkpoint pruning. + * The immutable source key makes retries address the same retained object. */ + async retainForHostedApp(runtimeSessionId: string, sourceKey: string): Promise { + const prefix = checkpointPrefixFor(runtimeSessionId); + if (!sourceKey.startsWith(prefix) || !sourceKey.endsWith('.tar.gz')) { + throw new Error('Checkpoint pointer is outside the runtime session prefix'); + } + const key = `${prefix}hosted/${createHash('sha256').update(sourceKey).digest('hex')}.tar.gz`; + await this.send('hosted checkpoint retention', new CopyObjectCommand({ + Bucket: this.bucket, + Key: key, + CopySource: `${this.bucket}/${sourceKey}`.split('/').map(encodeURIComponent).join('/'), + TaggingDirective: 'REPLACE', + Tagging: 'codeapi-retention=hosted', + })); + return key; + } + + private hostedRevisionKey(runtimeId: string, revision: string): string { + return `${env.CHECKPOINT_PREFIX}hosted-revisions/${createHash('sha256') + .update(runtimeId).update('\0').update(revision).digest('hex')}.json`; + } + + async readHostedAppRevision(runtimeId: string, revision: string): Promise { + return this.withDeadline('hosted revision read', async abortSignal => { + let response: { Body?: Readable }; + try { + response = await this.client.send(new GetObjectCommand({ + Bucket: this.bucket, Key: this.hostedRevisionKey(runtimeId, revision), + }), { abortSignal }) as { Body?: Readable }; + } catch (error) { + if ((error as { name?: string }).name === 'NoSuchKey') return null; + throw error; // A storage outage is not proof that the revision is new. + } + const body = response.Body; + if (!body) throw new Error('Hosted revision body missing'); + try { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of body) { + const bytes = Buffer.from(chunk); + size += bytes.length; + if (size > 16_384) throw new Error('Hosted revision metadata too large'); + chunks.push(bytes); + } + const data = JSON.parse(Buffer.concat(chunks).toString('utf8')) as HostedAppRevision; + if (['tenantId', 'canonicalUserId', 'sourceRuntimeSessionId', 'revision', 'specFingerprint', 'checkpointKey'] + .some(key => typeof data?.[key as keyof HostedAppRevision] !== 'string') || data.revision !== revision) { + throw new Error('Hosted revision metadata invalid'); + } + return data; + } finally { body.destroy(); } + }); + } + + async retainHostedAppRevision(runtimeId: string, revision: HostedAppRevision): Promise { + try { + await this.send('hosted revision retention', new PutObjectCommand({ + Bucket: this.bucket, Key: this.hostedRevisionKey(runtimeId, revision.revision), + Body: JSON.stringify(revision), ContentType: 'application/json', + IfNoneMatch: '*', Tagging: 'codeapi-retention=hosted', + })); + return revision; + } catch (error) { + if ((error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode !== 412) throw error; + const existing = await this.readHostedAppRevision(runtimeId, revision.revision); + if (!existing) throw new Error('Hosted revision disappeared after conditional write'); + return existing; + } + } + async pruneOlderThan(runtimeSessionId: string, sequence: number): Promise { const keepKey = checkpointObjectKey(runtimeSessionId, sequence); const stale = (await this.listKeys(runtimeSessionId)).filter(key => { + if (key.startsWith(`${checkpointPrefixFor(runtimeSessionId)}hosted/`)) return false; if (key.endsWith('.tar.gz')) return key < keepKey; if (key.endsWith('.tar.gz.committed')) { return key.slice(0, -'.committed'.length) < keepKey; diff --git a/service/src/runtime-session/registry.ts b/service/src/runtime-session/registry.ts index f8d66507..deed37d9 100644 --- a/service/src/runtime-session/registry.ts +++ b/service/src/runtime-session/registry.ts @@ -6,6 +6,7 @@ import { RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS, } from '../config'; import logger from '../logger'; +import type { HostedAppRecordDetails } from '../hosted-app/record'; export { RUNTIME_SESSION_REDIS_COMMAND_TIMEOUT_MS } from '../config'; @@ -54,6 +55,10 @@ export interface RuntimeSessionRecord { workspace_checkpoint?: string; checkpointed_at?: number; last_error?: string; + /** Present only for a dedicated hosted-app MicroVM lease. Reusing this + * registry gives app launches the same lock fencing, crash recovery, and + * provider-idempotency guarantees as stateful execution VMs. */ + hosted_app?: HostedAppRecordDetails; } const SESS_PREFIX = 'rtsx:sess:'; diff --git a/service/src/secure-startup.test.ts b/service/src/secure-startup.test.ts index 78a49497..6820010a 100644 --- a/service/src/secure-startup.test.ts +++ b/service/src/secure-startup.test.ts @@ -6,6 +6,7 @@ import { validateApiSandboxBackendPolicy, validateEgressGatewayHardenedConfig, validateExecutionProfilePolicy, + validateHostedAppsApiConfig, validateSandboxBackendPolicy, validateWorkerHardenedConfig, } from './secure-startup'; @@ -47,6 +48,18 @@ const saved = { ledgerRequired: env.EGRESS_LEDGER_REQUIRED, fileServerUrl: env.EGRESS_GATEWAY_FILE_SERVER_URL, toolCallUrl: env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, + hostedAppsEnabled: env.HOSTED_APPS_ENABLED, + hostedAppImageArn: env.HOSTED_APP_IMAGE_ARN, + hostedAppImageVersion: env.HOSTED_APP_IMAGE_VERSION, + hostedAppControlPort: env.HOSTED_APP_CONTROL_PORT, + hostedAppPreviewPort: env.HOSTED_APP_PREVIEW_PORT, + hostedAppMaxDuration: env.HOSTED_APP_MAX_DURATION_SECONDS, + hostedAppIdle: env.HOSTED_APP_IDLE_SECONDS, + hostedAppSuspend: env.HOSTED_APP_SUSPEND_SECONDS, + hostedAppStartTimeout: env.HOSTED_APP_START_TIMEOUT_MS, + hostedAppCredentialKey: env.HOSTED_APP_CREDENTIAL_KEY, + hostedAppPreviewOrigin: env.HOSTED_APP_PREVIEW_ORIGIN, + hostedAppPreviewSigningKey: env.HOSTED_APP_PREVIEW_SIGNING_KEY, }; function restore(): void { @@ -89,6 +102,18 @@ function restore(): void { env.EGRESS_LEDGER_REQUIRED = saved.ledgerRequired; env.EGRESS_GATEWAY_FILE_SERVER_URL = saved.fileServerUrl; env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL = saved.toolCallUrl; + env.HOSTED_APPS_ENABLED = saved.hostedAppsEnabled; + env.HOSTED_APP_IMAGE_ARN = saved.hostedAppImageArn; + env.HOSTED_APP_IMAGE_VERSION = saved.hostedAppImageVersion; + env.HOSTED_APP_CONTROL_PORT = saved.hostedAppControlPort; + env.HOSTED_APP_PREVIEW_PORT = saved.hostedAppPreviewPort; + env.HOSTED_APP_MAX_DURATION_SECONDS = saved.hostedAppMaxDuration; + env.HOSTED_APP_IDLE_SECONDS = saved.hostedAppIdle; + env.HOSTED_APP_SUSPEND_SECONDS = saved.hostedAppSuspend; + env.HOSTED_APP_START_TIMEOUT_MS = saved.hostedAppStartTimeout; + env.HOSTED_APP_CREDENTIAL_KEY = saved.hostedAppCredentialKey; + env.HOSTED_APP_PREVIEW_ORIGIN = saved.hostedAppPreviewOrigin; + env.HOSTED_APP_PREVIEW_SIGNING_KEY = saved.hostedAppPreviewSigningKey; } afterEach(restore); @@ -603,3 +628,103 @@ describe('sandbox backend policy', () => { expect(() => validateSandboxBackendPolicy()).toThrow('LAMBDA_MICROVM_ALLOW_SHELL'); }); }); + +describe('hosted app startup policy', () => { + test('rejects remote bridges on both API and worker startup', () => { + configureHostedApps(); + env.SANDBOX_BACKEND = 'remote-bridge'; + expect(() => validateHostedAppsApiConfig()).toThrow('lambda-microvm'); + expect(() => validateSandboxBackendPolicy()).toThrow('lambda-microvm'); + }); + + test('requires HTTPS even outside production for Secure preview cookies', () => { + configureHostedApps(); + env.HOSTED_APP_PREVIEW_ORIGIN = 'http://apps.example.test'; + expect(() => validateHostedAppsApiConfig()).toThrow('bare HTTPS origin'); + }); + function configureHostedApps(): void { + env.HOSTED_APPS_ENABLED = true; + env.EXECUTION_PROFILE = 'stateful'; + env.EXECUTION_PROFILE_SOURCE = 'explicit'; + env.SANDBOX_BACKEND = 'lambda-microvm'; + env.RUNTIME_SESSION_MODE = 'affinity'; + env.SESSION_CHECKPOINTS = true; + env.HOSTED_APP_IMAGE_ARN = 'arn:aws:lambda:us-east-2:1:microvm-image:app-host'; + env.HOSTED_APP_IMAGE_VERSION = '4'; + env.HOSTED_APP_CONTROL_PORT = 8080; + env.HOSTED_APP_PREVIEW_PORT = 3000; + env.HOSTED_APP_MAX_DURATION_SECONDS = 28_800; + env.HOSTED_APP_IDLE_SECONDS = 300; + env.HOSTED_APP_SUSPEND_SECONDS = 900; + env.HOSTED_APP_START_TIMEOUT_MS = 30_000; + env.HOSTED_APP_CREDENTIAL_KEY = Buffer.alloc(32, 7).toString('base64'); + env.HOSTED_APP_PREVIEW_ORIGIN = 'https://apps.example.test'; + env.HOSTED_APP_PREVIEW_SIGNING_KEY = Buffer.alloc(32, 8).toString('base64'); + } + + test('API-only pods require the stateful profile and a 32-byte credential key', () => { + configureHostedApps(); + expect(() => validateHostedAppsApiConfig()).not.toThrow(); + + env.EXECUTION_PROFILE = 'default'; + expect(() => validateHostedAppsApiConfig()).toThrow('stateful execution profile'); + + env.EXECUTION_PROFILE = 'stateful'; + env.HOSTED_APP_CREDENTIAL_KEY = 'not-a-key'; + expect(() => validateHostedAppsApiConfig()).toThrow('exactly 32 bytes'); + + env.HOSTED_APP_CREDENTIAL_KEY = Buffer.alloc(32, 7).toString('base64'); + env.HOSTED_APP_PREVIEW_SIGNING_KEY = env.HOSTED_APP_CREDENTIAL_KEY; + expect(() => validateHostedAppsApiConfig()).toThrow('must be distinct'); + + env.HOSTED_APP_PREVIEW_SIGNING_KEY = Buffer.alloc(32, 8).toString('base64'); + env.HOSTED_APP_PREVIEW_ORIGIN = 'https://apps.example.test/path'; + expect(() => validateHostedAppsApiConfig()).toThrow('bare HTTPS origin'); + }); + + test('worker policy requires a pinned dedicated image and its fixed listener contract', () => { + configureHostedApps(); + /* Reuse the suite's valid Lambda/checkpoint baseline. */ + env.PTC_MODE = 'replay'; + env.HARDENED_SANDBOX_MODE = false; + env.LAMBDA_MICROVM_IMAGE_ARN = 'arn:aws:lambda:us-east-2:1:microvm-image:codeapi'; + env.LAMBDA_MICROVM_IMAGE_VERSION = '3'; + env.LAMBDA_MICROVM_PORT = 8080; + env.LAMBDA_MICROVM_MAX_DURATION_SECONDS = 28_800; + env.LAMBDA_MICROVM_IDLE_SECONDS = 1_800; + env.LAMBDA_MICROVM_SUSPEND_SECONDS = 1_800; + env.LAMBDA_MICROVM_AUTH_TOKEN_TTL_SECONDS = 300; + env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS = 60_000; + env.LAMBDA_MICROVM_HEALTH_TIMEOUT_MS = 5_000; + env.LAMBDA_MICROVM_LAUNCH_TPS = 4; + env.LAMBDA_MICROVM_TOKEN_TPS = 8; + env.LAMBDA_MICROVM_ALLOW_SHELL = false; + env.JOB_TIMEOUT = 300_000; + env.RUNTIME_SESSION_LOCK_WAIT_MS = 15_000; + env.CHECKPOINT_MAX_BYTES = 512 * 1024 * 1024; + env.CHECKPOINT_TIMEOUT_MS = 60_000; + process.env.MINIO_ENDPOINT = 'minio'; + process.env.CODEAPI_CHECKPOINT_BUCKET = 'codeapi-checkpoints'; + + expect(() => validateSandboxBackendPolicy()).not.toThrow(); + env.HOSTED_APP_PREVIEW_SIGNING_KEY = ''; + env.HOSTED_APP_PREVIEW_ORIGIN = ''; + expect(() => validateSandboxBackendPolicy()).not.toThrow(); + env.HOSTED_APP_MAX_DURATION_SECONDS = 60; + expect(() => validateSandboxBackendPolicy()).toThrow( + 'LAMBDA_MICROVM_APP_MAX_DURATION_SECONDS', + ); + env.HOSTED_APP_MAX_DURATION_SECONDS = 28_800; + env.HOSTED_APP_IMAGE_VERSION = undefined; + expect(() => validateSandboxBackendPolicy()).toThrow('LAMBDA_MICROVM_APP_IMAGE_VERSION'); + env.HOSTED_APP_IMAGE_VERSION = '4'; + env.HOSTED_APP_PREVIEW_PORT = 3001; + expect(() => validateSandboxBackendPolicy()).toThrow('preview port 3000'); + env.HOSTED_APP_PREVIEW_PORT = 3000; + env.HOSTED_APP_START_TIMEOUT_MS = 30_001; + expect(() => validateSandboxBackendPolicy()).toThrow('30000ms'); + env.HOSTED_APP_START_TIMEOUT_MS = 30_000; + process.env.LAMBDA_MICROVM_APP_PREVIEW_PORT = '4000'; + expect(() => validateSandboxBackendPolicy()).toThrow('cannot override'); + }); +}); diff --git a/service/src/secure-startup.ts b/service/src/secure-startup.ts index c1740790..79f69c42 100644 --- a/service/src/secure-startup.ts +++ b/service/src/secure-startup.ts @@ -102,6 +102,69 @@ export function validateWorkerHardenedConfig(): void { requireValue('CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY', env.EXECUTION_MANIFEST_PRIVATE_KEY); } +function hostedAppKey(name: string, raw: string): Buffer { + const normalized = raw.trim(); + const key = Buffer.from(normalized, 'base64'); + if ( + key.length !== 32 + || key.toString('base64').replace(/=+$/, '') !== normalized.replace(/=+$/, '') + ) { + throw new SecureStartupConfigError( + `${name} must be base64 encoding exactly 32 bytes`, + ); + } + return key; +} + +function validateHostedAppsSharedConfig(): Buffer { + if (env.SANDBOX_BACKEND !== 'lambda-microvm') { + throw new SecureStartupConfigError('Hosted apps require CODEAPI_SANDBOX_BACKEND=lambda-microvm'); + } + if (env.EXECUTION_PROFILE !== 'stateful' || env.RUNTIME_SESSION_MODE === 'stateless') { + throw new SecureStartupConfigError( + 'CODEAPI_HOSTED_APPS_ENABLED=true requires the stateful execution profile', + ); + } + return hostedAppKey('CODEAPI_HOSTED_APP_CREDENTIAL_KEY', env.HOSTED_APP_CREDENTIAL_KEY); +} + +/** API pods decrypt short-lived preview credentials but never receive AWS IAM + * control-plane permissions. Validate only their routing/key contract; the + * worker validator below owns image and checkpoint configuration. */ +export function validateHostedAppsApiConfig(): void { + if (!env.HOSTED_APPS_ENABLED) return; + const credentialKey = validateHostedAppsSharedConfig(); + const previewSigningKey = hostedAppKey( + 'CODEAPI_HOSTED_APP_PREVIEW_SIGNING_KEY', + env.HOSTED_APP_PREVIEW_SIGNING_KEY, + ); + if (credentialKey.equals(previewSigningKey)) { + throw new SecureStartupConfigError( + 'Hosted app credential and preview signing keys must be distinct', + ); + } + let previewOrigin: URL; + try { + previewOrigin = new URL(env.HOSTED_APP_PREVIEW_ORIGIN); + } catch { + throw new SecureStartupConfigError( + 'CODEAPI_HOSTED_APP_PREVIEW_ORIGIN must be an absolute URL', + ); + } + if ( + previewOrigin.protocol !== 'https:' + || previewOrigin.username + || previewOrigin.password + || previewOrigin.pathname !== '/' + || previewOrigin.search + || previewOrigin.hash + ) { + throw new SecureStartupConfigError( + 'CODEAPI_HOSTED_APP_PREVIEW_ORIGIN must be a bare HTTPS origin', + ); + } +} + /** * Make the endpoint identity trustworthy. Callers route by execution profile, * so accepting a contradictory backend/session tuple would silently send work @@ -147,6 +210,7 @@ export function validateExecutionProfilePolicy(options: { } export function validateApiSandboxBackendPolicy(): void { + if (env.HOSTED_APPS_ENABLED) validateHostedAppsSharedConfig(); if (env.BRIDGE_DYNAMIC_WORKERS && env.BRIDGE_AUTH_MODE !== 'paired') { throw new SecureStartupConfigError( 'Dynamic remote bridge workers require CODEAPI_BRIDGE_AUTH_MODE=paired', @@ -257,6 +321,58 @@ export function validateSandboxBackendPolicy(): void { ); } } + + if (env.HOSTED_APPS_ENABLED) { + /* Workers encrypt AWS port credentials but do not serve previews, so they + * need the credential key—not the separate URL-signing key or app origin. */ + validateHostedAppsSharedConfig(); + if (!env.SESSION_CHECKPOINTS) { + throw new SecureStartupConfigError( + 'CODEAPI_HOSTED_APPS_ENABLED=true requires CODEAPI_SESSION_CHECKPOINTS=true', + ); + } + requireValue('LAMBDA_MICROVM_APP_IMAGE_ARN', env.HOSTED_APP_IMAGE_ARN); + requireValue('LAMBDA_MICROVM_APP_IMAGE_VERSION', env.HOSTED_APP_IMAGE_VERSION); + for (const [name, expected] of [ + ['LAMBDA_MICROVM_APP_CONTROL_PORT', '8080'], + ['LAMBDA_MICROVM_APP_PREVIEW_PORT', '3000'], + ['LAMBDA_MICROVM_APP_START_TIMEOUT_MS', '30000'], + ] as const) { + const configured = process.env[name]?.trim(); + if (configured && configured !== expected) { + throw new SecureStartupConfigError( + `${name} cannot override the pinned app-host image contract (${expected})`, + ); + } + } + requireSafeWholeNumber('LAMBDA_MICROVM_APP_CONTROL_PORT', env.HOSTED_APP_CONTROL_PORT, 1_024); + requireSafeWholeNumber('LAMBDA_MICROVM_APP_PREVIEW_PORT', env.HOSTED_APP_PREVIEW_PORT, 1_024); + if (env.HOSTED_APP_CONTROL_PORT !== 8080 || env.HOSTED_APP_PREVIEW_PORT !== 3000) { + throw new SecureStartupConfigError( + 'Hosted app image contract requires control port 8080 and preview port 3000', + ); + } + requireSafeWholeNumber( + 'LAMBDA_MICROVM_APP_MAX_DURATION_SECONDS', + env.HOSTED_APP_MAX_DURATION_SECONDS, + 120, + ); + requireSafeWholeNumber('LAMBDA_MICROVM_APP_IDLE_SECONDS', env.HOSTED_APP_IDLE_SECONDS, 60); + requireSafeWholeNumber('LAMBDA_MICROVM_APP_SUSPEND_SECONDS', env.HOSTED_APP_SUSPEND_SECONDS, 0); + requireSafeWholeNumber('LAMBDA_MICROVM_APP_START_TIMEOUT_MS', env.HOSTED_APP_START_TIMEOUT_MS, 1); + if (env.HOSTED_APP_START_TIMEOUT_MS !== 30_000) { + throw new SecureStartupConfigError( + 'Hosted app image contract requires a 30000ms resident startup timeout', + ); + } + if ( + env.HOSTED_APP_MAX_DURATION_SECONDS > 28_800 + || env.HOSTED_APP_IDLE_SECONDS > 28_800 + || env.HOSTED_APP_SUSPEND_SECONDS > 28_800 + ) { + throw new SecureStartupConfigError('Hosted app lifetime controls must be at most 28800 seconds'); + } + } } export function validateEgressGatewayHardenedConfig(): void { diff --git a/service/src/service-api.ts b/service/src/service-api.ts index b5b7d52e..49516a29 100644 --- a/service/src/service-api.ts +++ b/service/src/service-api.ts @@ -10,11 +10,14 @@ import workspaceToolsRouter from './workspace-tools'; import { connection } from './queue'; import { env } from './config'; import logger from './logger'; +import hostedAppRouter from './hosted-app/router'; +import { hostedAppPreviewGateway } from './hosted-app/preview-gateway'; const app = express(); app.disable('x-powered-by'); app.set('trust proxy', 1); app.use(executionProfileMiddleware); +app.use(hostedAppPreviewGateway); const v1 = Router(); @@ -34,6 +37,7 @@ v1.use('/bridge', bridgeRouter); v1.use(apiKeyAuth); v1.use(workspaceToolsRouter); +v1.use('/hosted-apps', hostedAppRouter); v1.use(serviceRouter); v1.use(programmaticRouter); diff --git a/service/src/worker-server.ts b/service/src/worker-server.ts index 89049049..43f3574e 100644 --- a/service/src/worker-server.ts +++ b/service/src/worker-server.ts @@ -47,6 +47,7 @@ import { connection } from './queue'; * The workers are singletons, not created fresh on each import. */ import { pyWorker, otherWorker } from './workers'; +import { hostedAppWorker } from './hosted-app/worker'; const HEALTH_PORT = Number(process.env.WORKER_HEALTH_PORT) || 3113; @@ -101,14 +102,16 @@ const healthServer = http.createServer(async (req, res) => { // Check workers are running const pyRunning = pyWorker.isRunning(); const otherRunning = otherWorker.isRunning(); + const hostedAppsRunning = !env.HOSTED_APPS_ENABLED || hostedAppWorker?.isRunning() === true; - if (pyRunning && otherRunning) { + if (pyRunning && otherRunning && hostedAppsRunning) { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ status: 'healthy', workers: { python: pyRunning, - other: otherRunning + other: otherRunning, + hostedApps: hostedAppsRunning, }, config: { pythonConcurrency: env.PYTHON_CONCURRENCY, @@ -120,7 +123,7 @@ const healthServer = http.createServer(async (req, res) => { res.writeHead(503, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ status: 'unhealthy', - workers: { python: pyRunning, other: otherRunning } + workers: { python: pyRunning, other: otherRunning, hostedApps: hostedAppsRunning } })); } } catch (error) { @@ -136,8 +139,9 @@ const healthServer = http.createServer(async (req, res) => { await connection.ping(); const pyRunning = pyWorker.isRunning(); const otherRunning = otherWorker.isRunning(); + const hostedAppsRunning = !env.HOSTED_APPS_ENABLED || hostedAppWorker?.isRunning() === true; - if (pyRunning && otherRunning) { + if (pyRunning && otherRunning && hostedAppsRunning) { res.writeHead(200); res.end('ready'); } else { From 5cf56df54952fb16a12c2c8005d7aa5f6b2bc19b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 6 Sep 2026 17:38:29 -0400 Subject: [PATCH 044/116] =?UTF-8?q?=F0=9F=8F=BA=20fix:=20Report=20Artifact?= =?UTF-8?q?=20Delivery=20Failures=20(#119)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: Surface artifact delivery failures * fix: Await Worker Artifact Settlement in Blocking PTC --- api/src/api/v2.ts | 20 +-- api/src/delivery.test.ts | 73 +++++++++++ api/src/delivery.ts | 43 +++++++ api/src/job.ts | 2 + service/src/execution-log.test.ts | 17 ++- service/src/execution-log.ts | 21 +++- service/src/service/blocking-poll.test.ts | 66 ++++++++++ service/src/service/blocking-poll.ts | 76 ++++++++++++ service/src/service/programmatic-router.ts | 138 ++++----------------- service/src/types/service.ts | 11 ++ service/src/workers.ts | 3 + 11 files changed, 347 insertions(+), 123 deletions(-) create mode 100644 api/src/delivery.test.ts create mode 100644 api/src/delivery.ts create mode 100644 service/src/service/blocking-poll.test.ts create mode 100644 service/src/service/blocking-poll.ts diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 40ba2517..a883dc72 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -4,6 +4,7 @@ import type { TFile } from '../job'; import { getLatestRuntimeMatchingLanguageVersion, getRuntimes } from '../runtime'; import { logger } from '../logger'; import { config } from '../config'; +import { reconcileArtifactDelivery } from '../delivery'; import { Job, SessionWorkspaceDirtyError, @@ -563,15 +564,20 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn return new Set(); }); - const generatedIds = new Set(job.getGeneratedFileIds()); - const before = result.files.length; - result.files = result.files.filter( - f => !generatedIds.has(f.id) || uploaded.has(f.id), + const delivery = reconcileArtifactDelivery( + result.files, + job.getGeneratedFileIds(), + uploaded, ); - const dropped = before - result.files.length; - if (dropped > 0) { + result.files = delivery.files; + result.artifact_delivery = delivery.artifact_delivery; + if (delivery.artifact_delivery) { logger.warn( - { job: job.uuid, dropped, kept: result.files.length }, + { + job: job.uuid, + dropped: delivery.artifact_delivery.failed, + kept: result.files.length, + }, 'Pruned files from response because upload did not reach file_server', ); } diff --git a/api/src/delivery.test.ts b/api/src/delivery.test.ts new file mode 100644 index 00000000..eb488e8c --- /dev/null +++ b/api/src/delivery.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from 'bun:test'; +import { reconcileArtifactDelivery } from './delivery'; + +describe('reconcileArtifactDelivery', () => { + test('leaves successful and inherited file references unchanged', () => { + const files = [ + { id: 'generated', name: 'result.txt' }, + { id: 'inherited', name: 'input.txt', inherited: true as const }, + ]; + + expect( + reconcileArtifactDelivery( + files, + ['generated'], + new Set(['generated']), + ), + ).toEqual({ files }); + }); + + test('reports a complete delivery failure without returning phantom references', () => { + const files = [ + { id: 'generated', name: 'result.txt' }, + { id: 'inherited', name: 'input.txt', inherited: true as const }, + ]; + + expect( + reconcileArtifactDelivery(files, ['generated'], new Set()), + ).toEqual({ + files: [{ id: 'inherited', name: 'input.txt', inherited: true }], + artifact_delivery: { + code: 'artifact_delivery_failed', + status: 'failed', + attempted: 1, + delivered: 0, + failed: 1, + }, + }); + }); + + test('reports partial delivery and counts only expected generated ids', () => { + const files = [ + { id: 'first', name: 'first.txt' }, + { id: 'second', name: 'second.txt' }, + ]; + + expect( + reconcileArtifactDelivery( + files, + ['first', 'second'], + new Set(['first', 'unknown']), + ), + ).toEqual({ + files: [{ id: 'first', name: 'first.txt' }], + artifact_delivery: { + code: 'artifact_delivery_failed', + status: 'partial', + attempted: 2, + delivered: 1, + failed: 1, + }, + }); + }); + + test('does not report a failure when there were no generated files', () => { + const files = [ + { id: 'inherited', name: 'input.txt', inherited: true as const }, + ]; + + expect(reconcileArtifactDelivery(files, [], new Set())).toEqual({ + files, + }); + }); +}); diff --git a/api/src/delivery.ts b/api/src/delivery.ts new file mode 100644 index 00000000..ff159d6f --- /dev/null +++ b/api/src/delivery.ts @@ -0,0 +1,43 @@ +export interface ArtifactDeliveryFailure { + code: 'artifact_delivery_failed'; + status: 'partial' | 'failed'; + attempted: number; + delivered: number; + failed: number; +} + +export interface ArtifactDeliveryResult { + files: T[]; + artifact_delivery?: ArtifactDeliveryFailure; +} + +/** Removes unusable generated refs while preserving an explicit delivery failure for callers. */ +export function reconcileArtifactDelivery( + files: T[], + generatedFileIds: Iterable, + uploadedFileIds: ReadonlySet, +): ArtifactDeliveryResult { + const generatedIds = new Set(generatedFileIds); + if (generatedIds.size === 0) return { files }; + + let delivered = 0; + const retained = files.filter(file => { + if (!generatedIds.has(file.id)) return true; + if (!uploadedFileIds.has(file.id)) return false; + delivered++; + return true; + }); + const failed = generatedIds.size - delivered; + if (failed === 0) return { files: retained }; + + return { + files: retained, + artifact_delivery: { + code: 'artifact_delivery_failed', + status: delivered === 0 ? 'failed' : 'partial', + attempted: generatedIds.size, + delivered, + failed, + }, + }; +} diff --git a/api/src/job.ts b/api/src/job.ts index 1e78ae75..610b748b 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -7,6 +7,7 @@ import * as fsp from 'fs/promises'; import { pipeline } from 'stream/promises'; import { Readable, Transform } from 'stream'; import type { Logger } from 'pino'; +import type { ArtifactDeliveryFailure } from './delivery'; import type { NsJailResult } from './nsjail'; import type { Runtime } from './runtime'; import { logger as rootLogger } from './logger'; @@ -676,6 +677,7 @@ interface ExecuteResult { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRef[]; + artifact_delivery?: ArtifactDeliveryFailure; } const jobQueue: Array<() => void> = []; diff --git a/service/src/execution-log.test.ts b/service/src/execution-log.test.ts index d78fafc4..f04bad90 100644 --- a/service/src/execution-log.test.ts +++ b/service/src/execution-log.test.ts @@ -20,6 +20,14 @@ describe('execution log summaries', () => { { id: 'file_1', name: 'a.txt', inherited: true }, { id: 'file_2', name: 'b.txt', modified_from: { id: 'file_1', storage_session_id: 'sess_old' } }, ], + artifact_delivery: { + code: 'artifact_delivery_failed', + status: 'partial', + attempted: 3, + delivered: 2, + failed: 1, + detail: 'private storage failure', + }, run: { code: 0, stdout: 'top secret stdout', @@ -33,9 +41,17 @@ describe('execution log summaries', () => { expect(JSON.stringify(summary)).not.toContain('top secret stdout'); expect(JSON.stringify(summary)).not.toContain('sensitive stderr'); expect(JSON.stringify(summary)).not.toContain('combined output'); + expect(JSON.stringify(summary)).not.toContain('private storage failure'); expect(summary).toMatchObject({ session_id: 'sess_123', files: { count: 2, inheritedCount: 1, modifiedCount: 1 }, + artifact_delivery: { + code: 'artifact_delivery_failed', + status: 'partial', + attempted: 3, + delivered: 2, + failed: 1, + }, run: { stdout: { length: 17, present: true }, stderr: { length: 16, present: true }, @@ -56,4 +72,3 @@ describe('execution log summaries', () => { expect(summary).toEqual({ count: 3, skillCount: 1, agentCount: 1, userCount: 1 }); }); }); - diff --git a/service/src/execution-log.ts b/service/src/execution-log.ts index 311169e8..48a93cce 100644 --- a/service/src/execution-log.ts +++ b/service/src/execution-log.ts @@ -18,9 +18,28 @@ type SandboxResponseLike = { language?: unknown; version?: unknown; files?: unknown; + artifact_delivery?: unknown; run?: RunLike; }; +function summarizeArtifactDelivery(value: unknown): Record | undefined { + if (value == null || typeof value !== 'object' || Array.isArray(value)) return undefined; + const delivery = value as { + code?: unknown; + status?: unknown; + attempted?: unknown; + delivered?: unknown; + failed?: unknown; + }; + return { + code: delivery.code, + status: delivery.status, + attempted: delivery.attempted, + delivered: delivery.delivered, + failed: delivery.failed, + }; +} + export function summarizeText(value: unknown): { length: number; present: boolean } { if (typeof value !== 'string') { return { length: 0, present: false }; @@ -67,6 +86,7 @@ export function summarizeSandboxResponse(data: SandboxResponseLike): Record ({ jobCompleted: tick >= 2 }), + getBlockingResult: async () => result, + getPending: async () => ({ status: 'completed' }), + isNotFound: (error: unknown) => error === 'missing', + sleep: async (): Promise => { tick++; }, + now: () => tick, + }; +} + +describe('blocking worker settlement', () => { + test('completed callback waits for delayed upload reconciliation', async () => { + const deps = fixture(); + expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ + status: 'completed', stdout: result.stdout, stderr: '', files: [], + artifact_delivery: result.artifact_delivery, + }); + expect(deps.now()).toBe(2); + }); + + test('missing callback session still receives the worker result', async () => { + const deps = fixture(); + deps.getPending = async (): Promise => { throw 'missing'; }; + expect((await pollBlockingExecution('exec', 5, deps)).artifact_delivery).toEqual(result.artifact_delivery); + }); + + test('never reports success if uploads remain unsettled at timeout', async () => { + expect(await pollBlockingExecution('exec', 1, fixture())).toEqual({ status: 'error' }); + }); + + test('accepts the legacy inline worker result during rolling deployments', async () => { + const deps = fixture(); + expect(await pollBlockingExecution('exec', 5, { + ...deps, + getExecutionState: async () => ({ jobCompleted: true, jobResult: result }), + getBlockingResult: async () => null, + })).toMatchObject({ status: 'completed', artifact_delivery: result.artifact_delivery }); + }); + + test('preserves waiting calls and worker failures', async () => { + expect(await pollBlockingExecution('exec', 5, { + ...fixture(), + getPending: async () => ({ status: 'waiting', pending_calls: [ + { call_id: 'call', tool_name: 'search', tool_input: { query: 'hello' } }, + ] }), + })).toEqual({ status: 'waiting', pending_calls: [ + { id: 'call', name: 'search', input: { query: 'hello' } }, + ] }); + expect(await pollBlockingExecution('exec', 5, { + ...fixture(), getExecutionState: async () => ({ jobError: 'worker failed' }), + })).toEqual({ status: 'error' }); + }); +}); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts new file mode 100644 index 00000000..8ce07fa8 --- /dev/null +++ b/service/src/service/blocking-poll.ts @@ -0,0 +1,76 @@ +import type * as t from '../types'; + +export interface BlockingPendingState { + status: string; + pending_calls?: Array<{ + call_id: string; + tool_name: string; + tool_input: Record; + }>; +} + +export interface BlockingPollDependencies { + getExecutionState(id: string): Promise<{ + jobCompleted?: boolean; + jobResult?: t.ExecuteResult; + jobError?: string; + } | null>; + getBlockingResult(id: string): Promise; + getPending(id: string): Promise; + isNotFound(error: unknown): boolean; + sleep(): Promise; + now(): number; +} + +/** Tool-call completion precedes upload reconciliation. Only a worker result is final. */ +export async function pollBlockingExecution( + id: string, + timeout: number, + deps: BlockingPollDependencies, +): Promise<{ + status: 'waiting' | 'completed' | 'error'; + pending_calls?: t.ProgrammaticToolCall[]; + stdout?: string; + stderr?: string; + files?: t.FileRefs; + artifact_delivery?: t.ArtifactDeliveryFailure; +}> { + const start = deps.now(); + while (deps.now() - start < timeout) { + const execution = await deps.getExecutionState(id); + if (execution?.jobCompleted === true) { + // Preserve the inline result fallback for in-flight jobs from older binaries. + const result = (await deps.getBlockingResult(id)) ?? execution.jobResult; + if (result) { + return { + status: 'completed', + stdout: result.stdout, + stderr: result.stderr, + files: result.files, + artifact_delivery: result.artifact_delivery, + }; + } + } + if (execution?.jobError != null) return { status: 'error' }; + + try { + const pending = await deps.getPending(id); + if (pending.status === 'waiting' && pending.pending_calls != null && pending.pending_calls.length > 0) { + return { + status: 'waiting', + pending_calls: pending.pending_calls.map(call => ({ + id: call.call_id, + name: call.tool_name, + input: call.tool_input, + })), + }; + } + if (pending.status === 'error') return { status: 'error' }; + // Both completed and missing callback sessions must await the worker result. + } catch (error) { + if (!deps.isNotFound(error)) throw error; + } + await deps.sleep(); + } + return { status: 'error' }; +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 4536270b..9896518c 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -40,6 +40,7 @@ import { } from '../sandbox-egress'; import { findUnregisteredToolCall } from '../tool-scope'; import { summarizeRequestedFiles } from '../execution-log'; +import { pollBlockingExecution, type BlockingPendingState } from './blocking-poll'; import { clearSessionOwnership, recordSessionOwnership } from '../session-ownership'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; import { @@ -225,118 +226,24 @@ function decodeContinuationToken(token: string): { execution_id: string } | null // Blocking mode (legacy path) // --------------------------------------------------------------------------- -async function waitForExecutionState( - execution_id: string, - timeout: number, -): Promise<{ - status: 'waiting' | 'completed' | 'error' | 'running'; - pending_calls?: t.ProgrammaticToolCall[]; - stdout?: string; - stderr?: string; - files?: t.FileRefs; -}> { - const startTime = Date.now(); - - while (Date.now() - startTime < timeout) { - const execution = await getExecutionState(execution_id); - - /** Result lives in the `exec_result:` key (see setBlockingResult). The - * inline `execution.jobResult` branch is kept as a fallback so any - * in-flight executions whose state was written by an older binary - * mid-deploy still complete correctly without rolling back. */ - if (execution?.jobCompleted === true) { - const result = (await getBlockingResult(execution_id)) ?? execution.jobResult; - if (result) { - return { - status: 'completed', - stdout: result.stdout, - stderr: result.stderr, - files: result.files, - }; - } - } - - if (execution?.jobError != null) { - return { status: 'error' }; - } - - try { - const pendingResponse = await retryToolCallServerRequest( - () => axios.get<{ - status: string; - pending_calls?: Array<{ - call_id: string; - tool_name: string; - tool_input: Record; - timestamp: number; - }>; - }>(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/pending`, { - headers: internalServiceHeaders(), - }), +function waitForExecutionState(execution_id: string, timeout: number): ReturnType { + return pollBlockingExecution(execution_id, timeout, { + getExecutionState, + getBlockingResult, + getPending: async (id) => { + const response = await retryToolCallServerRequest( + () => axios.get( + `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, + { headers: internalServiceHeaders() }, + ), 'Get pending tool calls', ); - - const { status, pending_calls } = pendingResponse.data; - - if (status === 'waiting' && pending_calls && pending_calls.length > 0) { - return { - status: 'waiting', - pending_calls: pending_calls.map(call => ({ - id: call.call_id, - name: call.tool_name, - input: call.tool_input, - })), - }; - } - - if (status === 'completed') { - const statusResponse = await retryToolCallServerRequest( - () => axios.get<{ - status: string; - stdout?: string; - stderr?: string; - files?: t.FileRefs; - }>(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/status`, { - headers: internalServiceHeaders(), - }), - 'Get execution status', - ); - - return { - status: 'completed', - stdout: statusResponse.data.stdout, - stderr: statusResponse.data.stderr, - files: statusResponse.data.files, - }; - } - - if (status === 'error') { - return { status: 'error' }; - } - - await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)); - } catch (error) { - if (axios.isAxiosError(error) && error.response?.status === 404) { - const exec = await getExecutionState(execution_id); - if (exec?.jobCompleted === true) { - const result = (await getBlockingResult(execution_id)) ?? exec.jobResult; - if (result) { - return { - status: 'completed', - stdout: result.stdout, - stderr: result.stderr, - files: result.files, - }; - } - } - await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)); - continue; - } - throw error; - } - } - - return { status: 'error' }; + return response.data; + }, + isNotFound: (error) => axios.isAxiosError(error) && error.response?.status === 404, + sleep: () => new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)), + now: Date.now, + }); } // --------------------------------------------------------------------------- @@ -403,10 +310,10 @@ async function runReplayIteration( (state.bridgeWorkerId != null ? 'remote-bridge' : resolveQueuedSandboxBackend( - env.EXECUTION_PROFILE, - env.SANDBOX_BACKEND, - env.EXECUTION_PROFILE_SOURCE, - )); + env.EXECUTION_PROFILE, + env.SANDBOX_BACKEND, + env.EXECUTION_PROFILE_SOURCE, + )); const { queue, events, language } = getExecutionQueueBinding( state.language ?? 'python', replayBackend, @@ -1025,6 +932,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + artifact_delivery: result.artifact_delivery, session_id: state.session_id, }); } @@ -1243,6 +1151,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + artifact_delivery: state.artifact_delivery, session_id: execution.session_id, }); } @@ -1495,6 +1404,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + artifact_delivery: state.artifact_delivery, session_id, }); } diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 555056d5..0f97a532 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -103,6 +103,14 @@ export type RequestFile = { export type FileRefs = FileRef[]; +export interface ArtifactDeliveryFailure { + code: 'artifact_delivery_failed'; + status: 'partial' | 'failed'; + attempted: number; + delivered: number; + failed: number; +} + export type ExecuteResponse = { run?: { stdout: string; @@ -121,6 +129,7 @@ export type ExecuteResponse = { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRefs; + artifact_delivery?: ArtifactDeliveryFailure; }; export interface RequestBody { @@ -221,6 +230,7 @@ export type ExecuteResult = { stdout: string; stderr: string; files: FileRefs; + artifact_delivery?: ArtifactDeliveryFailure; code?: number | null; signal?: string | null; message?: string | null; @@ -358,6 +368,7 @@ export interface ProgrammaticResponse { stdout?: string; stderr?: string; files?: FileRefs; + artifact_delivery?: ArtifactDeliveryFailure; /** Top-level execution session id (one sandbox PTC invocation). */ session_id?: string; tool_calls_made?: number; diff --git a/service/src/workers.ts b/service/src/workers.ts index 215a4d1b..ad20fd0f 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -179,6 +179,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { * `[]` so the strictened response type from Phase B doesn't * surface a regression that wasn't there before. */ files: files ?? [], + ...(responseData.artifact_delivery != null + ? { artifact_delivery: responseData.artifact_delivery } + : {}), stdout, stderr, }; From f77e9603c8742ac87b49bccf1c48172be4722dab Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 13:38:40 -0400 Subject: [PATCH 045/116] fix: validate sandbox workspace requests before dispatch (#124) --- packages/code/src/workspace.test.ts | 110 ++++++++++++++++++++++++++++ packages/code/src/workspace.ts | 6 ++ 2 files changed, 116 insertions(+) diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 25894773..3dd7d2f4 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -29,6 +29,13 @@ import { WorkspaceToolError, } from './workspace.js'; +import { + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES, +} from './protocol.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + const execFileAsync = promisify(execFile); test('reads a bounded range from a registered local workspace', async (t) => { @@ -2018,3 +2025,106 @@ test('rejects empty, duplicate, and unknown sandbox workspace registration', asy ); } }); + + +test('validates sandbox workspace requests before either executor is invoked', async () => { + const commands: WorkspaceToolRequest[] = []; + const delegated: WorkspaceToolRequest[] = []; + const tools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + async execute(request) { + delegated.push(request); + throw new Error('Unexpected delegation'); + }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + async execute(request) { + commands.push(request); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'pwd', + }; + const malformed: unknown[] = [ + null, undefined, [], 'execute_command', {}, + { ...command, protocolVersion: 2 }, + { ...command, workspaceId: '' }, + { ...command, operation: 'unknown' }, + { ...command, command: undefined }, + { ...command, command: 123 }, + { ...command, command: ' ' }, + { ...command, command: 'echo\0secret' }, + { ...command, command: '\ud800' }, + { ...command, command: 'a'.repeat(BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + 1) }, + { ...command, command: 'é'.repeat(BRIDGE_WORKSPACE_COMMAND_MAX_BYTES / 2 + 1) }, + { ...command, cwd: '../outside' }, + { ...command, cwd: '/tmp' }, + { ...command, env: { UNSAFE: 'value' } }, + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: '../outside' }, + ]; + for (const [field, maximum] of [ + ['timeoutMs', BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS], + ['maxOutputBytes', BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES], + ] as const) { + for (const value of [0, -1, 1.5, NaN, Infinity, '1', null, maximum + 1]) { + malformed.push({ ...command, [field]: value }); + } + } + for (const request of malformed) { + await assert.rejects( + tools.execute(request as WorkspaceToolRequest), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'INVALID_REQUEST' && + error.mutationMayHaveCommitted === false, + ); + assert.deepEqual(commands, []); + assert.deepEqual(delegated, []); + } + for (const request of [ + command, + { ...command, timeoutMs: 1, maxOutputBytes: 1 }, + { + ...command, + command: 'é'.repeat(BRIDGE_WORKSPACE_COMMAND_MAX_BYTES / 2), + cwd: 'src', + timeoutMs: BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + maxOutputBytes: BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES, + }, + ]) { + await tools.execute(request); + assert.equal(commands.at(-1), request); + } + assert.equal(commands.length, 3); + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + tools.execute(command, controller.signal), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + error.mutationMayHaveCommitted === false, + ); + assert.equal(commands.length, 3); + assert.deepEqual(delegated, []); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 0fa3c8ba..5302593a 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1663,6 +1663,12 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { request: WorkspaceToolRequest, signal?: AbortSignal, ): Promise { + if (!isWorkspaceToolRequest(request)) { + throw new WorkspaceToolError( + 'Invalid workspace tool request', + 'INVALID_REQUEST', + ); + } if (request.operation !== 'execute_command') { return this.options.workspaceTools.execute(request, signal); } From 489bc9a74c251979c2bbe288b389ce9adf7a48d2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 13:38:56 -0400 Subject: [PATCH 046/116] fix: arbitrate bridge settlement and dispatch closure atomically (#125) --- service/src/bridge/settlement-race.test.ts | 272 +++++++++++++++++++++ service/src/bridge/store.ts | 58 +++-- 2 files changed, 304 insertions(+), 26 deletions(-) create mode 100644 service/src/bridge/settlement-race.test.ts diff --git a/service/src/bridge/settlement-race.test.ts b/service/src/bridge/settlement-race.test.ts new file mode 100644 index 00000000..4de5d3f3 --- /dev/null +++ b/service/src/bridge/settlement-race.test.ts @@ -0,0 +1,272 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { randomUUID } from 'node:crypto'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import type * as t from '../types'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; + +function barrier(): { promise: Promise; release: () => void } { + let release!: () => void; + const promise = new Promise(resolve => { + release = resolve; + }); + return { promise, release }; +} + +// Optional real Redis run: use a test server. Keys are isolated per test and +// cleaned up by prefix, with separate dispatcher and worker connections. +const redisUrl = process.env.BRIDGE_TEST_REDIS_URL; +for (const backend of ['mock', 'redis'] as const) { + const suite = + backend === 'redis' && (redisUrl == null || redisUrl === '') + ? describe.skip + : describe; + suite(`settlement/close arbitration (${backend})`, () => { + let redis: Redis; + let workerRedis: Redis; + let admin: Redis | undefined; + let prefix: string; + let store: RedisBridgeStore; + let worker: RedisBridgeStore; + let originalEval: Redis['eval']; + let originalGet: Redis['get']; + const incarnationId = 'incarnation-settlement-race'; + const workerId = 'settlement-race'; + const markerPattern = + 'codeapi:bridge:v1:worker:settlement-race:workspace:*:quarantined'; + + beforeEach(() => { + prefix = `race-test:${randomUUID()}:`; + if (backend === 'redis') { + admin = new Redis(redisUrl!); + redis = new Redis(redisUrl!, { keyPrefix: prefix }); + workerRedis = new Redis(redisUrl!, { keyPrefix: prefix }); + } else { + redis = new RedisMock() as unknown as Redis; + workerRedis = redis; + } + store = new RedisBridgeStore(redis, 60, 100); + worker = new RedisBridgeStore(workerRedis, 60, 100); + originalEval = redis.eval.bind(redis) as Redis['eval']; + originalGet = redis.get.bind(redis) as Redis['get']; + }); + afterEach(async () => { + if (admin) { + const keys = await admin.keys(`${prefix}*`); + if (keys.length) await admin.del(...keys); + admin.disconnect(); + admin = undefined; + } else { + await redis.flushall(); + } + redis.disconnect(); + workerRedis.disconnect(); + }); + async function start(stateful = true): Promise<{ + controller: AbortController; + completion: Promise; + assignment: CodeBridgeAssignment; + finalizations: string[]; + }> { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: stateful, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const finalizations: string[] = []; + const completion = store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + ...(stateful ? { runtimeSessionId: 'race-workspace' } : {}), + deadlineAtMs: Date.now() + 10_000, + signal: controller.signal, + finalize: async settlement => { + finalizations.push(settlement.status); + return settlement; + }, + }); + void completion.catch(() => undefined); + const assignment = (await worker.lease(workerId, incarnationId, 1_000))!; + expect(assignment).toBeDefined(); + await worker.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + return { controller, completion, assignment, finalizations }; + } + function result(assignment: CodeBridgeAssignment): CodeBridgeSettlement { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled' as const, + result: { + language: 'bash', + version: '5.2', + session_id: 'race-result', + files: [], + }, + }; + } + async function markers(): Promise { + return admin + ? admin.keys(`${prefix}${markerPattern}`) + : redis.keys(markerPattern); + } + for (const stateful of [true, false]) { + test(`close rejects fulfillment already past preflight (stateful=${stateful})`, async () => { + const { controller, completion, assignment, finalizations } = + await start(stateful); + const entered = barrier(); + const resume = barrier(); + const workerEval = workerRedis.eval.bind(workerRedis); + workerRedis.eval = (async (...args: Parameters) => { + if ( + String(args[0]).includes( + 'local existing = redis.call(\'GET\', KEYS[2])', + ) + ) { + entered.release(); + await resume.promise; + } + return workerEval(...args); + }) as Redis['eval']; + const settling = worker.settle( + workerId, + assignment.assignmentId, + result(assignment), + ); + void settling.catch(() => undefined); + await entered.promise; + controller.abort(); + try { + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + } finally { + resume.release(); + } + await expect(settling).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + expect( + await redis.get( + `codeapi:bridge:v1:assignment:${assignment.assignmentId}:settlement`, + ), + ).toBeNull(); + expect( + await redis.exists( + `codeapi:bridge:v1:assignment:${assignment.assignmentId}:deadline`, + ), + ).toBe(0); + expect(finalizations).toEqual([]); + expect(await markers()).toHaveLength(stateful ? 1 : 0); + if (stateful) { + await worker.settle(workerId, assignment.assignmentId, { + ...result(assignment), + status: 'rejected', + error: 'not executed', + }); + expect(await markers()).toHaveLength(0); + } + }); + } + test('settlement wins before close and commits despite caller abort', async () => { + const { controller, completion, assignment, finalizations } = + await start(); + const entered = barrier(); + const resume = barrier(); + redis.eval = (async (...args: Parameters) => { + if ( + String(args[0]).includes( + 'local settlement = redis.call(\'GET\', KEYS[2])', + ) + ) { + entered.release(); + await resume.promise; + } + return originalEval(...args); + }) as Redis['eval']; + controller.abort(); + await entered.promise; + try { + await worker.settle( + workerId, + assignment.assignmentId, + result(assignment), + ); + } finally { + resume.release(); + } + await expect(completion).resolves.toEqual(result(assignment)); + expect(finalizations).toEqual(['fulfilled']); + expect(await markers()).toHaveLength(0); + await expect( + worker.settle(workerId, assignment.assignmentId, result(assignment)), + ).resolves.toBeUndefined(); + }); + for (const failure of ['abort', 'timeout', 'error'] as const) { + test(`a poll ${failure} still closes fulfillment`, async () => { + const entered = barrier(); + let intercept = false; + redis.get = ((key: string) => { + if (intercept && key.endsWith(':settlement')) { + entered.release(); + return failure === 'error' + ? Promise.reject(new Error('poll unavailable')) + : new Promise(() => {}); + } + return originalGet(key); + }) as Redis['get']; + const { controller, completion, assignment } = await start(); + intercept = true; + await entered.promise; + if (failure === 'abort') controller.abort(); + const messages = { + abort: 'deadline', + error: 'poll unavailable', + timeout: 'poll timed out', + }; + await expect(completion).rejects.toThrow(messages[failure]); + redis.get = originalGet; + await expect( + worker.settle(workerId, assignment.assignmentId, result(assignment)), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + expect(await markers()).toHaveLength(1); + }); + } + test('an unconfirmed close preserves the workspace fence', async () => { + const { controller, completion } = await start(); + redis.eval = ((...args: Parameters) => { + if ( + String(args[0]).includes( + 'local settlement = redis.call(\'GET\', KEYS[2])', + ) + ) + return new Promise(() => {}); + return originalEval(...args); + }) as Redis['eval']; + controller.abort(); + await expect(completion).rejects.toThrow( + 'Bridge settlement close timed out', + ); + expect(await markers()).toHaveLength(1); + expect( + await redis.get('codeapi:bridge:v1:worker:settlement-race:lock'), + ).not.toBeNull(); + }); + }); +} diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index da3956d7..de302654 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -796,12 +796,7 @@ export class RedisBridgeStore { args.finalize == null ? settlement : await args.finalize(settlement, registration); - await this.commitPendingWorkspace( - assignment, - settlement, - args.deadlineAtMs, - args.signal, - ); + await this.commitPendingWorkspace(assignment, settlement); resultCommitted = true; return result; } catch (error) { @@ -1450,22 +1445,30 @@ export class RedisBridgeStore { deadlineAtMs: number, signal: AbortSignal, ): Promise { - while (!signal.aborted && Date.now() < deadlineAtMs) { - const raw = await boundedCommand( - this.redis.get(settlementKey(assignment.assignmentId)), - Math.max( - 1, - Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), - ), - 'Bridge settlement poll', - signal, - ); - if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement; - await delay(POLL_INTERVAL_MS, signal); + let pollError: unknown; + try { + while (!signal.aborted && Date.now() < deadlineAtMs) { + const raw = await boundedCommand( + this.redis.get(settlementKey(assignment.assignmentId)), + Math.max( + 1, + Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), + ), + 'Bridge settlement poll', + signal, + ); + if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement; + await delay(POLL_INTERVAL_MS, signal); + } + } catch (error) { + // A failed/aborted poll does not cancel Redis work. Arbitrate with + // settlement before returning an error, even when the caller is gone. + pollError = error; } const closeKeys = [ assignmentKey(assignment.assignmentId), settlementKey(assignment.assignmentId), + assignmentDeadlineKey(assignment.assignmentId), ]; if (assignment.runtimeSessionId !== undefined) { closeKeys.push( @@ -1475,10 +1478,14 @@ export class RedisBridgeStore { ), ); } + // The deadline key is also the fulfillment gate checked by settle(). + // Keep acknowledged assignment metadata for late clean rejection recovery, + // but atomically revoke fulfillment when no settlement has won yet. const closeScript = [ 'local settlement = redis.call(\'GET\', KEYS[2])', 'if settlement then return settlement end', - 'if #KEYS == 3 and redis.call(\'GET\', KEYS[3]) == ARGV[1] then return nil end', + 'redis.call(\'DEL\', KEYS[3])', + 'if #KEYS == 4 and redis.call(\'GET\', KEYS[4]) == ARGV[1] then return nil end', 'redis.call(\'DEL\', KEYS[1])', 'return nil', ].join('\n'); @@ -1495,6 +1502,9 @@ export class RedisBridgeStore { if (finalSettlement != null) { return JSON.parse(String(finalSettlement)) as CodeBridgeSettlement; } + if (pollError != null && !signal.aborted && Date.now() < deadlineAtMs) { + throw pollError; + } throw new BridgeStoreError( 'ASSIGNMENT_EXPIRED', 'Bridge assignment exceeded its deadline', @@ -1619,8 +1629,6 @@ export class RedisBridgeStore { private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, - deadlineAtMs: number, - signal: AbortSignal, ): Promise { if ( assignment.runtimeSessionId === undefined || @@ -1646,12 +1654,10 @@ export class RedisBridgeStore { ), assignment.assignmentId, ), - Math.max( - 1, - Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), - ), + // Once settlement wins, caller cancellation must not prevent its + // workspace commit. Redis availability still has a bounded budget. + this.redisCommandTimeoutMs, 'Bridge workspace commit', - signal, ), ); if (committed !== 1) { From 49d1b673b7062ec33cec14993eb46a54691439c7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 13:46:48 -0400 Subject: [PATCH 047/116] fix: preserve terminal bridge failures and recovery guidance (#126) --- .../src/sandbox-backend/remote-bridge.test.ts | 61 ++++++++++++++- service/src/sandbox-backend/remote-bridge.ts | 43 +++++----- service/src/sandbox-backend/types.ts | 10 ++- service/src/utils.ts | 78 ++++++++++++++++--- 4 files changed, 157 insertions(+), 35 deletions(-) diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index c1b9e2c9..b0121d2e 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from 'bun:test'; -import type { SandboxExecuteContext, SandboxTransportRequest } from './types'; +import type { SandboxBackendErrorCode, SandboxExecuteContext, SandboxTransportRequest } from './types'; +import { SandboxBackendError } from './types'; +import { publicExecutionFailure } from '../utils'; import type { RedisBridgeStore } from '../bridge/store'; import { BridgeStoreError } from '../bridge/store'; @@ -73,6 +75,63 @@ describe('RemoteBridgeSandboxBackend', () => { }); }); + const failures = { + WORKER_OFFLINE: ['BRIDGE_WORKER_OFFLINE', true, 503, 'Code environment is offline'], + WORKER_UNAUTHORIZED: ['BRIDGE_WORKER_UNAUTHORIZED', false, 403, 'Code environment is not authorized for this tenant'], + WORKER_BUSY: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], + ASSIGNMENT_EXPIRED: ['BRIDGE_DEADLINE_EXCEEDED', false, 504, 'Code environment execution timed out'], + ASSIGNMENT_FENCED: ['BRIDGE_ASSIGNMENT_FENCED', false, 409, 'Code environment assignment is fenced; inspect the execution before retrying'], + ASSIGNMENT_NOT_FOUND: ['BRIDGE_ASSIGNMENT_NOT_FOUND', false, 409, 'Code environment assignment is no longer available; inspect the execution before retrying'], + WORKER_FENCED: ['BRIDGE_WORKER_FENCED', false, 409, 'Code environment worker changed during execution; inspect the execution before retrying'], + WORKER_QUARANTINED: ['BRIDGE_WORKER_QUARANTINED', false, 409, 'Code environment is quarantined; recover the worker before retrying'], + WORKSPACE_QUARANTINED: ['BRIDGE_WORKSPACE_QUARANTINED', false, 409, 'Code environment workspace is quarantined; reset the workspace before retrying'], + WORKER_MISMATCH: ['BRIDGE_WORKER_MISMATCH', false, 409, 'Code environment does not support this execution; select a compatible worker'], + ASSIGNMENT_INVALID: ['BRIDGE_ASSIGNMENT_INVALID', false, 400, 'Code environment assignment is invalid'], + RESULT_INVALID: ['BRIDGE_RESULT_INVALID', false, 502, 'Code environment returned an invalid result'], + } satisfies Record; + + for (const [storeCode, [code, transient, status, message]] of Object.entries(failures)) { + test(`preserves ${storeCode} recovery through the backend and public response`, async () => { + const cause = new BridgeStoreError(storeCode as BridgeStoreError['code'], + 'worker vm-private at redis.internal\nprivate tenant-secret'); + const store = { + dispatch: async (): ReturnType => { throw cause; }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + const error: unknown = await backend.execute(request(), context()).catch((failure: unknown) => failure); + expect(error).toBeInstanceOf(SandboxBackendError); + if (!(error instanceof SandboxBackendError)) throw new Error('Expected backend failure'); + expect(error).toMatchObject({ code, transient, message: cause.message }); + expect(error.cause).toMatchObject({ message: cause.message }); + // The worker carries only the code/message through BullMQ, not transient. + const failure = publicExecutionFailure(new Error(`${error.code}: ${error.message}`)); + expect(failure).toEqual({ status, body: { error: code.toLowerCase(), message } }); + expect(JSON.stringify(failure)).not.toContain('vm-private'); + expect(JSON.stringify(failure)).not.toContain('redis.internal'); + expect(JSON.stringify(failure)).not.toContain('tenant-secret'); + }); + } + + test('preserves failures that are not bridge store errors', async () => { + const cause = new Error('result finalization failed'); + const backend = new RemoteBridgeSandboxBackend({ + dispatch: async (): ReturnType => { throw cause; }, + }, 'default-vm'); + await expect(backend.execute(request(), context())).rejects.toBe(cause); + }); + + test('keeps a rejected settlement non-transient', async () => { + const backend = new RemoteBridgeSandboxBackend({ + dispatch: async (): ReturnType => ({ + protocolVersion: 1, generation: 1, leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', status: 'rejected', error: 'sandbox rejected execution', + }), + }, 'default-vm'); + await expect(backend.execute(request(), context())).rejects.toMatchObject({ + code: 'BRIDGE_EXECUTION_FAILED', transient: false, + }); + }); + test('keeps an explicitly selected singleton on its unbound compatibility route', async () => { let dispatched: Parameters[0] | undefined; const store = { diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index b1a94ae0..719e7004 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -1,5 +1,6 @@ import type { SandboxBackend, + SandboxBackendErrorCode, SandboxExecuteContext, SandboxRawResponse, SandboxTransportRequest, @@ -11,6 +12,23 @@ import { bridgeStore } from '../bridge'; import { BridgeStoreError } from '../bridge/store'; import { SandboxBackendError } from './types'; +// Every store failure needs an explicit recovery classification. New store +// codes must not silently fall through to a retryable worker outage. +const bridgeErrorCodes = { + WORKER_OFFLINE: 'BRIDGE_WORKER_OFFLINE', + WORKER_UNAUTHORIZED: 'BRIDGE_WORKER_UNAUTHORIZED', + WORKER_BUSY: 'BRIDGE_WORKER_BUSY', + ASSIGNMENT_EXPIRED: 'BRIDGE_DEADLINE_EXCEEDED', + ASSIGNMENT_FENCED: 'BRIDGE_ASSIGNMENT_FENCED', + ASSIGNMENT_NOT_FOUND: 'BRIDGE_ASSIGNMENT_NOT_FOUND', + WORKER_FENCED: 'BRIDGE_WORKER_FENCED', + WORKER_QUARANTINED: 'BRIDGE_WORKER_QUARANTINED', + WORKSPACE_QUARANTINED: 'BRIDGE_WORKSPACE_QUARANTINED', + WORKER_MISMATCH: 'BRIDGE_WORKER_MISMATCH', + ASSIGNMENT_INVALID: 'BRIDGE_ASSIGNMENT_INVALID', + RESULT_INVALID: 'BRIDGE_RESULT_INVALID', +} satisfies Record; + export class RemoteBridgeSandboxBackend implements SandboxBackend { readonly name = 'remote-bridge' as const; @@ -63,32 +81,11 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { return settlement.result as SandboxRawResponse; } catch (error) { if (!(error instanceof BridgeStoreError)) throw error; - if (error.code === 'WORKER_UNAUTHORIZED') { - throw new SandboxBackendError( - 'BRIDGE_WORKER_UNAUTHORIZED', - error.message, - error, - ); - } - if (error.code === 'WORKER_BUSY') { - throw new SandboxBackendError( - 'BRIDGE_WORKER_BUSY', - error.message, - error, - ); - } - if (error.code === 'ASSIGNMENT_EXPIRED') { - throw new SandboxBackendError( - 'BRIDGE_DEADLINE_EXCEEDED', - error.message, - error, - ); - } throw new SandboxBackendError( - 'BRIDGE_WORKER_OFFLINE', + bridgeErrorCodes[error.code], error.message, error, - true, + error.code === 'WORKER_OFFLINE', ); } } diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index 96151dde..fbaa2d20 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -78,13 +78,21 @@ export type SandboxBackendErrorCode = | 'BRIDGE_WORKER_BUSY' | 'BRIDGE_EXECUTION_FAILED' | 'BRIDGE_DEADLINE_EXCEEDED' + | 'BRIDGE_ASSIGNMENT_FENCED' + | 'BRIDGE_ASSIGNMENT_NOT_FOUND' + | 'BRIDGE_WORKER_FENCED' + | 'BRIDGE_WORKER_QUARANTINED' + | 'BRIDGE_WORKSPACE_QUARANTINED' + | 'BRIDGE_WORKER_MISMATCH' + | 'BRIDGE_ASSIGNMENT_INVALID' + | 'BRIDGE_RESULT_INVALID' | 'MICROVM_LAUNCH_FAILED' | 'MICROVM_LAUNCH_THROTTLED' | 'MICROVM_UNHEALTHY' | 'MICROVM_FENCED' | 'MICROVM_DEADLINE_EXCEEDED'; -/** Lambda-only failure modes; the worker prefixes messages with the code so +/** Typed sandbox backend failure modes; the worker prefixes messages with the code so * the router can map them (e.g. RUNTIME_SESSION_BUSY -> 409). Axios errors * from the sandbox POST itself are rethrown raw by every backend. */ export class SandboxBackendError extends Error { diff --git a/service/src/utils.ts b/service/src/utils.ts index aae2d05e..6fec60b7 100644 --- a/service/src/utils.ts +++ b/service/src/utils.ts @@ -1,5 +1,66 @@ import axios from 'axios'; import type { AxiosError } from 'axios'; +import type { SandboxBackendErrorCode } from './sandbox-backend/types'; + +// Keep the public response exhaustive too: a new terminal backend code must +// not silently become an availability-related 503 after crossing BullMQ. +const bridgePublicFailures: Partial> = { + BRIDGE_WORKER_UNAUTHORIZED: { + status: 403, + message: 'Code environment is not authorized for this tenant', + }, + BRIDGE_WORKER_OFFLINE: { + status: 503, + message: 'Code environment is offline', + }, + BRIDGE_WORKER_BUSY: { + status: 409, + message: 'Code environment is busy', + }, + BRIDGE_EXECUTION_FAILED: { + status: 502, + message: 'Code environment execution failed', + }, + BRIDGE_DEADLINE_EXCEEDED: { + status: 504, + message: 'Code environment execution timed out', + }, + BRIDGE_ASSIGNMENT_FENCED: { + status: 409, + message: 'Code environment assignment is fenced; inspect the execution before retrying', + }, + BRIDGE_ASSIGNMENT_NOT_FOUND: { + status: 409, + message: 'Code environment assignment is no longer available; inspect the execution before retrying', + }, + BRIDGE_WORKER_FENCED: { + status: 409, + message: 'Code environment worker changed during execution; inspect the execution before retrying', + }, + BRIDGE_WORKER_QUARANTINED: { + status: 409, + message: 'Code environment is quarantined; recover the worker before retrying', + }, + BRIDGE_WORKSPACE_QUARANTINED: { + status: 409, + message: 'Code environment workspace is quarantined; reset the workspace before retrying', + }, + BRIDGE_WORKER_MISMATCH: { + status: 409, + message: 'Code environment does not support this execution; select a compatible worker', + }, + BRIDGE_ASSIGNMENT_INVALID: { + status: 400, + message: 'Code environment assignment is invalid', + }, + BRIDGE_RESULT_INVALID: { + status: 502, + message: 'Code environment returned an invalid result', + }, +} satisfies Record< + Extract, + { status: number; message: string } +>; export function applySystemReplacements(input: string): string { return input; @@ -135,13 +196,15 @@ export function publicExecutionFailure(error: unknown): { status: number; body: ); if (backendMatch) { const code = backendMatch[1]; + const bridgeFailure = bridgePublicFailures[code]; + if (bridgeFailure != null) { + return { + status: bridgeFailure.status, + body: { error: code.toLowerCase(), message: bridgeFailure.message }, + }; + } const statuses: Record = { RUNTIME_SESSION_BUSY: 409, - BRIDGE_WORKER_UNAUTHORIZED: 403, - BRIDGE_WORKER_OFFLINE: 503, - BRIDGE_WORKER_BUSY: 409, - BRIDGE_EXECUTION_FAILED: 502, - BRIDGE_DEADLINE_EXCEEDED: 504, SESSION_INPUT_TOO_LARGE: 413, SESSION_INPUT_UNAVAILABLE: 422, SESSION_INPUT_SOURCE_FAILED: 502, @@ -152,11 +215,6 @@ export function publicExecutionFailure(error: unknown): { status: number; body: const status = statuses[code] ?? (sessionInputFailure ? 500 : 503); const publicMessages: Record = { RUNTIME_SESSION_BUSY: 'Runtime session is busy', - BRIDGE_WORKER_UNAUTHORIZED: 'Code environment is not authorized for this tenant', - BRIDGE_WORKER_OFFLINE: 'Code environment is offline', - BRIDGE_WORKER_BUSY: 'Code environment is busy', - BRIDGE_EXECUTION_FAILED: 'Code environment execution failed', - BRIDGE_DEADLINE_EXCEEDED: 'Code environment execution timed out', MICROVM_LAUNCH_FAILED: 'Sandbox launch failed', MICROVM_LAUNCH_THROTTLED: 'Sandbox capacity is temporarily unavailable', MICROVM_UNHEALTHY: 'Sandbox runtime is unavailable', From 4650240c1ea629fe237ed50f62a076e88155cf61 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 16:03:53 -0400 Subject: [PATCH 048/116] fix: preserve required native sandbox environment names (#129) --- packages/code/README.md | 6 ++ packages/code/src/native-sandbox.test.ts | 95 ++++++++++++++++++++++++ packages/code/src/native-sandbox.ts | 39 +++++++++- 3 files changed, 137 insertions(+), 3 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 54284c56..b0bcad80 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -86,6 +86,12 @@ policy: an allowed destination can receive workspace data. The normalized allowlist is included in the worker policy digest. Tool approval hooks remain the user-facing allow/deny boundary for each invocation. +The native sandbox preserves standard `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, +and `NO_PROXY` names (including lowercase forms), plus Windows process and profile +variables on Windows. SRT remains responsible for the final sandbox environment +and can replace proxy values with its filtered proxy endpoints. This does not +expand the allowed domains or expose unrelated inherited credentials. + ### GitHub authentication The native BYOM worker can provide Git HTTPS authentication without exposing a diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 50f8e5c3..d344cabf 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -34,6 +34,7 @@ function fakeManager( beforeWrap?: () => Promise; appendGitSafeDirectory?: boolean; inheritedGitEnvironment?: Record; + wrappedEnvironment?: NodeJS.ProcessEnv; } = {}, ) { let config: SandboxRuntimeConfig | undefined; @@ -72,6 +73,7 @@ function fakeManager( env: { PATH: process.env.PATH, ...gitEnvironment, + ...options.wrappedEnvironment, ...(credentialSeenDuringWrap ? { LIBRECHAT_CODE_TEST_CREDENTIAL: @@ -148,6 +150,99 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.equal(fake.reset, true); }); +const proxyEnvironment = { + HTTP_PROXY: 'http://upstream.invalid:8080', + HTTPS_PROXY: 'http://upstream.invalid:8080', + ALL_PROXY: 'socks5://upstream.invalid:1080', + NO_PROXY: 'upstream.internal', + http_proxy: 'http://upstream.invalid:8080', + https_proxy: 'http://upstream.invalid:8080', + all_proxy: 'socks5://upstream.invalid:1080', + no_proxy: 'upstream.internal', +}; +const windowsEnvironment = { + SYSTEMROOT: 'C:\\Windows', + SystemRoot: 'C:\\Windows', + SYSTEMDRIVE: 'C:', + windir: 'C:\\Windows', + ComSpec: 'C:\\Windows\\System32\\cmd.exe', + PATHEXT: '.COM;.EXE;.BAT;.CMD', + TEMP: 'C:\\Temp', + Temp: 'C:\\Temp', + TMP: 'C:\\Temp', + USERPROFILE: 'C:\\Users\\sandbox', + HOMEDRIVE: 'C:', + HOMEPATH: '\\Users\\sandbox', + APPDATA: 'C:\\Users\\sandbox\\AppData\\Roaming', + LOCALAPPDATA: 'C:\\Users\\sandbox\\AppData\\Local', +}; + +for (const platform of ['darwin', 'linux', 'win32'] as const) { + test(`preserves required ${platform} environment names without allowing credentials`, async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const credentials = { + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_HTTP_PROXY: 'worker-secret', + AWS_SECRET_ACCESS_KEY: 'aws-secret', + GITHUB_TOKEN: 'github-secret', + HTTP_PROXY_TOKEN: 'proxy-secret', + CUSTOM_PROXY: 'proxy-secret', + SYSTEMROOT_TOKEN: 'runtime-secret', + NODE_OPTIONS: '--require /host/private.js', + LD_PRELOAD: '/host/private.so', + }; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, platform, allowedDomains: ['github.com'], + environment: { + ...proxyEnvironment, ...windowsEnvironment, ...credentials, + HtTp_PrOxY: 'http://mixed-case.invalid:8080', + PATH: '/usr/bin', LC_ALL: 'C.UTF-8', + }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const denied = new Set(fake.config?.credentials?.envVars + ?.filter(({ mode }) => mode === 'deny').map(({ name }) => name)); + for (const name of [...Object.keys(proxyEnvironment), 'PATH', 'LC_ALL']) { + assert.equal(denied.has(name), false, `${name} must remain available`); + } + for (const name of Object.keys(windowsEnvironment)) { + assert.equal(denied.has(name), platform !== 'win32', `${name} must be platform-specific`); + } + assert.equal(denied.has('HtTp_PrOxY'), platform !== 'win32'); + for (const name of Object.keys(credentials)) { + assert.equal(denied.has(name), true, `${name} must remain denied`); + } + assert.deepEqual(fake.config?.network.allowedDomains, ['github.com']); + assert.equal(fake.config?.network.strictAllowlist, true); + }); +} + +test('uses SRT proxy values without restoring inherited proxies or credentials', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const wrappedEnvironment = { + HTTP_PROXY: 'http://localhost:3128', HTTPS_PROXY: 'http://localhost:3128', + ALL_PROXY: 'http://localhost:3128', NO_PROXY: 'localhost', + }; + const fake = fakeManager({ wrappedEnvironment }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { ...proxyEnvironment, GITHUB_TOKEN: 'host-secret' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + const result = await sandbox.execute({ + ...request, maxOutputBytes: 256, + command: 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', + }); + assert.equal(result.exitCode, 0); + assert.equal(result.stdout, `${Object.values(wrappedEnvironment).join('|')}|unset`); +}); + test('masks a host credential for only its injection host and restores the parent environment', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 1b1d96cb..716072a6 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -48,6 +48,36 @@ const SAFE_CHILD_ENV_NAMES = new Set([ 'USER', ]); +// Preserve the conventional proxy names, not arbitrary *_PROXY variables. +// SRT owns their final values and may replace them with its filtered proxy. +const PROXY_CHILD_ENV_NAMES = new Set([ + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'ALL_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'all_proxy', + 'no_proxy', +]); + +// Windows resolves these names case-insensitively. Keep the exception +// platform-specific so similarly named POSIX variables remain denied. +const WINDOWS_CHILD_ENV_NAMES = new Set([ + 'SYSTEMROOT', + 'SYSTEMDRIVE', + 'WINDIR', + 'COMSPEC', + 'PATHEXT', + 'TEMP', + 'TMP', + 'USERPROFILE', + 'HOMEDRIVE', + 'HOMEPATH', + 'APPDATA', + 'LOCALAPPDATA', +]); + let hostEnvironmentMutationQueue: Promise = Promise.resolve(); const TRUSTED_GIT_ENVIRONMENT = { @@ -148,7 +178,7 @@ function boundedUtf8(buffer: Buffer, budget: number): string { return ''; } -function safeEnvironmentNames( +function deniedEnvironmentNames( environment: NodeJS.ProcessEnv, platform: NodeJS.Platform, ): string[] { @@ -157,7 +187,10 @@ function safeEnvironmentNames( const normalized = platform === 'win32' ? name.toUpperCase() : name; return ( normalized.startsWith('LIBRECHAT_CODE_') || - (!SAFE_CHILD_ENV_NAMES.has(normalized) && !normalized.startsWith('LC_')) + (!SAFE_CHILD_ENV_NAMES.has(normalized) && + !PROXY_CHILD_ENV_NAMES.has(normalized) && + !(platform === 'win32' && WINDOWS_CHILD_ENV_NAMES.has(normalized)) && + !normalized.startsWith('LC_')) ); }) .sort(); @@ -272,7 +305,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox mode: 'deny' as const, })), envVars: [ - ...safeEnvironmentNames(this.environment, this.platform) + ...deniedEnvironmentNames(this.environment, this.platform) .filter((name) => { const normalized = normalizedEnvironmentName( name, From 70e8d53253b529339815fc1ec065ed3291dace84 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 16:04:10 -0400 Subject: [PATCH 049/116] fix: clean sandbox state on pre-spawn exits (#130) --- packages/code/src/native-sandbox.test.ts | 97 ++++++++++++++++++++++++ packages/code/src/native-sandbox.ts | 26 ++++--- 2 files changed, 112 insertions(+), 11 deletions(-) diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index d344cabf..5ee411fb 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -647,3 +647,100 @@ test('maps platform-native exit statuses into the bridge protocol range', async const result = await sandbox.execute(request); assert.equal(result.exitCode, 1); }); + +test('cleans allocated command state exactly once on every execution exit', async (t) => { + for (const outcome of [ + 'abort-before-spawn', + 'spawn-throw', + 'close', + 'error', + 'abort-after-spawn', + 'timeout', + 'wrap-throw', + ] as const) { + for (const cleanupThrows of [false, true]) { + await t.test(`${outcome}, cleanup throws: ${cleanupThrows}`, async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const controller = new AbortController(); + let cleanupCalls = 0; + let spawnCalls = 0; + let allocated = false; + const fake = fakeManager({ + async beforeWrap() { + if (outcome === 'wrap-throw') throw new Error('wrap failed'); + allocated = true; + if (outcome === 'abort-before-spawn') controller.abort(); + }, + }); + fake.manager.cleanupAfterCommand = () => { + cleanupCalls += 1; + assert.equal(allocated, true); + allocated = false; + if (cleanupThrows) throw new Error('cleanup failed'); + }; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + spawnCommand() { + spawnCalls += 1; + assert.equal(allocated, true); + if (outcome === 'spawn-throw') throw new Error('spawn failed'); + const child = new EventEmitter() as ChildProcessWithoutNullStreams; + let closeQueued = false; + const close = () => { + if (!closeQueued) { + closeQueued = true; + queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + } + return true; + }; + Object.assign(child, { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + pid: undefined, + kill: close, + }); + queueMicrotask(() => { + assert.equal(cleanupCalls, 0); + if (outcome === 'error') { + child.emit('error', new Error('spawn failed')); + } else if (outcome === 'abort-after-spawn') { + controller.abort(); + } else if (outcome === 'close') { + child.emit('close', 0, null); + } + }); + return child; + }, + }); + const execution = sandbox.execute( + { ...request, timeoutMs: 10 }, + controller.signal, + ); + if (outcome === 'close' || outcome === 'timeout') { + const result = await execution; + assert.equal(result.exitCode, outcome === 'close' ? 0 : null); + assert.equal(result.timedOut, outcome === 'timeout'); + } else { + await assert.rejects(execution, (error: unknown) => + error instanceof WorkspaceToolError && + error.code === (outcome.startsWith('abort') + ? 'EXECUTION_ABORTED' + : 'COMMAND_UNAVAILABLE') && + error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn'), + ); + } + assert.equal( + spawnCalls, + outcome === 'abort-before-spawn' || outcome === 'wrap-throw' ? 0 : 1, + ); + assert.equal(cleanupCalls, outcome === 'wrap-throw' ? 0 : 1); + assert.equal(allocated, false); + await sandbox.close(); + assert.equal(fake.reset, true); + }); + } + } +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 716072a6..9743ad76 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -413,13 +413,22 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'COMMAND_UNAVAILABLE', ); } - if (signal?.aborted) { - throw new WorkspaceToolError( - 'Workspace command execution aborted', - 'EXECUTION_ABORTED', - ); + try { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + ); + } + return await this.runWrapped(request, wrapped, cwd, commandId, signal); + } finally { + // A successful wrap owns command state even when no child is spawned. + try { + this.manager.cleanupAfterCommand(); + } catch { + // Cleanup is retried by close(); command settlement must still finish. + } } - return await this.runWrapped(request, wrapped, cwd, commandId, signal); } private async withTemporaryHostEnvironment( @@ -521,11 +530,6 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const cleanup = (): void => { clearTimeout(timer); signal?.removeEventListener('abort', abort); - try { - this.manager.cleanupAfterCommand(); - } catch { - // Cleanup is retried by close(); command settlement must still finish. - } }; child.once('error', () => { if (settled) return; From 6365863b200c3be8b313a6f727fa3b1c50680a63 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 16:04:25 -0400 Subject: [PATCH 050/116] fix: continue bounded listings after skipped candidate windows (#132) --- packages/code/src/workspace-listing.test.ts | 133 +++++++ packages/code/src/workspace.ts | 374 ++++++++++---------- 2 files changed, 327 insertions(+), 180 deletions(-) create mode 100644 packages/code/src/workspace-listing.test.ts diff --git a/packages/code/src/workspace-listing.test.ts b/packages/code/src/workspace-listing.test.ts new file mode 100644 index 00000000..90845b02 --- /dev/null +++ b/packages/code/src/workspace-listing.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { PassThrough } from 'node:stream'; +import test from 'node:test'; + +import type { TestContext } from 'node:test'; +import type { ChildProcess } from 'node:child_process'; +import { BRIDGE_WORKSPACE_LIST_MAX_RESULTS } from './protocol.js'; +import { isWorkspaceToolResult, LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +const request = { + protocolVersion: 1 as const, + operation: 'list_files' as const, + workspaceId: 'primary', + maxResults: 1, +}; +const skippedPaths = Array.from( + { length: 2 * (BRIDGE_WORKSPACE_LIST_MAX_RESULTS + request.maxResults) + 5 }, + (_, index) => `a-${String(index).padStart(4, '0')}`, +); + +// Control candidate discovery independently of filesystem verification, as +// files can vanish or be replaced by symlinks after rg has enumerated them. +function candidateSource(t: TestContext, paths: string[], onScan?: (scan: number) => void) { + let scans = 0; + let cappedScans = 0; + t.mock.method(childProcess, 'spawn', (command: string, args: string[]) => { + assert.equal(command, 'rg'); + assert.ok(args.includes('--no-follow')); + assert.ok(args.includes('--null')); + assert.ok(args.includes('--sort')); + scans += 1; + const child = new EventEmitter() as ChildProcess; + const stdout = new PassThrough(); + let closed = false; + const close = () => { + if (closed) return; + closed = true; + queueMicrotask(() => child.emit('close', 0)); + }; + Object.assign(child, { + stdout, + kill: () => { cappedScans += 1; close(); return true; }, + }); + onScan?.(scans); + queueMicrotask(() => { + if (closed) return; + stdout.end(Buffer.from(`${paths.join('\0')}\0`)); + close(); + }); + return child; + }); + syncBuiltinESMExports(); + t.after(() => { + t.mock.restoreAll(); + syncBuiltinESMExports(); + }); + return { get scans() { return scans; }, get cappedScans() { return cappedScans; } }; +} + +async function workspace(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-list-windows-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root }] }); + return { root, tools }; +} + +for (const skippedKind of ['missing', 'symlink'] as const) { + test(`lists and paginates valid files after multiple windows of ${skippedKind} candidates`, async (t) => { + const { root, tools } = await workspace(t); + await writeFile(join(root, 'z-first.txt'), 'first'); + await writeFile(join(root, 'z-last.txt'), 'last'); + if (skippedKind === 'symlink') { + await Promise.all(skippedPaths.map(path => symlink('z-first.txt', join(root, path)))); + } + const source = candidateSource(t, [...skippedPaths, 'z-first.txt', 'z-last.txt']); + const first = await tools.execute(request); + assert.equal(isWorkspaceToolResult(request, first, tools.capabilities), true); + assert.deepEqual(first, { + protocolVersion: 1, operation: 'list_files', workspaceId: 'primary', + paths: ['z-first.txt'], truncated: true, nextAfterPath: 'z-first.txt', + }); + assert.equal(source.scans, 3); + assert.equal(source.cappedScans, 2, 'each full candidate window still stops rg'); + const nextRequest = { ...request, afterPath: 'z-first.txt' }; + const last = await tools.execute(nextRequest); + assert.equal(isWorkspaceToolResult(nextRequest, last, tools.capabilities), true); + assert.deepEqual(last, { + protocolVersion: 1, operation: 'list_files', workspaceId: 'primary', + paths: ['z-last.txt'], truncated: false, + }); + assert.equal(source.scans, 4); + }); +} + +test('returns a complete empty page when successive skipped windows exhaust the listing', async (t) => { + const { tools } = await workspace(t); + const source = candidateSource(t, skippedPaths); + const result = await tools.execute(request); + assert.equal(isWorkspaceToolResult(request, result, tools.capabilities), true); + assert.deepEqual(result, { + protocolVersion: 1, operation: 'list_files', workspaceId: 'primary', + paths: [], truncated: false, + }); + assert.equal(source.scans, 3); + assert.equal(source.cappedScans, 2); +}); + +test('successive candidate windows share the original listing deadline', async (t) => { + const { tools } = await workspace(t); + let now = Date.now(); + t.mock.method(Date, 'now', () => now); + const source = candidateSource(t, skippedPaths, () => { now += 6_000; }); + await assert.rejects(tools.execute(request), (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'LIST_TIMEOUT'); + assert.equal(source.scans, 2, 'a later window must not reset the ten-second budget'); +}); + +test('cancellation interrupts a later candidate window', async (t) => { + const { tools } = await workspace(t); + const controller = new AbortController(); + const source = candidateSource(t, skippedPaths, scan => { + if (scan === 2) controller.abort(); + }); + await assert.rejects(tools.execute(request, controller.signal), (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'EXECUTION_ABORTED'); + assert.equal(source.scans, 2); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 5302593a..f7770e76 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1157,202 +1157,213 @@ async function listWorkspaceFiles( .filter((segment) => segment.length > 0 && segment !== '.') .join('/'); const requestedResultPath = normalizedRequestedResultPath || undefined; - const afterPath = request.afterPath; + let afterPath = request.afterPath; - const candidates: Array<{ filesystemPath: string; resultPath: string }> = []; - let truncated = false; - let pending: Buffer = Buffer.alloc(0); - let stoppedForLimit = false; - await new Promise((resolvePromise, reject) => { - const args = [ - '--files', - '--no-config', - '--no-follow', - '--no-messages', - '--sort', - 'path', - '--null', - ]; - if (portableCanonicalListPath !== '.') { - args.push( - '--glob', - canonicalTargetIsDirectory - ? `${portableCanonicalListPath}/**` - : portableCanonicalListPath, - ); - } - args.push('--', '.'); - const child = spawn( - 'rg', - args, - { cwd: root, stdio: ['ignore', 'pipe', 'ignore'] }, - ); - let aborted = false; - let timedOut = false; - const abort = () => { - aborted = true; - child.kill(); - }; - signal?.addEventListener('abort', abort, { once: true }); - if (signal?.aborted) abort(); - const timeout = setTimeout(() => { - timedOut = true; - child.kill(); - }, Math.max(0, deadline - Date.now())); - const cleanup = () => { - clearTimeout(timeout); - signal?.removeEventListener('abort', abort); - }; - const pathDecoder = new TextDecoder('utf-8', { - fatal: true, - ignoreBOM: true, - }); - const consumePath = (rawPath: Buffer) => { - if (rawPath.length === 0 || stoppedForLimit) return; - let path: string; - try { - path = pathDecoder.decode(rawPath); - } catch { - return; + for (;;) { + await withinListDeadline(Promise.resolve(), signal, deadline); + const candidates: Array<{ filesystemPath: string; resultPath: string }> = []; + let truncated = false; + let pending: Buffer = Buffer.alloc(0); + let stoppedForLimit = false; + await new Promise((resolvePromise, reject) => { + const args = [ + '--files', + '--no-config', + '--no-follow', + '--no-messages', + '--sort', + 'path', + '--null', + ]; + if (portableCanonicalListPath !== '.') { + args.push( + '--glob', + canonicalTargetIsDirectory + ? `${portableCanonicalListPath}/**` + : portableCanonicalListPath, + ); } - if (!Buffer.from(path).equals(rawPath)) return; - if (candidates.length === maxResults + BRIDGE_WORKSPACE_LIST_MAX_RESULTS) { - truncated = true; - stoppedForLimit = true; + args.push('--', '.'); + const child = spawn( + 'rg', + args, + { cwd: root, stdio: ['ignore', 'pipe', 'ignore'] }, + ); + let aborted = false; + let timedOut = false; + const abort = () => { + aborted = true; child.kill(); - return; - } - const portablePath = sep === '\\' ? path.split(sep).join('/') : path; - const normalizedPath = portablePath.startsWith('./') - ? portablePath.slice(2) - : portablePath; - const resultPath = - requestedResultPath == null - ? normalizedPath - : portableCanonicalListPath === '.' - ? `${requestedResultPath}/${normalizedPath}` - : normalizedPath === portableCanonicalListPath || - normalizedPath.startsWith(`${portableCanonicalListPath}/`) - ? `${requestedResultPath}${normalizedPath.slice(portableCanonicalListPath.length)}` - : normalizedPath; - if (!isSafePortableRelativePath(resultPath)) return; - if ( - afterPath !== undefined && - comparePortableRelativePaths(resultPath, afterPath) <= 0 - ) { - return; - } - candidates.push({ filesystemPath: normalizedPath, resultPath }); - }; + }; + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + const timeout = setTimeout(() => { + timedOut = true; + child.kill(); + }, Math.max(0, deadline - Date.now())); + const cleanup = () => { + clearTimeout(timeout); + signal?.removeEventListener('abort', abort); + }; + const pathDecoder = new TextDecoder('utf-8', { + fatal: true, + ignoreBOM: true, + }); + const consumePath = (rawPath: Buffer) => { + if (rawPath.length === 0 || stoppedForLimit) return; + let path: string; + try { + path = pathDecoder.decode(rawPath); + } catch { + return; + } + if (!Buffer.from(path).equals(rawPath)) return; + if (candidates.length === maxResults + BRIDGE_WORKSPACE_LIST_MAX_RESULTS) { + truncated = true; + stoppedForLimit = true; + child.kill(); + return; + } + const portablePath = sep === '\\' ? path.split(sep).join('/') : path; + const normalizedPath = portablePath.startsWith('./') + ? portablePath.slice(2) + : portablePath; + const resultPath = + requestedResultPath == null + ? normalizedPath + : portableCanonicalListPath === '.' + ? `${requestedResultPath}/${normalizedPath}` + : normalizedPath === portableCanonicalListPath || + normalizedPath.startsWith(`${portableCanonicalListPath}/`) + ? `${requestedResultPath}${normalizedPath.slice(portableCanonicalListPath.length)}` + : normalizedPath; + if (!isSafePortableRelativePath(resultPath)) return; + if ( + afterPath !== undefined && + comparePortableRelativePaths(resultPath, afterPath) <= 0 + ) { + return; + } + candidates.push({ filesystemPath: normalizedPath, resultPath }); + }; - child.stdout.on('data', (chunk: Buffer) => { - pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]); - let delimiter = pending.indexOf(0); - while (delimiter >= 0) { - consumePath(pending.subarray(0, delimiter)); - pending = pending.subarray(delimiter + 1); - delimiter = pending.indexOf(0); - } - }); - child.once('error', () => { - cleanup(); - reject( - new WorkspaceToolError( - 'Workspace listing unavailable', - 'LIST_UNAVAILABLE', - ), - ); - }); - child.once('close', (code) => { - cleanup(); - consumePath(pending); - if (aborted) { - reject( - new WorkspaceToolError( - 'Workspace tool execution aborted', - 'EXECUTION_ABORTED', - ), - ); - } else if (timedOut) { - reject( - new WorkspaceToolError('Workspace listing timed out', 'LIST_TIMEOUT'), - ); - } else if (stoppedForLimit || code === 0 || code === 1) { - resolvePromise(); - } else { + child.stdout.on('data', (chunk: Buffer) => { + pending = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]); + let delimiter = pending.indexOf(0); + while (delimiter >= 0) { + consumePath(pending.subarray(0, delimiter)); + pending = pending.subarray(delimiter + 1); + delimiter = pending.indexOf(0); + } + }); + child.once('error', () => { + cleanup(); reject( new WorkspaceToolError( 'Workspace listing unavailable', 'LIST_UNAVAILABLE', ), ); - } + }); + child.once('close', (code) => { + cleanup(); + consumePath(pending); + if (aborted) { + reject( + new WorkspaceToolError( + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ); + } else if (timedOut) { + reject( + new WorkspaceToolError('Workspace listing timed out', 'LIST_TIMEOUT'), + ); + } else if (stoppedForLimit || code === 0 || code === 1) { + resolvePromise(); + } else { + reject( + new WorkspaceToolError( + 'Workspace listing unavailable', + 'LIST_UNAVAILABLE', + ), + ); + } + }); }); - }); - const paths: string[] = []; - const seenPaths = new Set(); - for (const candidate of candidates) { - let canonicalPath: string; - try { - canonicalPath = await withinListDeadline( - realpath(resolveWorkspacePath(root, candidate.filesystemPath)), - signal, - deadline, - ); - } catch (error) { - if (error instanceof WorkspaceToolError) throw error; - continue; - } - if (!isWithinRoot(root, canonicalPath)) { - continue; - } - const reportedPath = resolveWorkspacePath(root, candidate.resultPath); - try { - const reportedPathStat = await withinListDeadline( - lstat(reportedPath), - signal, - deadline, - ); - if (reportedPathStat.isSymbolicLink()) continue; - const canonicalReportedPath = await withinListDeadline( - realpath(reportedPath), - signal, - deadline, - ); - if (canonicalReportedPath !== canonicalPath) continue; - } catch (error) { - if (error instanceof WorkspaceToolError) throw error; - continue; + const paths: string[] = []; + const seenPaths = new Set(); + for (const candidate of candidates) { + let canonicalPath: string; + try { + canonicalPath = await withinListDeadline( + realpath(resolveWorkspacePath(root, candidate.filesystemPath)), + signal, + deadline, + ); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + continue; + } + if (!isWithinRoot(root, canonicalPath)) { + continue; + } + const reportedPath = resolveWorkspacePath(root, candidate.resultPath); + try { + const reportedPathStat = await withinListDeadline( + lstat(reportedPath), + signal, + deadline, + ); + if (reportedPathStat.isSymbolicLink()) continue; + const canonicalReportedPath = await withinListDeadline( + realpath(reportedPath), + signal, + deadline, + ); + if (canonicalReportedPath !== canonicalPath) continue; + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + continue; + } + let regularFile = false; + try { + regularFile = ( + await withinListDeadline(stat(canonicalPath), signal, deadline) + ).isFile(); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + continue; + } + if (!regularFile || seenPaths.has(candidate.resultPath)) continue; + if (paths.length === maxResults) { + truncated = true; + break; + } + seenPaths.add(candidate.resultPath); + paths.push(candidate.resultPath); } - let regularFile = false; - try { - regularFile = ( - await withinListDeadline(stat(canonicalPath), signal, deadline) - ).isFile(); - } catch (error) { - if (error instanceof WorkspaceToolError) throw error; + + if (truncated && paths.length === 0) { + // A scan window can consist entirely of vanished files or symlinks. + // Advance the internal scan cursor, not the public page cursor, and + // keep the original deadline and per-window candidate limit. + afterPath = candidates[candidates.length - 1].resultPath; continue; } - if (!regularFile || seenPaths.has(candidate.resultPath)) continue; - if (paths.length === maxResults) { - truncated = true; - break; - } - seenPaths.add(candidate.resultPath); - paths.push(candidate.resultPath); - } - return { - protocolVersion: BRIDGE_PROTOCOL_VERSION, - operation: 'list_files', - workspaceId: request.workspaceId, - paths, - truncated, - ...(truncated && paths.length > 0 - ? { nextAfterPath: paths[paths.length - 1] } - : {}), - }; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'list_files', + workspaceId: request.workspaceId, + paths, + truncated, + ...(truncated && paths.length > 0 + ? { nextAfterPath: paths[paths.length - 1] } + : {}), + }; + } } async function withinListDeadline( @@ -1360,6 +1371,9 @@ async function withinListDeadline( signal: AbortSignal | undefined, deadline: number, ): Promise { + // The filesystem operation has already started. Observe its rejection even + // when cancellation or the deadline prevents us from waiting for it. + void operation.catch(() => undefined); if (signal?.aborted) { throw new WorkspaceToolError( 'Workspace tool execution aborted', From 2634f31ff50304c654d4a378cc13db8e2efa861c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 18:14:18 -0400 Subject: [PATCH 051/116] fix: fail closed when credential ACL verification is unavailable (#138) --- packages/code/README.md | 13 ++++- packages/code/src/github.ts | 2 + packages/code/src/private-storage.test.ts | 63 +++++++++++++++++++++++ packages/code/src/private-storage.ts | 16 ++++++ packages/code/src/storage.ts | 16 +++--- 5 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 packages/code/src/private-storage.test.ts create mode 100644 packages/code/src/private-storage.ts diff --git a/packages/code/README.md b/packages/code/README.md index b0bcad80..faddc481 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -35,6 +35,14 @@ LIBRECHAT_CODE_SANDBOX_ENDPOINT=http://127.0.0.1:2000/api/v2 \ librechat-code run ``` +Credential and quarantine storage currently requires Linux (including WSL2), +where ownership and POSIX mode/ACL-mask checks can establish owner-only access. +Native Windows and macOS fail closed before pairing-code redemption, credential +loads, or storage mutations because their extended ACLs cannot yet be verified. +Use storage on a native Linux filesystem, not a Windows drive under `/mnt`. +Support for these platforms requires a native ACL verifier; `chmod` alone is +not sufficient. + Use `--identity ` while pairing and `LIBRECHAT_CODE_IDENTITY_FILE=` while running to override the identity file location. @@ -110,8 +118,9 @@ read only by the trusted worker, which mints and refreshes short-lived installation tokens. A personal access token is supported as a fallback with `LIBRECHAT_CODE_GITHUB_TOKEN`, but the GitHub App is the safer default because its repository access and permissions can be narrowly installed and revoked. -Native Windows currently requires token mode because the worker cannot -reliably validate private-key ACLs there; use WSL2 for GitHub App mode. +Native Windows and macOS credential storage are unavailable until native ACL +verification is implemented; use Linux or WSL2. This also applies to GitHub App +private keys. Git receives authentication through process-scoped `GIT_CONFIG_*` variables. The same isolated config supplies the standard Git LFS filters; hosts using LFS diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 9d265e58..058cea83 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -2,6 +2,7 @@ import { constants } from 'node:fs'; import { createHash, createPrivateKey, sign } from 'node:crypto'; import { open, realpath, stat } from 'node:fs/promises'; import { dirname } from 'node:path'; +import { assertPrivateStorageSupported } from './private-storage.js'; export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; export const GITHUB_ALLOWED_DOMAINS = [ @@ -44,6 +45,7 @@ function assertPositiveIdentifier(name: string, value: string): void { } async function readPrivateKey(path: string): Promise { + assertPrivateStorageSupported(); if (process.platform !== 'win32') { const directory = await stat(await realpath(dirname(path))); const uid = process.getuid?.(); diff --git a/packages/code/src/private-storage.test.ts b/packages/code/src/private-storage.test.ts new file mode 100644 index 00000000..c43772ef --- /dev/null +++ b/packages/code/src/private-storage.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { assertPrivateStorageSupported } from './private-storage.js'; +import { assertIdentityPathIsPrivate, saveBridgeIdentity, loadBridgeIdentity, + saveWorkspaceMutationQuarantine, loadWorkspaceMutationQuarantine, + clearWorkspaceMutationQuarantine, ensurePrivateWorkspaceDirectory } from './storage.js'; +import { GitHubAppCredentialProvider } from './github.js'; + +const identity = { protocolVersion: 1 as const, workerId: 'worker', + codeApiUrl: 'https://example.com', credential: 'private', privateKey: 'private', + publicKey: 'public', expiresAt: '2099-01-01T00:00:00Z' }; +const quarantine = { version: 1 as const, workerId: 'worker', workspaceId: 'primary', + reason: 'uncertain mutation', quarantinedAt: '2026-01-01T00:00:00Z' }; + +for (const unsupported of ['win32', 'darwin', 'freebsd']) { + test(`${unsupported} refuses storage before creation, reads, or deletion`, async t => { + const root = await mkdtemp(join(tmpdir(), 'private-storage-')); + t.after(() => rm(root, { recursive: true, force: true })); + const existing = join(root, 'existing.json'); + await writeFile(existing, JSON.stringify(identity), { mode: 0o600 }); + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + t.after(() => Object.defineProperty(process, 'platform', platform)); + Object.defineProperty(process, 'platform', { ...platform, value: unsupported }); + const path = join(root, 'missing', 'identity.json'); + for (const action of [ + () => assertIdentityPathIsPrivate(path), () => saveBridgeIdentity(path, identity), + () => loadBridgeIdentity(existing), () => saveWorkspaceMutationQuarantine(path, quarantine), + () => loadWorkspaceMutationQuarantine(path), () => clearWorkspaceMutationQuarantine(existing), + () => ensurePrivateWorkspaceDirectory(join(root, 'workspace')), + ]) await assert.rejects(action(), /ACL verification is unavailable/); + assert.deepEqual(await readdir(root), ['existing.json']); + assert.equal(await readFile(existing, 'utf8'), JSON.stringify(identity)); + }); +} + +test('GitHub App credentials also reject unsupported ACL verification before signing or fetching', async t => { + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + t.after(() => Object.defineProperty(process, 'platform', platform)); + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + let fetched = false; + const provider = new GitHubAppCredentialProvider({ + appId: '1', installationId: '1', privateKeyPath: '/must-not-be-read', + fetch: async () => { fetched = true; throw new Error('must not fetch'); }, + }); + await assert.rejects(provider.getCredential(), /ACL verification is unavailable/); + assert.equal(fetched, false); +}); + +test('Linux still requires an available ownership API', t => { + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + const getuid = Object.getOwnPropertyDescriptor(process, 'getuid'); + t.after(() => { + Object.defineProperty(process, 'platform', platform); + if (getuid) Object.defineProperty(process, 'getuid', getuid); + else delete process.getuid; + }); + Object.defineProperty(process, 'platform', { ...platform, value: 'linux' }); + Object.defineProperty(process, 'getuid', { configurable: true, value: undefined }); + assert.throws(assertPrivateStorageSupported, /ACL verification is unavailable/); +}); diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts new file mode 100644 index 00000000..90337c5c --- /dev/null +++ b/packages/code/src/private-storage.ts @@ -0,0 +1,16 @@ +import { BridgeProtocolError } from './protocol.js'; + +/** Linux POSIX ACL masks are reflected in group mode bits. macOS extended + * ACLs and Windows DACLs are not: chmod/stat alone cannot establish privacy. + * Fail before creating files, reading credentials, or redeeming pairing codes + * until a native verifier can inspect the actual opened object's ACLs. + */ +export function assertPrivateStorageSupported(): void { + if (process.platform !== 'linux' || process.getuid === undefined) { + throw new BridgeProtocolError( + 'Owner-only storage ACL verification is unavailable on this platform. ' + + 'Worker credentials, GitHub App keys, and quarantine state require Linux ' + + '(including WSL2) with storage on a native Linux filesystem, not /mnt.', + ); + } +} diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index af3e90a7..adf2e526 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -14,6 +14,7 @@ import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; +import { assertPrivateStorageSupported } from './private-storage.js'; import type { PairedBridgeWorkerIdentity } from './pairing.js'; @@ -168,12 +169,8 @@ export function defaultWorkspaceQuarantinePath( * Symlinks are resolved: a link's own mode is always `0777` and ignored by the * kernel, so the file the bytes live in is what counts. * - * This reads POSIX mode bits, which is not the whole access story everywhere. - * A Linux POSIX ACL surfaces its mask in the group bits and so is caught, but - * a macOS extended ACL inherited from the parent directory is invisible here - * and survives `chmod`, and Windows is exempt entirely. Establishing owner-only - * storage on those needs real ACL inspection; until then this verifies what the - * mode can express and nothing more. + * Linux POSIX ACL masks are reflected in group mode bits. Platforms whose + * ACLs cannot be verified are rejected at the storage entry points. */ async function groupOrOtherAccessMode( path: string, @@ -310,6 +307,7 @@ async function assertOwnerOnlyPath( export async function ensurePrivateWorkspaceDirectory( path: string, ): Promise { + assertPrivateStorageSupported(); await mkdir(path, { recursive: true, mode: 0o700 }); const metadata = await lstat(path); if (!metadata.isDirectory() || metadata.isSymbolicLink()) { @@ -390,6 +388,7 @@ async function assertSiblingPublishable(path: string): Promise { export async function assertIdentityPathIsPrivate( path: string, ): Promise { + assertPrivateStorageSupported(); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); await assertIdentityDestinationIsReplaceable(path); let created = false; @@ -452,6 +451,7 @@ export async function saveBridgeIdentity( path: string, identity: PairedBridgeWorkerIdentity, ): Promise { + assertPrivateStorageSupported(); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); const temporaryPath = `${path}.${randomBytes(8).toString('hex')}.tmp`; try { @@ -492,6 +492,7 @@ export async function saveWorkspaceMutationQuarantine( path: string, record: WorkspaceMutationQuarantineRecord, ): Promise { + assertPrivateStorageSupported(); await ensureDurableDirectory(dirname(path)); const file = await open(path, 'wx', 0o600); try { @@ -521,6 +522,7 @@ export async function saveWorkspaceMutationQuarantine( export async function loadWorkspaceMutationQuarantine( path: string, ): Promise { + assertPrivateStorageSupported(); /* A marker another account can rewrite is not a control: it could be cleared * to resume mutations, or forged to wedge the worker under a foreign owner. */ let content: string; @@ -553,6 +555,7 @@ export async function clearWorkspaceMutationQuarantine( path: string, ownerId?: string, ): Promise { + assertPrivateStorageSupported(); if (ownerId != null) { const record = await loadWorkspaceMutationQuarantine(path); if (record == null || record.ownerId !== ownerId) { @@ -586,6 +589,7 @@ export async function assertWorkspaceMutationQuarantineOwner( export async function loadBridgeIdentity( path: string, ): Promise { + assertPrivateStorageSupported(); /* An identity written before this check, or by an older release, is still a * private key other local accounts can read. Refuse it rather than booting. */ const content = await readGuardedFile( From 10bd0db4eebcdc46d3336a81b6d33124b45cc571 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 18:21:20 -0400 Subject: [PATCH 052/116] fix: validate every credential storage ancestor (#139) --- packages/code/README.md | 6 + packages/code/src/github.test.ts | 2 +- packages/code/src/github.ts | 22 +--- packages/code/src/private-storage.ts | 56 +++++++++ packages/code/src/storage-ancestors.test.ts | 119 ++++++++++++++++++++ packages/code/src/storage.ts | 72 +++--------- 6 files changed, 202 insertions(+), 75 deletions(-) create mode 100644 packages/code/src/storage-ancestors.test.ts diff --git a/packages/code/README.md b/packages/code/README.md index faddc481..f9573ee5 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -43,6 +43,12 @@ Use storage on a native Linux filesystem, not a Windows drive under `/mnt`. Support for these platforms requires a native ACL verifier; `chmod` alone is not sufficient. +Every storage ancestor, including intermediate symlink entries and targets, must +be owned by this account or root and must not allow group/other writes unless +protected by the sticky bit. A private directory inside a shared writable parent +is insufficient: that parent can replace the directory. This also applies when +loading GitHub App keys or clearing quarantine state. + Use `--identity ` while pairing and `LIBRECHAT_CODE_IDENTITY_FILE=` while running to override the identity file location. diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 8006f2fe..81dbb2c3 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -197,7 +197,7 @@ test('rejects a GitHub App key in a shared writable directory', async (t) => { await assert.rejects( provider.getCredential(), - /private key directory must not be writable/, + /writable by other accounts/, ); }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 058cea83..7bacc297 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,8 +1,8 @@ import { constants } from 'node:fs'; import { createHash, createPrivateKey, sign } from 'node:crypto'; -import { open, realpath, stat } from 'node:fs/promises'; +import { open } from 'node:fs/promises'; import { dirname } from 'node:path'; -import { assertPrivateStorageSupported } from './private-storage.js'; +import { assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; export const GITHUB_ALLOWED_DOMAINS = [ @@ -46,23 +46,7 @@ function assertPositiveIdentifier(name: string, value: string): void { async function readPrivateKey(path: string): Promise { assertPrivateStorageSupported(); - if (process.platform !== 'win32') { - const directory = await stat(await realpath(dirname(path))); - const uid = process.getuid?.(); - if (uid !== undefined && directory.uid !== uid && directory.uid !== 0) { - throw new Error( - 'GitHub App private key directory must be owned by this user or root', - ); - } - const mode = directory.mode & 0o7777; - const protectedByStickyBit = - (mode & 0o1000) !== 0 && (directory.uid === uid || directory.uid === 0); - if ((mode & 0o022) !== 0 && !protectedByStickyBit) { - throw new Error( - 'GitHub App private key directory must not be writable by group or other users', - ); - } - } + await assertPrivateStorageAncestors(dirname(path)); const handle = await open( path, diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts index 90337c5c..ea9952a7 100644 --- a/packages/code/src/private-storage.ts +++ b/packages/code/src/private-storage.ts @@ -1,3 +1,6 @@ +import { lstat, readlink } from 'node:fs/promises'; +import { dirname, isAbsolute } from 'node:path'; + import { BridgeProtocolError } from './protocol.js'; /** Linux POSIX ACL masks are reflected in group mode bits. macOS extended @@ -14,3 +17,56 @@ export function assertPrivateStorageSupported(): void { ); } } + +/** + * Walk from the trust root before touching a descendant. Checking only a + * canonical parent misses replaceable ancestors and symlink entries. Resolve + * links one component at a time so even intermediate link targets are checked. + * Other local accounts cannot replace a checked entry: its parent is either + * non-writable or sticky and the entry belongs to this account or root. + */ +export async function assertPrivateStorageAncestors( + path: string, + allowMissing = false, +): Promise { + assertPrivateStorageSupported(); + const uid = process.getuid!(); + let current = '/'; + const pending = (isAbsolute(path) ? path : `${process.cwd()}/${path}`).split('/'); + let links = 0; + while (true) { + const metadata = await lstat(current).catch((error: NodeJS.ErrnoException) => { + if (allowMissing && error.code === 'ENOENT') return undefined; + throw error; + }); + if (metadata === undefined) return; + if (metadata.uid !== uid && metadata.uid !== 0) { + throw new BridgeProtocolError( + `${current} is owned by another account (uid ${metadata.uid}), ` + + `which can replace ${path}. Keep worker storage on paths this account or root owns.`, + ); + } + if (metadata.isSymbolicLink()) { + if (++links > 40) throw new BridgeProtocolError(`Too many storage symlinks: ${path}`); + const target = await readlink(current); + current = isAbsolute(target) ? '/' : dirname(current); + pending.unshift(...target.split('/')); + continue; + } + if (metadata.isDirectory()) { + const mode = metadata.mode & 0o7777; + if ((mode & 0o022) !== 0 && (mode & 0o1000) === 0) { + throw new BridgeProtocolError( + `Directory ${current} is writable by other accounts (mode ${mode.toString(8)}), ` + + `so ${path} can be replaced even while owner-only.`, + ); + } + } else if (pending.some((part) => part !== '' && part !== '.')) { + throw new BridgeProtocolError(`Storage ancestor must be a directory: ${current}`); + } + let next = pending.shift(); + while (next === '' || next === '.') next = pending.shift(); + if (next === undefined) return; + current = next === '..' ? dirname(current) : `${current === '/' ? '' : current}/${next}`; + } +} diff --git a/packages/code/src/storage-ancestors.test.ts b/packages/code/src/storage-ancestors.test.ts new file mode 100644 index 00000000..1afc7705 --- /dev/null +++ b/packages/code/src/storage-ancestors.test.ts @@ -0,0 +1,119 @@ +import assert from 'node:assert/strict'; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { GitHubAppCredentialProvider } from './github.js'; +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { + assertIdentityPathIsPrivate, clearWorkspaceMutationQuarantine, + ensurePrivateWorkspaceDirectory, loadBridgeIdentity, loadWorkspaceMutationQuarantine, + saveBridgeIdentity, saveWorkspaceMutationQuarantine, +} from './storage.js'; + +const identity = { + protocolVersion: 1 as const, workerId: 'worker', codeApiUrl: 'https://code.example/v1', + credential: 'secret', expiresAt: '2099-01-01T00:00:00Z', publicKey: 'public', privateKey: 'private', +}; +const marker = { + version: 1 as const, workerId: 'worker', workspaceId: 'workspace', ownerId: 'owner', + quarantinedAt: '2026-01-01T00:00:00Z', reason: 'uncertain result', +}; + +test('a writable ancestor blocks every storage operation before changing private descendants', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'storage-ancestors-')); + t.after(() => rm(root, { recursive: true, force: true })); + const parent = join(root, 'shared'); + const privateDir = join(parent, 'private'); + await mkdir(privateDir, { recursive: true, mode: 0o700 }); + const credential = join(privateDir, 'identity.json'); + const quarantine = join(privateDir, 'quarantine.json'); + await saveBridgeIdentity(credential, identity); + await saveWorkspaceMutationQuarantine(quarantine, marker); + const reservation = await assertIdentityPathIsPrivate(join(privateDir, 'reserved.json')); + await chmod(parent, 0o777); + for (const operation of [ + () => loadBridgeIdentity(credential), + () => saveBridgeIdentity(credential, identity), + () => assertIdentityPathIsPrivate(credential), + () => assertIdentityPathIsPrivate(join(privateDir, 'missing', 'identity.json')), + () => loadWorkspaceMutationQuarantine(quarantine), + () => loadWorkspaceMutationQuarantine(join(privateDir, 'absent.json')), + () => saveWorkspaceMutationQuarantine(join(privateDir, 'new.json'), marker), + () => clearWorkspaceMutationQuarantine(quarantine), + () => clearWorkspaceMutationQuarantine(quarantine, 'owner'), + () => ensurePrivateWorkspaceDirectory(join(privateDir, 'workspace')), + () => reservation.release(), + ]) await assert.rejects(operation(), /writable by other accounts/); + assert.deepEqual((await readdir(privateDir)).sort(), ['identity.json', 'quarantine.json', 'reserved.json']); + assert.deepEqual(JSON.parse(await readFile(credential, 'utf8')), identity); + assert.deepEqual(JSON.parse(await readFile(quarantine, 'utf8')), marker); + + // A trusted sticky parent protects this account's private directory entry. + await chmod(parent, 0o1777); + assert.deepEqual(await loadBridgeIdentity(credential), identity); + await clearWorkspaceMutationQuarantine(quarantine, 'owner'); + await reservation.release(); +}); + +test('intermediate symlinks cannot hide replaceable entry or target ancestors', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'storage-links-')); + t.after(() => rm(root, { recursive: true, force: true })); + const shared = join(root, 'shared'); + const privateDir = join(shared, 'private'); + const safe = join(root, 'safe'); + await mkdir(privateDir, { recursive: true, mode: 0o700 }); + await mkdir(safe, { mode: 0o700 }); + const file = join(safe, 'identity.json'); + await saveBridgeIdentity(file, identity); + await symlink(safe, join(privateDir, 'alias')); + await symlink(join(privateDir, 'alias'), join(root, 'indirect')); + await symlink(privateDir, join(root, 'target')); + await chmod(shared, 0o777); + for (const path of [ + join(privateDir, 'alias', 'identity.json'), + join(root, 'indirect', 'identity.json'), + join(root, 'target', 'missing.json'), + // Do not lexically normalize away a component the kernel traverses. + `${root}/indirect/../safe/identity.json`, + ]) await assert.rejects(assertIdentityPathIsPrivate(path), /writable by other accounts/); + await assert.rejects(loadBridgeIdentity(join(root, 'indirect', 'identity.json')), /writable by other accounts/); +}); + +test('a symlink owned by another account is rejected even under a trusted sticky parent', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'storage-link-owner-')); + t.after(() => rm(root, { recursive: true, force: true })); + const link = join(root, 'alias'); + await symlink(root, link); + // Use a metadata fixture: changing real ownership requires administrator access. + const fs = await import('node:fs/promises'); + const { syncBuiltinESMExports } = await import('node:module'); + const original = fs.default.lstat; + t.after(() => { fs.default.lstat = original; syncBuiltinESMExports(); }); + fs.default.lstat = (async (...args: Parameters) => { + const metadata = await original(...args); + if (String(args[0]).endsWith('/alias')) Object.defineProperty(metadata, 'uid', { value: process.getuid!() + 1000 }); + return metadata; + }) as typeof original; + syncBuiltinESMExports(); + await chmod(root, 0o1777); + await assert.rejects(assertPrivateStorageAncestors(link), /owned by another account/); +}); + +test('GitHub App keys reject writable ancestors before making a token request', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'github-ancestors-')); + t.after(() => rm(root, { recursive: true, force: true })); + const privateDir = join(root, 'private'); + await mkdir(privateDir, { mode: 0o700 }); + const path = join(privateDir, 'app.pem'); + await writeFile(path, 'never read', { mode: 0o600 }); + await chmod(root, 0o777); + let requested = false; + const provider = new GitHubAppCredentialProvider({ + appId: '1', installationId: '2', privateKeyPath: path, + fetch: async () => { requested = true; throw new Error('unexpected request'); }, + }); + await assert.rejects(provider.getCredential(), /writable by other accounts/); + assert.equal(requested, false); +}); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index adf2e526..c4dab0bd 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -4,8 +4,6 @@ import { lstat, mkdir, open, - readFile, - realpath, rename, rm, stat, @@ -14,7 +12,7 @@ import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; -import { assertPrivateStorageSupported } from './private-storage.js'; +import { assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; import type { PairedBridgeWorkerIdentity } from './pairing.js'; @@ -182,60 +180,13 @@ async function groupOrOtherAccessMode( return (mode & 0o077) === 0 ? undefined : mode; } -/** - * A `0600` file in a directory other accounts can write is not owner-only in - * practice: they cannot read it, but they can unlink and substitute it, so a - * swapped credential or a forged quarantine marker would be trusted. The sticky - * bit counts as protection, which keeps shared `/tmp`-style parents usable. A - * writable ancestor above a private directory could still have that directory - * renamed out from under us, which is broader hardening than this addresses. - * - * Deliberately not applied to the registered workspace, which is the user's own - * project directory and may legitimately be shared. - */ -async function assertDirectoryNotSharedWritable( - directory: string, - path: string, -): Promise { - const metadata = await stat(directory); - const mode = metadata.mode & 0o7777; - const uid = process.getuid?.(); - if (uid !== undefined && !isTrustedOwner(metadata.uid, uid)) { - throw new BridgeProtocolError( - `Directory ${directory} is owned by another account (uid ${metadata.uid}), ` + - `which can grant itself write access and replace ${path}. Keep worker ` + - 'credentials in a directory this account owns.', - ); - } - if ((mode & 0o022) === 0) return; - if ((mode & 0o1000) !== 0 && (metadata.uid === uid || metadata.uid === 0)) { - return; - } - throw new BridgeProtocolError( - `Directory ${directory} is writable by other accounts (mode ${mode.toString(8)}), ` + - `so ${path} can be replaced even while owner-only. Keep worker credentials ` + - 'in a directory only this account can write.', - ); -} - -/** Publishing goes through `rename`, which replaces the named entry itself. */ +/** Publishing replaces the entry itself; reading also follows its target. */ async function assertWriteContainerPrivate(path: string): Promise { - if (process.platform === 'win32' || process.getuid === undefined) return; - await assertDirectoryNotSharedWritable(await realpath(dirname(path)), path); + await assertPrivateStorageAncestors(dirname(path), true); } -/** - * Reading follows the link, so both the entry and the file it names are trust - * boundaries: a writable directory at either end allows a substitution. - */ async function assertReadPathPrivate(path: string): Promise { - if (process.platform === 'win32' || process.getuid === undefined) return; - const entryDirectory = await realpath(dirname(path)); - await assertDirectoryNotSharedWritable(entryDirectory, path); - const targetDirectory = dirname(await realpath(path)); - if (targetDirectory !== entryDirectory) { - await assertDirectoryNotSharedWritable(targetDirectory, path); - } + await assertPrivateStorageAncestors(path); } /** Root is the trust root; anyone else holding a credential path is not. */ @@ -270,6 +221,7 @@ async function readGuardedFile( path: string, exposed: (mode: string) => string, ): Promise { + await assertReadPathPrivate(path); const handle = await open(path, 'r'); try { const stats = await handle.stat(); @@ -308,16 +260,18 @@ export async function ensurePrivateWorkspaceDirectory( path: string, ): Promise { assertPrivateStorageSupported(); + await assertWriteContainerPrivate(path); await mkdir(path, { recursive: true, mode: 0o700 }); + await assertWriteContainerPrivate(path); const metadata = await lstat(path); if (!metadata.isDirectory() || metadata.isSymbolicLink()) { throw new BridgeProtocolError('Default workspace path must be a directory'); } - await chmod(path, 0o700); - await assertOwnerOnlyPath(path); /* This directory is application-owned by contract; a pre-existing one under * another account lets that owner alter workspace inputs and results. */ await assertOwnedByWorker(path); + await chmod(path, 0o700); + await assertOwnerOnlyPath(path); } /** @@ -389,7 +343,9 @@ export async function assertIdentityPathIsPrivate( path: string, ): Promise { assertPrivateStorageSupported(); + await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await assertWriteContainerPrivate(path); await assertIdentityDestinationIsReplaceable(path); let created = false; let reservedInode: bigint | undefined; @@ -431,6 +387,7 @@ export async function assertIdentityPathIsPrivate( return { async release(): Promise { if (!created || reservedInode === undefined) return; + await assertWriteContainerPrivate(path); /* Only ever drop the placeholder this call made. A concurrent `pair` * may have published a real identity over the name since, and removing * that would destroy a credential whose code is already spent. */ @@ -452,7 +409,9 @@ export async function saveBridgeIdentity( identity: PairedBridgeWorkerIdentity, ): Promise { assertPrivateStorageSupported(); + await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await assertWriteContainerPrivate(path); const temporaryPath = `${path}.${randomBytes(8).toString('hex')}.tmp`; try { const file = await open(temporaryPath, 'wx', 0o600); @@ -493,7 +452,9 @@ export async function saveWorkspaceMutationQuarantine( record: WorkspaceMutationQuarantineRecord, ): Promise { assertPrivateStorageSupported(); + await assertWriteContainerPrivate(path); await ensureDurableDirectory(dirname(path)); + await assertWriteContainerPrivate(path); const file = await open(path, 'wx', 0o600); try { try { @@ -556,6 +517,7 @@ export async function clearWorkspaceMutationQuarantine( ownerId?: string, ): Promise { assertPrivateStorageSupported(); + await assertWriteContainerPrivate(path); if (ownerId != null) { const record = await loadWorkspaceMutationQuarantine(path); if (record == null || record.ownerId !== ownerId) { From 3104227f3a612d9497e708cc6fcbbb585b574c0c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 18:24:53 -0400 Subject: [PATCH 053/116] fix: reject identity mount targets before pairing (#140) --- packages/code/README.md | 8 ++ packages/code/src/identity-mount.test.ts | 103 +++++++++++++++++++++++ packages/code/src/identity-mount.ts | 61 ++++++++++++++ packages/code/src/storage.ts | 9 +- 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 packages/code/src/identity-mount.test.ts create mode 100644 packages/code/src/identity-mount.ts diff --git a/packages/code/README.md b/packages/code/README.md index f9573ee5..43b681a0 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -53,6 +53,14 @@ Use `--identity ` while pairing and `LIBRECHAT_CODE_IDENTITY_FILE=` while running to override the identity file location. +The identity file itself must not be a bind-mount target: saving a paired +credential atomically replaces that entry. Mount its containing directory +instead. Pairing preflight checks `/proc/self/mountinfo` before redeeming the +one-time code and fails closed if mount information cannot be verified (including +a mount table larger than 4 MiB). Existing identity reads remain supported. +The check describes the current mount namespace; administrators must keep mount +configuration stable during pairing. + ## Native BYOM sandbox (default) The MVP command sandbox runs directly on the user's chosen laptop or VM. It diff --git a/packages/code/src/identity-mount.test.ts b/packages/code/src/identity-mount.test.ts new file mode 100644 index 00000000..13f99904 --- /dev/null +++ b/packages/code/src/identity-mount.test.ts @@ -0,0 +1,103 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs, { mkdtemp, readFile, readdir, realpath, rm, symlink, writeFile } from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { identityIsMountPoint } from './identity-mount.js'; +import { assertIdentityPathIsPrivate, saveBridgeIdentity } from './storage.js'; + +const rootMount = '1 0 8:1 / / rw - ext4 /dev/root rw\n'; +const encode = (path: string) => path.replace(/[\\ \t\n]/g, (char) => + `\\${char.charCodeAt(0).toString(8).padStart(3, '0')}`, +); +const mount = (path: string) => `${rootMount}2 1 8:1 /source ${encode(path)} rw shared:1 - ext4 /dev/root rw\n`; +const identity = { + protocolVersion: 1 as const, workerId: 'worker', codeApiUrl: 'https://code.example/v1', + credential: 'secret', expiresAt: '2099-01-01T00:00:00Z', publicKey: 'public', privateKey: 'private', +}; + +test('mount parsing detects same-device bind mounts and escaped path names', () => { + for (const path of ['/home/worker/key.json', '/home/worker/key with\tline\nbreak\\040']) { + assert.equal(identityIsMountPoint(mount(path), path), true); + assert.equal(identityIsMountPoint(mount(path), `${path}.sibling`), false); + } + assert.equal(identityIsMountPoint(mount('/home/worker'), '/home/worker/key.json'), false); + for (const invalid of ['', 'malformed\n', '1 0 8:1 / /bad\\999 rw - ext4 /dev/root rw\n']) { + assert.throws(() => identityIsMountPoint(invalid, '/key'), /malformed/); + } +}); + +test('preflight and direct saves refuse mounted destinations without altering them', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'identity-mount-')); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'identity.json'); + const fixture = join(root, 'mountinfo'); + await writeFile(path, 'existing credential', { mode: 0o600 }); + await writeFile(fixture, mount(await realpath(path))); + const original = fs.open; + fs.open = ((path, ...args) => original(path === '/proc/self/mountinfo' ? fixture : path, ...args)) as typeof fs.open; + syncBuiltinESMExports(); + t.after(() => { fs.open = original; syncBuiltinESMExports(); }); + await assert.rejects(assertIdentityPathIsPrivate(path), /is a mount point/); + await assert.rejects(saveBridgeIdentity(path, identity), /is a mount point/); + assert.equal(await readFile(path, 'utf8'), 'existing credential'); + assert.deepEqual((await readdir(root)).sort(), ['identity.json', 'mountinfo']); + + // Parent aliases still name the mounted entry; a leaf link can be replaced. + const parentAlias = join(root, 'alias'); + await symlink(root, parentAlias); + await assert.rejects(assertIdentityPathIsPrivate(join(parentAlias, 'identity.json')), /is a mount point/); + const leaf = join(root, 'leaf.json'); + await symlink(path, leaf); + await assertIdentityPathIsPrivate(leaf); + await saveBridgeIdentity(leaf, identity); + assert.deepEqual(JSON.parse(await readFile(leaf, 'utf8')), identity); + assert.equal(await readFile(path, 'utf8'), 'existing credential'); +}); + +test('unavailable, malformed, and oversized mount tables fail before reserving a new identity', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'identity-mount-info-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fixture = join(root, 'mountinfo'); + const path = join(root, 'identity.json'); + const original = fs.open; + fs.open = ((path, ...args) => original(path === '/proc/self/mountinfo' ? fixture : path, ...args)) as typeof fs.open; + syncBuiltinESMExports(); + t.after(() => { fs.open = original; syncBuiltinESMExports(); }); + await assert.rejects(assertIdentityPathIsPrivate(path), /Cannot verify identity mount status/); + for (const content of ['bad', 'x'.repeat(4 * 1024 * 1024 + 1)]) { + await writeFile(fixture, content); + await assert.rejects(assertIdentityPathIsPrivate(path), /Cannot verify identity mount status/); + } + assert.deepEqual(await readdir(root), ['mountinfo']); +}); + +test('CLI refuses a mounted identity before redeeming the one-time pairing code', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'identity-mount-cli-')); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'identity.json'); + const fixture = join(root, 'mountinfo'); + const preload = join(root, 'preload.mjs'); + await writeFile(path, 'existing credential', { mode: 0o600 }); + await writeFile(fixture, mount(await realpath(path))); + await writeFile(preload, ` +import fs from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +Object.defineProperty(process, 'platform', { value: 'linux' }); +const original = fs.open; +fs.open = (path, ...args) => original(path === '/proc/self/mountinfo' ? ${JSON.stringify(fixture)} : path, ...args); +syncBuiltinESMExports(); +globalThis.fetch = async () => { process.stderr.write('PAIRING_REQUEST_ATTEMPTED'); throw new Error('unexpected request'); }; +`); + const result = spawnSync(process.execPath, [ + '--import', preload, new URL('./cli.js', import.meta.url).pathname, + 'pair', 'https://code.example/v1', 'one-time-code', '--worker-id', 'worker', '--identity', path, + ], { encoding: 'utf8', timeout: 10_000 }); + assert.equal(result.status, 1, result.stderr); + assert.match(result.stderr, /is a mount point/); + assert.doesNotMatch(result.stderr, /PAIRING_REQUEST_ATTEMPTED/); + assert.equal(await readFile(path, 'utf8'), 'existing credential'); +}); diff --git a/packages/code/src/identity-mount.ts b/packages/code/src/identity-mount.ts new file mode 100644 index 00000000..ba87d6d6 --- /dev/null +++ b/packages/code/src/identity-mount.ts @@ -0,0 +1,61 @@ +import { open, realpath } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; + +import { BridgeProtocolError } from './protocol.js'; + +const MOUNTINFO_MAX_BYTES = 4 * 1024 * 1024; + +/** mountinfo describes this process's namespace, including same-device bind mounts. */ +export function identityIsMountPoint(mountinfo: string, entryPath: string): boolean { + const lines = mountinfo.trimEnd().split('\n'); + let mounted = false; + for (const line of lines) { + const fields = line.split(' '); + const separator = fields.indexOf('-', 6); + const mountpoint = fields[4]; + if ( + separator < 6 || fields.length !== separator + 4 || + !/^\d+$/.test(fields[0] ?? '') || !/^\d+$/.test(fields[1] ?? '') || + !/^\d+:\d+$/.test(fields[2] ?? '') || !mountpoint?.startsWith('/') || + /\\(?!040|011|012|134)/.test(mountpoint) + ) throw new BridgeProtocolError('Cannot verify identity mount status: malformed /proc/self/mountinfo'); + const decoded = mountpoint.replace(/\\(040|011|012|134)/g, (_, octal: string) => + String.fromCharCode(parseInt(octal, 8)), + ); + if (decoded === entryPath) mounted = true; + } + return mounted; +} + +export async function assertIdentityIsNotMountPoint(path: string): Promise { + // Resolve the parent, not the leaf: rename replaces a leaf symlink itself. + const entryPath = join(await realpath(dirname(path)), basename(path)); + let content: string; + try { + const handle = await open('/proc/self/mountinfo', 'r'); + try { + const buffer = Buffer.alloc(MOUNTINFO_MAX_BYTES + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, null); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > MOUNTINFO_MAX_BYTES) throw new Error('mount table exceeds limit'); + content = buffer.toString('utf8', 0, length); + } finally { + await handle.close(); + } + } catch { + throw new BridgeProtocolError( + 'Cannot verify identity mount status: /proc/self/mountinfo must be readable ' + + 'and no larger than 4 MiB. Use an environment with procfs available.', + ); + } + if (identityIsMountPoint(content, entryPath)) { + throw new BridgeProtocolError( + `Bridge identity path ${path} is a mount point and cannot be atomically replaced. ` + + 'Mount its containing directory instead, or choose an unmounted identity file.', + ); + } +} diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index c4dab0bd..bb0c3ba4 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -14,6 +14,8 @@ import { dirname, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; import { assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; +import { assertIdentityIsNotMountPoint } from './identity-mount.js'; + import type { PairedBridgeWorkerIdentity } from './pairing.js'; function isRecord(value: unknown): value is Record { @@ -287,7 +289,10 @@ async function assertIdentityDestinationIsReplaceable( try { metadata = await lstat(path); } catch (error) { - if (isMissingPathError(error)) return; + if (isMissingPathError(error)) { + await assertIdentityIsNotMountPoint(path); + return; + } throw error; } if (metadata.isDirectory()) { @@ -295,6 +300,7 @@ async function assertIdentityDestinationIsReplaceable( `Bridge identity path ${path} is a directory. Point --identity at a file.`, ); } + await assertIdentityIsNotMountPoint(path); const uid = process.platform === 'win32' ? undefined : process.getuid?.(); if (uid === undefined || uid === 0 || metadata.uid === uid) return; /* Ownership only blocks `rename` under the sticky bit, and owning the @@ -412,6 +418,7 @@ export async function saveBridgeIdentity( await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); await assertWriteContainerPrivate(path); + await assertIdentityDestinationIsReplaceable(path); const temporaryPath = `${path}.${randomBytes(8).toString('hex')}.tmp`; try { const file = await open(temporaryPath, 'wx', 0o600); From 2163130ba968c1bf1a0ff53e40d4f8b2725d4084 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:08:57 -0400 Subject: [PATCH 054/116] fix(codeapi): continue bounded listings after skipped candidate windows (#144) Source: ClickHouse/ai@472c19ab1a6e4cccb185a1716b5015f5be6358d6 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: danny-avila <110412045+danny-avila@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45c85202..37ecc984 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Thanks for your interest in Code Interpreter! This repository is published from an internal ClickHouse monorepo, which is the source of truth. Internal changes that are not already public are mirrored here as a snapshot commit on the `sync/main` branch (spot them by the -`Source: ClickHouse/ai@` trailer); a maintainer merges the resulting sync +`Source: ClickHouse/ai@` trailer); a maintainer merges the resulting sync pull request to release it to `main`. Practical consequences: From f300263a13bb69cbfe2754f1fd8e986ee5c1ab5e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 21:42:26 -0400 Subject: [PATCH 055/116] fix: restore macOS private storage with native ACL verification (#146) * fix: restore macOS private storage with native ACL verification * fix: reject inheritable ACL grants before storage creation --- .github/workflows/ci.yml | 16 ++ packages/code/README.md | 26 +- packages/code/package-lock.json | 321 +++++++++++++++++++++- packages/code/package.json | 3 +- packages/code/src/github.ts | 3 +- packages/code/src/identity-mount.ts | 15 +- packages/code/src/macos-storage.test.ts | 166 +++++++++++ packages/code/src/macos-storage.ts | 94 +++++++ packages/code/src/private-storage.test.ts | 4 +- packages/code/src/private-storage.ts | 51 +++- packages/code/src/storage.ts | 87 +++--- 11 files changed, 720 insertions(+), 66 deletions(-) create mode 100644 packages/code/src/macos-storage.test.ts create mode 100644 packages/code/src/macos-storage.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b25285b..a1a4b6e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,22 @@ jobs: - name: Tests run: npm test + macos-storage-tests: + name: macOS Storage ACL Tests + runs-on: macos-14 + defaults: + run: + working-directory: packages/code + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 24.16.0 + - run: npm ci + - run: npm run build + - name: Native ACL and credential lifecycle tests + run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js + lambda-microvm-provisioning: name: Lambda MicroVM Provisioning runs-on: ubuntu-latest diff --git a/packages/code/README.md b/packages/code/README.md index 43b681a0..84adfc91 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -35,13 +35,21 @@ LIBRECHAT_CODE_SANDBOX_ENDPOINT=http://127.0.0.1:2000/api/v2 \ librechat-code run ``` -Credential and quarantine storage currently requires Linux (including WSL2), -where ownership and POSIX mode/ACL-mask checks can establish owner-only access. -Native Windows and macOS fail closed before pairing-code redemption, credential -loads, or storage mutations because their extended ACLs cannot yet be verified. -Use storage on a native Linux filesystem, not a Windows drive under `/mnt`. -Support for these platforms requires a native ACL verifier; `chmod` alone is -not sufficient. +Credential and quarantine storage supports macOS and Linux (including WSL2). +On macOS, native descriptor-based ACL calls remove inherited ACLs from new +credential/state files before writing secrets and verify the result. Reads reject +ACL-exposed identities and GitHub App keys; ancestor checks reject ACL write +grants and inheritable allow entries before any child is created. Removing an +ACL after creation cannot revoke descriptors opened while the grant existed. Existing sharing ACLs on parent directories +are never silently removed. Default application-owned workspace directories have +their ACLs removed and modes restricted to `0700`. + +macOS requires the packaged Koffi native dependency (prebuilt for Apple Silicon +and Intel); no Python interpreter or local compiler is needed with those builds. +If it cannot load or ACL inspection fails, storage fails closed before pairing. +Native Windows remains explicitly unsupported until DACL removal and verification +are implemented. Use WSL2 with storage on a native Linux filesystem, not a Windows +drive under `/mnt`. Linux retains ownership and POSIX mode/ACL-mask checks. Every storage ancestor, including intermediate symlink entries and targets, must be owned by this account or root and must not allow group/other writes unless @@ -132,8 +140,8 @@ read only by the trusted worker, which mints and refreshes short-lived installation tokens. A personal access token is supported as a fallback with `LIBRECHAT_CODE_GITHUB_TOKEN`, but the GitHub App is the safer default because its repository access and permissions can be narrowly installed and revoked. -Native Windows and macOS credential storage are unavailable until native ACL -verification is implemented; use Linux or WSL2. This also applies to GitHub App +Native Windows credential storage is unavailable until native DACL removal and +verification are implemented; use macOS, Linux, or WSL2. This also applies to GitHub App private keys. Git receives authentication through process-scoped `GIT_CONFIG_*` variables. diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index a2da04a0..ac9affc8 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -9,7 +9,8 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.75" + "@anthropic-ai/sandbox-runtime": "0.0.75", + "koffi": "3.2.1" }, "bin": { "librechat-code": "dist/cli.js" @@ -40,6 +41,294 @@ "node": ">=20.11.0" } }, + "node_modules/@koromix/koffi-android-arm64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-arm64/-/koffi-android-arm64-3.2.1.tgz", + "integrity": "sha512-1pJQ4jnZlUJduK9u9DC5CGy3aOgDUPvIXpNb6syV3+Dh5Q/ugezAIGCqvY+w+1mgXsve0pd0NVvJRjdZNHQ6MA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-android-x64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-x64/-/koffi-android-x64-3.2.1.tgz", + "integrity": "sha512-HH40xGh3gVQifjOBnhwT2tECC0lL1lYe+nxHvWNSzxDIyQNcVPXg38ta7vuONRFpD+uIrw7fqGYLzbZIagkVcg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-arm64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-arm64/-/koffi-darwin-arm64-3.2.1.tgz", + "integrity": "sha512-Vj4h+xcjc5+Cn0DhPHjgRX4omKAv96Kehtcd+1YgYuY2W7FvQn9vS+3SmzVwhC5Qmg9bIwUZObYQ8T/4hBqQqA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-x64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-x64/-/koffi-darwin-x64-3.2.1.tgz", + "integrity": "sha512-gFCWxNBTZIvxo1p+PURWfsy2Ctj5FGnVVs1f03lTLhBvmxEto70pdIiFztdFLDFkAJ1pmtQmruRKapeK+E8YPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-arm64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-arm64/-/koffi-freebsd-arm64-3.2.1.tgz", + "integrity": "sha512-qj+f1s2e6vULaUG1cdlTcCXmunCq2t+rjxku1+esaMIqVnHpOwj0QzPuInG0AFdXjwBNQhyVR/HpDj8daEwwsQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-ia32": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-ia32/-/koffi-freebsd-ia32-3.2.1.tgz", + "integrity": "sha512-6olHb1Qfgai0jjs6ddlDDD0ZfsCxy7SPi8rMRpuYQWH0qhgtyQu82hw5b1p7z+TJ0zZP3ZeQQ6l+U/MlM1ICHQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-x64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-x64/-/koffi-freebsd-x64-3.2.1.tgz", + "integrity": "sha512-Dikhw1ySYNVMkmeFvFVjnU5Wdk6mffNoOjJxm9bTG96vg7OlemylxqdEven47R1YJ3yzNVJn/MlQ207ORWfi2w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm/-/koffi-linux-arm-3.2.1.tgz", + "integrity": "sha512-OfwUwZylidq95wQKp6ClInULrfB2giu7dqM6Rhe0zAe6lES5I2SXNw15T9+GnRHk3/9hKT2XZ37OZLaKSyWNLA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm64/-/koffi-linux-arm64-3.2.1.tgz", + "integrity": "sha512-K+cGUL5iBcDqxmsocrjmlASqDf24gc7artbVW3PewG2c9AqwC63lezgwvB85Nx4lZAQjB6zIFHh9A7t1yGbwhw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ia32": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ia32/-/koffi-linux-ia32-3.2.1.tgz", + "integrity": "sha512-rxj6UYjU1qd98gxNQOSCdLpc5cPRi5Giq9rNd3jnGuSNIyMkwa6Dxw4cUjmhIBCYESMJtmNt5NWnJp5u9wTfYQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-loong64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-loong64/-/koffi-linux-loong64-3.2.1.tgz", + "integrity": "sha512-aHhnHzkPRmT/IHDlGvESJ/Bs32m8N6UE6Ab6kMeJzgk74IN8af2m/81/wZJtybR1M2UxCV4NmlVNUYQQvSAO3Q==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-riscv64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-riscv64/-/koffi-linux-riscv64-3.2.1.tgz", + "integrity": "sha512-qtQBsjbm3LiirLJvajWmKkNb7ARk7fvJVXdftJ7NtAnF3Xw8EbDvrtvmvtNI1yLPlYcBmlzCCD71hwhWYk0SIA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-x64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-x64/-/koffi-linux-x64-3.2.1.tgz", + "integrity": "sha512-c7hw7Qs/r5gnFRTQLcbifBwRU7wiocj+2pVuDQ5Ahb3r36SZmupmgYbTWcLvTW+hul1jd7SKRV0d14ZJq/tvSw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-ia32": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-ia32/-/koffi-openbsd-ia32-3.2.1.tgz", + "integrity": "sha512-mmY8fY8LQ/CB52+h3yrMYmVyoxzW3x08S0yI6VNfHWdfU6yJtZkKCbhjmQCYMrWbYKC4gMvwZwCywIGPAkLyeA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-x64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-x64/-/koffi-openbsd-x64-3.2.1.tgz", + "integrity": "sha512-k4ig6aAPbFSRATOIIOfdf/KtlOGH4SVls6L9fy0QnTxRJYvY2oSltTsQtBDANgEQldlq8Kl5WnpRa1VSibP4Lw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-arm64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-arm64/-/koffi-win32-arm64-3.2.1.tgz", + "integrity": "sha512-cTWBJGK//pDMeKQJE/79Aq9MiOAF4H8QyLZHSQ9IWm8czOfwjG4J1AhsQ9DjI9KFOykH77hhnpmQTGVMIubGig==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-ia32": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-ia32/-/koffi-win32-ia32-3.2.1.tgz", + "integrity": "sha512-Z50EM6TAZ7CFyMmyX6thv8eNpJchqe9eenhibSIy2Eq/FQYF76gU2VK/LEoaF46L8hfC7TpTp9b10MvReHEyFA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-x64": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-x64/-/koffi-win32-x64-3.2.1.tgz", + "integrity": "sha512-ZmZNiBO6bkOSh3QNzgfvb1cMY0yMobn6ZQrSMqbAce21qyYL8niIbyipz9N/PIRDciGhsV0wUxnZsxIO+yWsHQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, "node_modules/@pondwader/socks5-server": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", @@ -65,6 +354,36 @@ "node": ">=18" } }, + "node_modules/koffi": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-3.2.1.tgz", + "integrity": "sha512-0qE3lZ8jllRqPN4Ob6Ajl7c2bJSJDhQWuKLGP5hIEpHLllJWv1ydHFMhHmHc5p/W9GticKVDbYzZd7TBoQ4CZg==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "url": "https://liberapay.com/Koromix" + }, + "optionalDependencies": { + "@koromix/koffi-android-arm64": "3.2.1", + "@koromix/koffi-android-x64": "3.2.1", + "@koromix/koffi-darwin-arm64": "3.2.1", + "@koromix/koffi-darwin-x64": "3.2.1", + "@koromix/koffi-freebsd-arm64": "3.2.1", + "@koromix/koffi-freebsd-ia32": "3.2.1", + "@koromix/koffi-freebsd-x64": "3.2.1", + "@koromix/koffi-linux-arm": "3.2.1", + "@koromix/koffi-linux-arm64": "3.2.1", + "@koromix/koffi-linux-ia32": "3.2.1", + "@koromix/koffi-linux-loong64": "3.2.1", + "@koromix/koffi-linux-riscv64": "3.2.1", + "@koromix/koffi-linux-x64": "3.2.1", + "@koromix/koffi-openbsd-ia32": "3.2.1", + "@koromix/koffi-openbsd-x64": "3.2.1", + "@koromix/koffi-win32-arm64": "3.2.1", + "@koromix/koffi-win32-ia32": "3.2.1", + "@koromix/koffi-win32-x64": "3.2.1" + } + }, "node_modules/node-forge": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.4.0.tgz", diff --git a/packages/code/package.json b/packages/code/package.json index c9f810b7..a819af9d 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -60,6 +60,7 @@ "node": ">=20.11" }, "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.75" + "@anthropic-ai/sandbox-runtime": "0.0.75", + "koffi": "3.2.1" } } diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 7bacc297..e71188c8 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -2,7 +2,7 @@ import { constants } from 'node:fs'; import { createHash, createPrivateKey, sign } from 'node:crypto'; import { open } from 'node:fs/promises'; import { dirname } from 'node:path'; -import { assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; +import { assertPrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; export const GITHUB_ALLOWED_DOMAINS = [ @@ -69,6 +69,7 @@ async function readPrivateKey(path: string): Promise { 'GitHub App private key must not be accessible by group or other users', ); } + await assertPrivateStorageAcl(handle, path); return await handle.readFile('utf8'); } finally { await handle.close(); diff --git a/packages/code/src/identity-mount.ts b/packages/code/src/identity-mount.ts index ba87d6d6..ea4cc7f4 100644 --- a/packages/code/src/identity-mount.ts +++ b/packages/code/src/identity-mount.ts @@ -1,4 +1,4 @@ -import { open, realpath } from 'node:fs/promises'; +import { lstat, open, realpath } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; import { BridgeProtocolError } from './protocol.js'; @@ -30,6 +30,19 @@ export function identityIsMountPoint(mountinfo: string, entryPath: string): bool export async function assertIdentityIsNotMountPoint(path: string): Promise { // Resolve the parent, not the leaf: rename replaces a leaf symlink itself. const entryPath = join(await realpath(dirname(path)), basename(path)); + if (process.platform === 'darwin') { + const metadata = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return undefined; + throw error; + }); + // A missing entry cannot be mounted; rename replaces a symlink itself. + if (metadata === undefined || metadata.isSymbolicLink()) return; + const { macOsMountPoint } = await import('./macos-storage.js'); + if (await realpath(macOsMountPoint(entryPath)) === entryPath) { + throw new BridgeProtocolError(`Bridge identity path ${path} is a mount point and cannot be atomically replaced.`); + } + return; + } let content: string; try { const handle = await open('/proc/self/mountinfo', 'r'); diff --git a/packages/code/src/macos-storage.test.ts b/packages/code/src/macos-storage.test.ts new file mode 100644 index 00000000..afbcdf77 --- /dev/null +++ b/packages/code/src/macos-storage.test.ts @@ -0,0 +1,166 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import fs, { chmod, mkdtemp, mkdir, open, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { assertPrivateStorageAcl, assertPrivateStorageAncestors, removePrivateStorageAcl } from './private-storage.js'; +import { assertIdentityPathIsPrivate, saveBridgeIdentity, loadBridgeIdentity, + saveWorkspaceMutationQuarantine, loadWorkspaceMutationQuarantine, + clearWorkspaceMutationQuarantine, ensurePrivateWorkspaceDirectory } from './storage.js'; +import { GitHubAppCredentialProvider } from './github.js'; + +const exec = promisify(execFile); +const mac = { skip: process.platform !== 'darwin' }; +const identity = { protocolVersion: 1 as const, workerId: 'worker', + codeApiUrl: 'https://example.com', credential: 'secret', privateKey: 'private', + publicKey: 'public', expiresAt: '2099-01-01T00:00:00Z' }; +const quarantine = { version: 1 as const, workerId: 'worker', workspaceId: 'primary', + ownerId: 'owner', reason: 'uncertain', quarantinedAt: '2026-01-01T00:00:00Z' }; + +async function grant(path: string, permissions: string): Promise { + await exec('/bin/chmod', ['+a', `everyone allow ${permissions}`, path]); +} + +// Real kernel ACLs: chmod(0600) alone leaves these grants effective. +test('macOS removes inherited deny ACLs before publishing identity and quarantine state', mac, async t => { + const root = await mkdtemp(join(tmpdir(), 'macos-acl-')); + t.after(() => rm(root, { recursive: true, force: true })); + await exec('/bin/chmod', ['+a', 'everyone deny read,file_inherit,directory_inherit,only_inherit', root]); + let guardedWrites = 0; + const originalOpen = fs.open; + fs.open = (async (...args: Parameters) => { + const handle = await originalOpen(...args); + if (args[1] === 'wx') { + const write = handle.writeFile.bind(handle); + handle.writeFile = async (...values) => { + await assertPrivateStorageAcl(handle, String(args[0])); + guardedWrites += 1; + return write(...values); + }; + } + return handle; + }) as typeof fs.open; + syncBuiltinESMExports(); + t.after(() => { fs.open = originalOpen; syncBuiltinESMExports(); }); + const path = join(root, 'identity.json'); + const reservation = await assertIdentityPathIsPrivate(path); + assert.equal(await readFile(path, 'utf8'), ''); + await saveBridgeIdentity(path, identity); + await reservation.release(); + assert.deepEqual(await loadBridgeIdentity(path), identity); + const marker = join(root, 'quarantine.json'); + await saveWorkspaceMutationQuarantine(marker, quarantine); + assert.deepEqual(await loadWorkspaceMutationQuarantine(marker), quarantine); + for (const file of [path, marker]) { + const listing = await exec('/bin/ls', ['-lde', file]); + assert.doesNotMatch(listing.stdout, /\n\s*0:/); + } + // Re-pairing exercises native mount verification and sibling publication. + await (await assertIdentityPathIsPrivate(path)).release(); + await clearWorkspaceMutationQuarantine(marker, 'owner'); + assert.equal(guardedWrites, 2); + const workspace = join(root, 'workspace'); + await mkdir(workspace); + await grant(workspace, 'read,write,delete'); + await ensurePrivateWorkspaceDirectory(workspace); + assert.doesNotMatch((await exec('/bin/ls', ['-lde', workspace])).stdout, /\n\s*0:/); +}); + +test('macOS refuses exposed 0600 identities, quarantine markers, and GitHub keys', mac, async t => { + const root = await mkdtemp(join(tmpdir(), 'macos-acl-')); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'identity.json'); + const marker = join(root, 'quarantine.json'); + await saveBridgeIdentity(path, identity); + await saveWorkspaceMutationQuarantine(marker, quarantine); + for (const file of [path, marker]) { + await grant(file, 'read'); + await chmod(file, 0o600); + } + await assert.rejects(loadBridgeIdentity(path), /macOS ACL grants/); + await assert.rejects(assertIdentityPathIsPrivate(path), /macOS ACL grants/); + await assert.rejects(loadWorkspaceMutationQuarantine(marker), /macOS ACL grants/); + let fetched = false; + const provider = new GitHubAppCredentialProvider({ + appId: '1', installationId: '1', privateKeyPath: path, + fetch: async () => { fetched = true; throw new Error('unexpected fetch'); }, + }); + await assert.rejects(provider.getCredential(), /macOS ACL grants/); + assert.equal(fetched, false); + assert.equal(await readFile(path, 'utf8'), `${JSON.stringify(identity, null, 2)}\n`); +}); + +test('macOS rejects ACL-writable ancestors and accepts deny-only home-style ACLs', mac, async t => { + const root = await mkdtemp(join(tmpdir(), 'macos-acl-')); + t.after(async () => { + await exec('/bin/chmod', ['-N', root]); + await rm(root, { recursive: true, force: true }); + }); + await exec('/bin/chmod', ['+a', 'everyone deny delete', root]); + await assertPrivateStorageAncestors(root); + await grant(root, 'add_file,delete_child'); + await chmod(root, 0o700); + await assert.rejects(assertIdentityPathIsPrivate(join(root, 'identity.json')), /macOS ACL grants/); +}); + +test('macOS ACL checks and removal operate on the held inode after path replacement', mac, async t => { + const root = await mkdtemp(join(tmpdir(), 'macos-acl-')); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'file'); + await writeFile(path, '', { mode: 0o600 }); + await grant(path, 'read'); + const handle = await open(path, 'r'); + try { + await rename(path, join(root, 'old')); + await writeFile(path, '', { mode: 0o600 }); + await assert.rejects(assertPrivateStorageAcl(handle, path), /macOS ACL grants/); + await removePrivateStorageAcl(handle, path); + await assertPrivateStorageAcl(handle, path); + } finally { + await handle.close(); + } + await assert.rejects(assertPrivateStorageAcl(handle, path), /Cannot verify macOS/); + await assert.rejects(removePrivateStorageAcl(handle, path), /Cannot verify macOS/); +}); + + +test('macOS checks mount status without procfs', mac, async () => { + const { macOsMountPoint } = await import('./macos-storage.js'); + assert.equal(macOsMountPoint('/'), '/'); + assert.throws(() => macOsMountPoint('/nonexistent-macos-acl-test'), /Cannot verify macOS identity mount/); +}); + + +test('macOS rejects inheritable allow ACLs before creating any storage inode', mac, async t => { + for (const inheritance of ['file_inherit', 'directory_inherit', 'file_inherit,only_inherit']) { + await t.test(inheritance, async t => { + const root = await mkdtemp(join(tmpdir(), 'macos-inherited-acl-')); + t.after(() => rm(root, { recursive: true, force: true })); + const existing = join(root, 'existing.json'); + await saveBridgeIdentity(existing, identity); + await chmod(root, 0o755); + await grant(root, `read,search,${inheritance}`); + let creates = 0; + const originalOpen = fs.open; + fs.open = (async (...args: Parameters) => { + if (args[1] === 'wx') creates += 1; + return originalOpen(...args); + }) as typeof fs.open; + syncBuiltinESMExports(); + t.after(() => { fs.open = originalOpen; syncBuiltinESMExports(); }); + const path = join(root, 'nested', 'identity.json'); + for (const action of [ + () => assertIdentityPathIsPrivate(path), + () => assertIdentityPathIsPrivate(existing), + () => saveBridgeIdentity(path, identity), + () => saveWorkspaceMutationQuarantine(path, quarantine), + () => ensurePrivateWorkspaceDirectory(join(root, 'workspace')), + ]) await assert.rejects(action(), /macOS ACL grants/); + assert.equal(creates, 0); + assert.deepEqual(await readdir(root), ['existing.json']); + }); + } +}); diff --git a/packages/code/src/macos-storage.ts b/packages/code/src/macos-storage.ts new file mode 100644 index 00000000..e75261d9 --- /dev/null +++ b/packages/code/src/macos-storage.ts @@ -0,0 +1,94 @@ +import koffi from 'koffi'; +import { BridgeProtocolError } from './protocol.js'; + +// Darwin sys/acl.h. Use the held descriptor, never /dev/fd path metadata. +const ACL_TYPE_EXTENDED = 0x100; +const ACL_EXTENDED_ALLOW = 1; +const ACL_EXTENDED_DENY = 2; +const ACL_NEXT_ENTRY = -1; +const ACL_ENTRY_FILE_INHERIT = 1 << 5; +const ACL_ENTRY_DIRECTORY_INHERIT = 1 << 6; +const lib = koffi.load('/usr/lib/libSystem.B.dylib'); +const getAcl = lib.func('void *acl_get_fd_np(int fd, int type)'); +const setAcl = lib.func('int acl_set_fd_np(int fd, void *acl, int type)'); +const initAcl = lib.func('void *acl_init(int count)'); +const freeAcl = lib.func('int acl_free(void *acl)'); +const getEntry = lib.func('int acl_get_entry(void *acl, int index, _Out_ void **entry)'); +const getTag = lib.func('int acl_get_tag_type(void *entry, _Out_ int *tag)'); +const getMask = lib.func('int acl_get_permset_mask_np(void *entry, _Out_ uint64_t *mask)'); +const getFlags = lib.func('int acl_get_flagset_np(void *entry, _Out_ void **flags)'); +const getFlag = lib.func('int acl_get_flag_np(void *flags, int flag)'); +// Only non-inheritable read/list, search/execute, and metadata reads are safe. +// Removing an inherited grant after open cannot revoke an attacker's held fd. +const ANCESTOR_READ_PERMISSIONS = (1 << 1) | (1 << 3) | (1 << 7) | (1 << 9) | (1 << 11); + +function unavailable(): never { + throw new BridgeProtocolError('Cannot verify macOS storage ACLs on the opened object; use a local filesystem with ACL support.'); +} + +export function verifyMacOsAcl(fd: number, path: string, directory = false, empty = false): void { + const acl = getAcl(fd, ACL_TYPE_EXTENDED); + if (acl == null) { + // On a valid descriptor Darwin reports ENOENT when no extended ACL exists. + if (koffi.errno() === koffi.os.errno.ENOENT) return; + unavailable(); + } + try { + for (let index = 0; ; index = ACL_NEXT_ENTRY) { + const entry = [null]; + if (getEntry(acl, index, entry) !== 0) { + // Darwin uses EINVAL at end of the ACL (unlike Linux's zero result). + if (koffi.errno() === koffi.os.errno.EINVAL) return; + unavailable(); + } + const tag = [0]; + const mask = [0]; + const flags = [null]; + if (getTag(entry[0], tag) !== 0 || getMask(entry[0], mask) !== 0 || + getFlags(entry[0], flags) !== 0) unavailable(); + const fileInherit = getFlag(flags[0], ACL_ENTRY_FILE_INHERIT); + const directoryInherit = getFlag(flags[0], ACL_ENTRY_DIRECTORY_INHERIT); + if (fileInherit < 0 || directoryInherit < 0) unavailable(); + if (empty || (tag[0] !== ACL_EXTENDED_DENY && + (tag[0] !== ACL_EXTENDED_ALLOW || !directory || fileInherit || directoryInherit || + (BigInt(mask[0]) & ~BigInt(ANCESTOR_READ_PERMISSIONS)) !== 0n))) { + throw new BridgeProtocolError( + `macOS ACL grants access beyond owner-only storage at ${path}. ` + + 'Remove the sharing ACL before using this path. If a credential was exposed, revoke it and pair again.', + ); + } + } + } finally { + freeAcl(acl); + } +} + +export function removeMacOsAcl(fd: number, path: string): void { + const acl = initAcl(0); + if (acl == null) unavailable(); + try { + if (setAcl(fd, acl, ACL_TYPE_EXTENDED) !== 0) unavailable(); + } finally { + freeAcl(acl); + } + verifyMacOsAcl(fd, path, false, true); +} + +// Darwin's statfs64 layout is identical on arm64 and x86_64 (sys/mount.h). +const Statfs = koffi.struct({ + bsize: 'uint32_t', iosize: 'int32_t', + blocks: 'uint64_t', bfree: 'uint64_t', bavail: 'uint64_t', + files: 'uint64_t', ffree: 'uint64_t', fsid: 'int32_t[2]', + owner: 'uint32_t', type: 'uint32_t', flags: 'uint32_t', subtype: 'uint32_t', + typename: 'char[16]', mountpoint: 'char[1024]', source: 'char[1024]', + flagsExt: 'uint32_t', reserved: 'uint32_t[7]', +}); +const statfs = lib.func('statfs64', 'int', ['str', koffi.out(koffi.pointer(Statfs))]); + +export function macOsMountPoint(path: string): string { + const result: { mountpoint?: string } = {}; + if (statfs(path, result) !== 0 || !result.mountpoint?.startsWith('/')) { + throw new BridgeProtocolError('Cannot verify macOS identity mount status.'); + } + return result.mountpoint; +} diff --git a/packages/code/src/private-storage.test.ts b/packages/code/src/private-storage.test.ts index c43772ef..c4b7e760 100644 --- a/packages/code/src/private-storage.test.ts +++ b/packages/code/src/private-storage.test.ts @@ -15,7 +15,7 @@ const identity = { protocolVersion: 1 as const, workerId: 'worker', const quarantine = { version: 1 as const, workerId: 'worker', workspaceId: 'primary', reason: 'uncertain mutation', quarantinedAt: '2026-01-01T00:00:00Z' }; -for (const unsupported of ['win32', 'darwin', 'freebsd']) { +for (const unsupported of ['win32', 'freebsd']) { test(`${unsupported} refuses storage before creation, reads, or deletion`, async t => { const root = await mkdtemp(join(tmpdir(), 'private-storage-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -39,7 +39,7 @@ for (const unsupported of ['win32', 'darwin', 'freebsd']) { test('GitHub App credentials also reject unsupported ACL verification before signing or fetching', async t => { const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; t.after(() => Object.defineProperty(process, 'platform', platform)); - Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + Object.defineProperty(process, 'platform', { ...platform, value: 'freebsd' }); let fetched = false; const provider = new GitHubAppCredentialProvider({ appId: '1', installationId: '1', privateKeyPath: '/must-not-be-read', diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts index ea9952a7..af4f185a 100644 --- a/packages/code/src/private-storage.ts +++ b/packages/code/src/private-storage.ts @@ -1,23 +1,44 @@ -import { lstat, readlink } from 'node:fs/promises'; +import { lstat, open, readlink } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import type { FileHandle } from 'node:fs/promises'; import { dirname, isAbsolute } from 'node:path'; import { BridgeProtocolError } from './protocol.js'; -/** Linux POSIX ACL masks are reflected in group mode bits. macOS extended - * ACLs and Windows DACLs are not: chmod/stat alone cannot establish privacy. - * Fail before creating files, reading credentials, or redeeming pairing codes - * until a native verifier can inspect the actual opened object's ACLs. - */ +/** Linux exposes POSIX ACL masks in mode bits; macOS needs native ACL calls. */ export function assertPrivateStorageSupported(): void { - if (process.platform !== 'linux' || process.getuid === undefined) { + if (!['linux', 'darwin'].includes(process.platform) || process.getuid === undefined) { throw new BridgeProtocolError( 'Owner-only storage ACL verification is unavailable on this platform. ' + - 'Worker credentials, GitHub App keys, and quarantine state require Linux ' + - '(including WSL2) with storage on a native Linux filesystem, not /mnt.', + 'Native Windows is unsupported until DACL removal and verification are implemented. ' + + 'Use macOS or Linux (including WSL2 with a native Linux filesystem, not /mnt).', ); } } +async function macOsStorage() { + try { + return await import('./macos-storage.js'); + } catch { + throw new BridgeProtocolError('macOS ACL verification is unavailable: reinstall @librechat/code with its Koffi native dependency.'); + } +} + +export async function assertPrivateStorageAcl( + handle: FileHandle, path: string, directory = false, +): Promise { + if (process.platform === 'darwin') { + (await macOsStorage()).verifyMacOsAcl(handle.fd, path, directory); + } +} + +/** Only application-owned files/directories may have their ACLs removed. */ +export async function removePrivateStorageAcl(handle: FileHandle, path: string): Promise { + if (process.platform === 'darwin') { + (await macOsStorage()).removeMacOsAcl(handle.fd, path); + } +} + /** * Walk from the trust root before touching a descendant. Checking only a * canonical parent misses replaceable ancestors and symlink entries. Resolve @@ -54,6 +75,18 @@ export async function assertPrivateStorageAncestors( continue; } if (metadata.isDirectory()) { + if (process.platform === 'darwin') { + const handle = await open(current, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const opened = await handle.stat(); + if (opened.dev !== metadata.dev || opened.ino !== metadata.ino) { + throw new BridgeProtocolError(`Storage directory changed during ACL verification: ${current}`); + } + await assertPrivateStorageAcl(handle, current, true); + } finally { + await handle.close(); + } + } const mode = metadata.mode & 0o7777; if ((mode & 0o022) !== 0 && (mode & 0o1000) === 0) { throw new BridgeProtocolError( diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index bb0c3ba4..af2f1295 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -8,11 +8,12 @@ import { rm, stat, } from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; -import { assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; +import { assertPrivateStorageAcl, removePrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; import { assertIdentityIsNotMountPoint } from './identity-mount.js'; @@ -160,28 +161,6 @@ export function defaultWorkspaceQuarantinePath( ); } -/** - * Verify a path really is owner-only. `chmod` reports success without effect on - * mounts that do not implement POSIX permissions - notably WSL2 DrvFs - * (`/mnt/`), where the result stays world-accessible - so a credential - * that cannot be protected must fail closed rather than appear protected. - * - * Symlinks are resolved: a link's own mode is always `0777` and ignored by the - * kernel, so the file the bytes live in is what counts. - * - * Linux POSIX ACL masks are reflected in group mode bits. Platforms whose - * ACLs cannot be verified are rejected at the storage entry points. - */ -async function groupOrOtherAccessMode( - path: string, -): Promise { - if (process.platform === 'win32') return undefined; - /* Resolve symlinks: the bytes live at the target, and a link's own mode is - * always 0777 and ignored by the kernel. */ - const mode = (await stat(path)).mode & 0o777; - return (mode & 0o077) === 0 ? undefined : mode; -} - /** Publishing replaces the entry itself; reading also follows its target. */ async function assertWriteContainerPrivate(path: string): Promise { await assertPrivateStorageAncestors(dirname(path), true); @@ -238,24 +217,31 @@ async function readGuardedFile( const mode = stats.mode & 0o777; if ((mode & 0o077) !== 0) throw new BridgeProtocolError(exposed(mode.toString(8))); } + await assertPrivateStorageAcl(handle, path); return await handle.readFile('utf8'); } finally { await handle.close(); } } -async function assertOwnerOnlyPath( - path: string, - reportedPath: string = path, -): Promise { - const mode = await groupOrOtherAccessMode(path); - if (mode === undefined) return; - throw new BridgeProtocolError( - `Cannot restrict ${reportedPath} to owner-only access (mode ${mode.toString(8)}). ` + - 'Filesystems that ignore POSIX permissions, such as Windows drives mounted ' + - 'under /mnt, cannot protect worker credentials or workspaces. Use a path on a ' + - 'native Linux filesystem.', - ); +async function assertOwnerOnlyFile(handle: FileHandle, path: string): Promise { + const mode = (await handle.stat()).mode & 0o777; + if ((mode & 0o077) !== 0) { + throw new BridgeProtocolError( + `Cannot restrict ${path} to owner-only access (mode ${mode.toString(8)}). ` + + 'Use a native macOS or Linux filesystem that enforces permissions, not a Windows drive under /mnt.', + ); + } + await assertPrivateStorageAcl(handle, path); +} + +async function assertOwnerOnlyPath(path: string): Promise { + const handle = await open(path, 'r'); + try { + await assertOwnerOnlyFile(handle, path); + } finally { + await handle.close(); + } } export async function ensurePrivateWorkspaceDirectory( @@ -272,7 +258,14 @@ export async function ensurePrivateWorkspaceDirectory( /* This directory is application-owned by contract; a pre-existing one under * another account lets that owner alter workspace inputs and results. */ await assertOwnedByWorker(path); - await chmod(path, 0o700); + const directory = await open(path, 'r'); + try { + await removePrivateStorageAcl(directory, path); + await directory.chmod(0o700); + await assertPrivateStorageAcl(directory, path); + } finally { + await directory.close(); + } await assertOwnerOnlyPath(path); } @@ -333,7 +326,14 @@ export interface IdentityPathReservation { async function assertSiblingPublishable(path: string): Promise { const probePath = `${path}.${randomBytes(8).toString('hex')}.probe`; try { - await (await open(probePath, 'wx', 0o600)).close(); + const probe = await open(probePath, 'wx', 0o600); + try { + await removePrivateStorageAcl(probe, path); + await probe.chmod(0o600); + await assertOwnerOnlyFile(probe, path); + } finally { + await probe.close(); + } } catch (error) { throw new BridgeProtocolError( `Cannot create a temporary file beside ${path} (${ @@ -363,8 +363,9 @@ export async function assertIdentityPathIsPrivate( const reserved = await open(path, 'wx', 0o600); try { created = true; + await removePrivateStorageAcl(reserved, path); await reserved.chmod(0o600); - await assertOwnerOnlyPath(path); + await assertOwnerOnlyFile(reserved, path); reservedInode = (await reserved.stat({ bigint: true })).ino; } finally { await reserved.close(); @@ -423,8 +424,9 @@ export async function saveBridgeIdentity( try { const file = await open(temporaryPath, 'wx', 0o600); try { + await removePrivateStorageAcl(file, path); await file.chmod(0o600); - await assertOwnerOnlyPath(temporaryPath, path); + await assertOwnerOnlyFile(file, path); await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); await file.sync(); } finally { @@ -465,8 +467,9 @@ export async function saveWorkspaceMutationQuarantine( const file = await open(path, 'wx', 0o600); try { try { + await removePrivateStorageAcl(file, path); await file.chmod(0o600); - await assertOwnerOnlyPath(path); + await assertOwnerOnlyFile(file, path); await assertWriteContainerPrivate(path); await file.writeFile(`${JSON.stringify(record, null, 2)}\n`, 'utf8'); await file.sync(); @@ -500,7 +503,7 @@ export async function loadWorkspaceMutationQuarantine( (mode) => `Workspace quarantine ${path} is accessible beyond its owner (mode ${mode}). ` + 'Another local account could clear or forge it. Keep worker state on a ' + - 'native Linux filesystem.', + 'native macOS or Linux filesystem.', ); await assertReadPathPrivate(path); } catch (error) { @@ -566,7 +569,7 @@ export async function loadBridgeIdentity( (mode) => `Bridge identity ${path} is accessible beyond its owner (mode ${mode}). ` + 'Treat its private key as compromised: revoke the worker and pair again with an ' + - 'identity path on a native Linux filesystem.', + 'identity path on a native macOS or Linux filesystem.', ); /* After the file's own verdict, so an exposed mode keeps its diagnosis. */ await assertReadPathPrivate(path); From 1ef326ab2075a9a05a8b11183c4b812461f21ba0 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 21:49:42 -0400 Subject: [PATCH 056/116] fix: clamp runtime timeout caps at the sandbox limit (#148) * fix: clamp runtime timeout caps at the sandbox limit * test: use target-compatible timeout assertions * test: preserve timeout fixture tuple types --- api/README.md | 12 ++++++ api/src/api/v2-timeout.test.ts | 74 ++++++++++++++++++++++++++++++++++ api/src/api/v2.ts | 11 +++-- 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 api/src/api/v2-timeout.test.ts diff --git a/api/README.md b/api/README.md index e0e8a6d0..bbbe36ec 100644 --- a/api/README.md +++ b/api/README.md @@ -109,3 +109,15 @@ curl -s http://localhost:2000/api/v2/execute \ -H 'Content-Type: application/json' \ -d '{"language":"python","version":"3.14.4","files":[{"content":"print(42)"}]}' | jq ``` + +### Requested runtime caps + +`POST /api/v2/execute` treats `run_timeout` as an upper bound in milliseconds. +A request above the effective runtime limit is clamped to that limit, including +language and package overrides. Smaller caps are preserved, and omission uses +the runtime default. Compile, CPU, and memory constraints retain their existing +validation behavior. + +Roll out this sandbox behavior before enabling timeout forwarding in the +service's plain `/exec` handler. Older sandboxes reject caps above their local +runtime limit; older services remain compatible with updated sandboxes. diff --git a/api/src/api/v2-timeout.test.ts b/api/src/api/v2-timeout.test.ts new file mode 100644 index 00000000..463ed370 --- /dev/null +++ b/api/src/api/v2-timeout.test.ts @@ -0,0 +1,74 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test'; +import express from 'express'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import type { Server } from 'http'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { config } from '../config'; +import { Job } from '../job'; +import { loadPackage } from '../runtime'; +import router from './v2'; + +let server: Server; +let url: string; +let directory: string; +const language = 'runtime-timeout-cap-test'; +const originalPrime = Job.prototype.prime; +const originalExecute = Job.prototype.execute; +const originalCleanup = Job.prototype.cleanup; +const requireManifest = config.require_execution_manifest; +const observed: number[] = []; + +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'runtime-timeout-')); + await writeFile(join(directory, 'pkg-info.json'), JSON.stringify({ + language, version: '1.0.0', aliases: [], + limit_overrides: { run_timeout: 15000, compile_timeout: 5000 }, + })); + loadPackage(directory); + const app = express(); + app.use(router); + await new Promise((resolve) => { server = app.listen(0, '127.0.0.1', () => resolve()); }); + const address = server.address(); + url = `http://127.0.0.1:${typeof address === 'object' && address ? address.port : 0}/execute`; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); +}); +afterEach(() => { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + config.require_execution_manifest = requireManifest; + observed.length = 0; +}); + +test('caps execution at the effective language runtime limit without rejecting larger caller caps', async () => { + config.require_execution_manifest = false; + Job.prototype.prime = async function () { observed.push(this.timeouts.run); }; + Job.prototype.execute = async function () { return {} as Awaited>; }; + Job.prototype.cleanup = async function () {}; + for (const [input, expected] of [[25000, 15000], [15000, 15000], [1000, 1000], [null, 15000], [undefined, 15000]] as const) { + const response = await fetch(url, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ language, version: '1.0.0', run_timeout: input, files: [{ name: 'main.txt', content: 'test' }] }), + }); + expect(response.status, await response.text()).toBe(200); + expect(observed[observed.length - 1]).toBe(expected); + } +}); + +test('invalid runtime types and compile limit violations still fail before priming', async () => { + config.require_execution_manifest = false; + Job.prototype.prime = async function () { observed.push(this.timeouts.run); }; + for (const limits of [{ run_timeout: '1000' }, { run_timeout: -1 }, { compile_timeout: 6000 }]) { + const response = await fetch(url, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ language, version: '1.0.0', ...limits, files: [{ name: 'main.txt', content: 'test' }] }), + }); + expect(response.status).toBe(400); + } + expect(observed).toEqual([]); +}); diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index a883dc72..e2f252ce 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -253,7 +253,7 @@ function getJob( const { session_id, language, version, args, stdin, files, compile_memory_limit, run_memory_limit, - run_timeout, compile_timeout, + compile_timeout, run_cpu_time, compile_cpu_time, env_vars, } = body; @@ -288,7 +288,12 @@ function getJob( throw { message: 'files must include at least one runnable source file' }; } - validateConstraints(body, rt); + // A runtime timeout is a cap, not a request to exceed the runtime's own + // limit. Resolve it here, where language/package overrides are available. + const runTimeout = typeof body.run_timeout === 'number' && rt.timeouts.run > 0 + ? Math.min(body.run_timeout, rt.timeouts.run) + : body.run_timeout; + validateConstraints({ ...body, run_timeout: runTimeout }, rt); /* Session mode is per-request opt-in: only run in the persistent workspace * when THIS request carried a valid X-Runtime-Session-Id. A headerless or @@ -329,7 +334,7 @@ function getJob( stdin: stdin ?? '', files, timeouts: { - run: run_timeout ?? rt.timeouts.run, + run: runTimeout ?? rt.timeouts.run, compile: compile_timeout ?? rt.timeouts.compile, }, cpu_times: { From 394edafa560d1b7616fe002c28d4a40a9b034c7f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 21:51:00 -0400 Subject: [PATCH 057/116] fix: honor timeout caps on plain execution requests (#145) * fix: honor timeout caps on plain execution requests * docs: publish the execution timeout contract --- service/openapi.yml | 11 ++++ service/src/service/exec-timeout.test.ts | 75 ++++++++++++++++++++++++ service/src/service/router.ts | 11 +++- service/src/types/service.ts | 2 + 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 service/src/service/exec-timeout.test.ts diff --git a/service/openapi.yml b/service/openapi.yml index 78ba082f..a257fd18 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -138,6 +138,17 @@ components: - code - lang properties: + timeout: + type: number + nullable: true + minimum: 0 + exclusiveMinimum: true + description: >- + Optional runtime cap in milliseconds. Positive finite values are + rounded up to whole milliseconds and clamped to JOB_TIMEOUT, then + to the sandbox's effective language runtime limit. Omitted or null + values preserve the worker's runtime default. This caps execution, + not queueing or request transport time. code: type: string lang: diff --git a/service/src/service/exec-timeout.test.ts b/service/src/service/exec-timeout.test.ts new file mode 100644 index 00000000..70e00d17 --- /dev/null +++ b/service/src/service/exec-timeout.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from 'bun:test'; +import { resolve } from 'path'; + +test('/exec validates timeout before enqueue and forwards its cap to both language queues', async () => { + // Isolate infrastructure mocks; exercise the real router, timeout policy, + // payload builder, and security preparation without Redis or a sandbox. + const probe = Bun.spawn([process.execPath, '-e', ` + import { mock } from 'bun:test'; + import assert from 'node:assert/strict'; + const passthrough = (_req, _res, next) => next(); + mock.module('./src/middleware/auth', () => ({ sessionAuth: passthrough })); + mock.module('./src/middleware/limits', () => ({ + executionLimiter: passthrough, uploadLimiter: passthrough, + downloadLimiter: passthrough, fetchLimiter: passthrough, + })); + mock.module('./src/lifecycle', () => ({ + checkServiceStartUp: () => false, checkServiceShutDown: () => false, + })); + let writes = 0; + let submitted = []; + const queue = (name) => ({ add: async (_type, data) => { + submitted.push({ name, data }); + return { waitUntilFinished: async () => ({ ok: true }), remove: async () => {} }; + }}); + mock.module('./src/queue', () => ({ + pyQueue: queue('python'), otherQueue: queue('other'), + pyQueueEvents: {}, otherQueueEvents: {}, queueNames: { python: 'python', other: 'other' }, + connection: { set: async () => { writes++; return 'OK'; } }, + })); + const { env } = await import('./src/config'); + env.JOB_TIMEOUT = 15000; + env.RUNTIME_SESSION_MODE = 'stateless'; + env.SANDBOX_BACKEND = 'http'; + env.HARDENED_SANDBOX_MODE = false; + env.EGRESS_GRANT_SECRET = ''; + env.EXECUTION_MANIFEST_SECRET = ''; + const { default: router } = await import('./src/service/router'); + const handler = router.stack.find(layer => layer.route?.path === '/exec').route.stack.at(-1).handle; + async function request(lang, timeout) { + submitted = []; + writes = 0; + const req = { + body: { code: 'print(1)', lang, timeout }, headers: {}, header: () => undefined, on: () => {}, + codeApiPrincipal: { userId: 'user', tenantId: 'tenant', principalSource: 'none' }, + codeApiAuthContext: { userId: 'user', tenantId: 'tenant' }, + }; + const res = { status(code) { this.statusCode = code; return this; }, json(body) { this.body = body; return this; } }; + await handler(req, res); + return res; + } + for (const lang of ['py', 'bash']) { + for (const [input, expected] of [[1000, 1000], [1000.1, 1001], [0.1, 1], [999999, 15000], [null, undefined], [undefined, undefined]]) { + const res = await request(lang, input); + assert.equal(res.statusCode, 200, JSON.stringify(res.body)); + assert.equal(submitted.length, 1); + assert.equal(submitted[0].name, lang === 'py' ? 'python' : 'other'); + assert.equal(submitted[0].data.payload.run_timeout, expected); + if (expected === undefined) assert.equal('run_timeout' in submitted[0].data.payload, false); + } + } + for (const input of [0, -1, '1000', true, {}, [], NaN, Infinity]) { + const res = await request('py', input); + assert.equal(res.statusCode, 400); + assert.match(res.body.error, /timeout must be a positive number of milliseconds/); + assert.equal(submitted.length, 0); + assert.equal(writes, 0, 'invalid timeout must not register a session'); + } + console.log('EXEC_TIMEOUT_OK'); + `], { cwd: resolve(__dirname, '../..'), stdout: 'pipe', stderr: 'pipe' }); + const [exitCode, stdout, stderr] = await Promise.all([ + probe.exited, new Response(probe.stdout).text(), new Response(probe.stderr).text(), + ]); + expect(exitCode, `${stdout}\n${stderr}`).toBe(0); + expect(stdout).toContain('EXEC_TIMEOUT_OK'); +}, 15000); diff --git a/service/src/service/router.ts b/service/src/service/router.ts index f1a840be..f89bbb17 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -25,7 +25,7 @@ import { Jobs, Languages } from '../enum'; import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; import { createUploadSessionRegistrar } from './upload-session'; import { recordSessionOwnership } from '../session-ownership'; -import { prepareSandboxJobSecurity } from '../sandbox-egress'; +import { normalizeProgrammaticTimeoutMs, prepareSandboxJobSecurity } from '../sandbox-egress'; import { BridgeWorkerSelectionError, CODEAPI_BRIDGE_WORKER_HEADER, @@ -147,6 +147,14 @@ router.post('/exec', executionLimiter, async (req: t.AuthenticatedRequest, res) return res.status(400).json({ error: `Unsupported language: ${rawLang}` }); } + // An omitted cap keeps the worker's existing language-specific default. + let timeout: number | undefined; + try { + if (body.timeout != null) timeout = normalizeProgrammaticTimeoutMs(body.timeout); + } catch (error) { + return res.status(400).json({ error: (error as Error).message }); + } + let bridgeWorkerId: string | undefined; try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -250,6 +258,7 @@ router.post('/exec', executionLimiter, async (req: t.AuthenticatedRequest, res) isPyPlot, session_id, }); + if (timeout != null) rawPayload.run_timeout = timeout; const sandboxSecurity = prepareSandboxJobSecurity({ req, executionId: execution_id, diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 0f97a532..2a90eac7 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -133,6 +133,8 @@ export type ExecuteResponse = { }; export interface RequestBody { + /** Optional positive runtime cap in milliseconds, clamped to JOB_TIMEOUT. */ + timeout?: number; code: string; lang: string; args?: string[]; From 6da7d0aeca5592c031fd6cbcad2b5996f6d25991 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 21:51:09 -0400 Subject: [PATCH 058/116] fix: validate private storage through open descriptors (#147) --- packages/code/src/storage.test.ts | 143 +++++++++++++++++++++++++++++- packages/code/src/storage.ts | 10 ++- 2 files changed, 151 insertions(+), 2 deletions(-) diff --git a/packages/code/src/storage.test.ts b/packages/code/src/storage.test.ts index f3e8243e..e246ff85 100644 --- a/packages/code/src/storage.test.ts +++ b/packages/code/src/storage.test.ts @@ -1,5 +1,17 @@ import assert from 'node:assert/strict'; -import { chmod, mkdir, mkdtemp, open, readdir, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { + chmod, + mkdir, + mkdtemp, + open, + readFile, + readdir, + rename, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -142,6 +154,135 @@ test('paired identity is persisted atomically with owner-only permissions', asyn } }); +test('identity saves validate permissions on the open temporary file', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-save-race-')); + const path = join(directory, 'identity.json'); + const displaced = join(directory, 'displaced.tmp'); + const identity = { + protocolVersion: 1 as const, + workerId: 'vm-1', + codeApiUrl: 'https://code.example/v1', + credential: 'must-not-be-written', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + publicKey: 'public-key', + privateKey: 'private-key', + }; + try { + const probe = await open(directory, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = fileHandlePrototype.chmod; + let swapped = false; + t.mock.method( + fileHandlePrototype, + 'chmod', + async function (this: FileHandle, mode: number) { + if (mode !== 0o600 || swapped) return originalChmod.call(this, mode); + swapped = true; + await originalChmod.call(this, 0o666); + const temporary = (await readdir(directory)).find((name) => + name.endsWith('.tmp'), + ); + assert.ok(temporary); + const temporaryPath = join(directory, temporary); + await rename(temporaryPath, displaced); + await writeFile(temporaryPath, 'owner-only decoy', { mode: 0o600 }); + }, + ); + + await assert.rejects( + saveBridgeIdentity(path, identity), + /Cannot restrict .*identity\.json.*mode 666/, + ); + assert.equal(await readFile(displaced, 'utf8'), ''); + assert.equal((await stat(displaced)).mode & 0o777, 0o666); + await assert.rejects(stat(path), { code: 'ENOENT' }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('pairing reservations validate permissions on the open destination', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-reserve-race-')); + const path = join(directory, 'identity.json'); + const displaced = join(directory, 'displaced.json'); + try { + const probe = await open(directory, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = fileHandlePrototype.chmod; + let swapped = false; + t.mock.method( + fileHandlePrototype, + 'chmod', + async function (this: FileHandle, mode: number) { + if (mode !== 0o600 || swapped) return originalChmod.call(this, mode); + swapped = true; + await originalChmod.call(this, 0o666); + await rename(path, displaced); + await writeFile(path, 'owner-only decoy', { mode: 0o600 }); + }, + ); + + await assert.rejects( + assertIdentityPathIsPrivate(path), + /Cannot restrict .*identity\.json.*mode 666/, + ); + assert.equal(await readFile(displaced, 'utf8'), ''); + assert.equal((await stat(displaced)).mode & 0o777, 0o666); + await assert.rejects(stat(path), { code: 'ENOENT' }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('quarantine saves validate permissions on the open marker file', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-quarantine-race-')); + const path = join(directory, 'quarantine.json'); + const displaced = join(directory, 'displaced.json'); + const record = { + version: 1 as const, + workerId: 'vm-1', + workspaceId: 'primary', + quarantinedAt: new Date().toISOString(), + reason: 'must-not-be-written', + }; + try { + const probe = await open(directory, 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = fileHandlePrototype.chmod; + let swapped = false; + t.mock.method( + fileHandlePrototype, + 'chmod', + async function (this: FileHandle, mode: number) { + if (mode !== 0o600 || swapped) return originalChmod.call(this, mode); + swapped = true; + await originalChmod.call(this, 0o666); + await rename(path, displaced); + await writeFile(path, 'owner-only decoy', { mode: 0o600 }); + }, + ); + + await assert.rejects( + saveWorkspaceMutationQuarantine(path, record), + /Cannot restrict .*quarantine\.json.*mode 666/, + ); + assert.equal(await readFile(displaced, 'utf8'), ''); + assert.equal((await stat(displaced)).mode & 0o777, 0o666); + await assert.rejects(stat(path), { code: 'ENOENT' }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + test('workspace mutation quarantine persists until explicitly cleared', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-quarantine-')); const path = join(directory, 'state', 'quarantine.json'); diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index af2f1295..1cbb7093 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -225,7 +225,15 @@ async function readGuardedFile( } async function assertOwnerOnlyFile(handle: FileHandle, path: string): Promise { - const mode = (await handle.stat()).mode & 0o777; + const metadata = await handle.stat(); + const self = process.getuid?.(); + if (self !== undefined && !isTrustedOwner(metadata.uid, self)) { + throw new BridgeProtocolError( + `${path} is owned by another account (uid ${metadata.uid}), ` + + 'which can rewrite it. Keep worker credentials on a path this account owns.', + ); + } + const mode = metadata.mode & 0o777; if ((mode & 0o077) !== 0) { throw new BridgeProtocolError( `Cannot restrict ${path} to owner-only access (mode ${mode.toString(8)}). ` + From dc34012558e21f1058f68425715be83ddcf40a63 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 23:27:48 -0400 Subject: [PATCH 059/116] fix: bind GitHub App token requests to the enterprise host (#150) * fix: bind GitHub App token requests to the enterprise host * fix: reject empty API URL query and fragment suffixes --- packages/code/README.md | 9 +++++-- packages/code/src/cli.test.ts | 41 ++++++++++++++++++++++++++++ packages/code/src/cli.ts | 1 + packages/code/src/github.test.ts | 46 ++++++++++++++++++++++++++++++++ packages/code/src/github.ts | 33 ++++++++++++++--------- 5 files changed, 116 insertions(+), 14 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 84adfc91..9dd3019a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -158,8 +158,13 @@ authentication is configured. The worker identity, GitHub App key path, token source variables, and mutation-quarantine record remain denied to sandboxed commands. -For GitHub Enterprise Server, set `LIBRECHAT_CODE_GITHUB_HOST` to its hostname -and `LIBRECHAT_CODE_GITHUB_API_URL` to its HTTPS API base URL. GitHub +For GitHub Enterprise Server, set `LIBRECHAT_CODE_GITHUB_HOST` to its hostname. +App authentication defaults to `https:///api/v3`; GitHub.com continues to +use `https://api.github.com`. Set `LIBRECHAT_CODE_GITHUB_API_URL` to override +the HTTPS API base URL, including a custom port or path. Its hostname must +match the configured Git host (with `api.github.com` corresponding to +`github.com`), and it must not contain credentials, a query, or a fragment. +App token requests do not follow redirects. GitHub authentication currently requires the `native-srt` command sandbox. Every clone, commit, or push command still crosses LibreChat's tool-approval policy; the credential boundary does not grant approval by itself. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index ba195dbc..f978fce5 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -371,3 +371,44 @@ test('CLI treats a whitespace-only file relay upstream as disabled', () => { /LIBRECHAT_CODE_(?:EXECUTION_MANIFEST_PUBLIC_KEY|FILE_RELAY_IMAGE) is required/, ); }); + +test('CLI host-only enterprise configuration sends App JWTs to GHES, never GitHub.com', async (t) => { + const { generateKeyPairSync } = await import('node:crypto'); + const { mkdtemp, rm, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const directory = await mkdtemp(join(tmpdir(), 'cli-ghes-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const preload = join(directory, 'fetch.mjs'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { mode: 0o600 }); + await writeFile(preload, ` + globalThis.fetch = async (input) => { + console.error('GITHUB_REQUEST:' + String(input)); + throw new Error('test stopped before network delivery'); + }; + `); + const result = spawnSync(process.execPath, [ + '--import', preload, fileURLToPath(new URL('./cli.js', import.meta.url)), + ], { + encoding: 'utf8', timeout: 10000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + LIBRECHAT_CODE_GITHUB_HOST: 'GitHub.Example.Test', + LIBRECHAT_CODE_GITHUB_API_URL: undefined, + }, + }); + assert.equal(result.status, 1, result.stderr); + assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app\/installations\/456\/access_tokens/); + assert.doesNotMatch(result.stderr, /GITHUB_REQUEST:https:\/\/api\.github\.com/); +}); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index e9442df9..28103891 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -195,6 +195,7 @@ function githubCredentials(): { appId: appId!, installationId: installationId!, privateKeyPath: privateKeyPath!, + host, apiUrl, }), }; diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 81dbb2c3..bd62acfd 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -229,3 +229,49 @@ test('allows the GitHub LFS object delivery hosts', () => { assert.ok(GITHUB_ALLOWED_DOMAINS.includes('*.githubusercontent.com')); assert.ok(GITHUB_ALLOWED_DOMAINS.includes('github-cloud.s3.amazonaws.com')); }); + +test('App JWT requests use the resolved public or enterprise endpoint', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'github-endpoint-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { mode: 0o600 }); + for (const [host, apiUrl, expected] of [ + [undefined, undefined, 'https://api.github.com'], + ['GitHub.COM', undefined, 'https://api.github.com'], + ['GitHub.Example.Test', undefined, 'https://github.example.test/api/v3'], + [undefined, 'https://github.example.test/api/v3/', 'https://github.example.test/api/v3'], + ['github.example.test', 'https://github.example.test:8443/custom/api/', 'https://github.example.test:8443/custom/api'], + ]) { + let calls = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', installationId: '456', privateKeyPath, host, apiUrl, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: async (input, init) => { + calls++; + assert.equal(String(input), `${expected}/app/installations/456/access_tokens`); + assert.equal(init?.method, 'POST'); + assert.equal(init?.redirect, 'error'); + assert.match(new Headers(init?.headers).get('authorization')!, /^Bearer eyJ/); + return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + }, + }); + await provider.getCredential(); + await provider.getCredential(); + assert.equal(calls, 1); + } +}); + +test('invalid or mismatched App endpoints are refused before any private key access', () => { + for (const apiUrl of [ + 'https://api.github.com', 'https://other.example.test/api/v3', + 'http://github.example.test/api/v3', 'https://user:password@github.example.test/api/v3', + 'https://github.example.test/api/v3?query=1', 'https://github.example.test/api/v3#fragment', + 'https://github.example.test/api/v3?', 'https://github.example.test/api/v3#', + ]) { + assert.throws(() => new GitHubAppCredentialProvider({ + appId: '123', installationId: '456', privateKeyPath: '/must-not-be-read', + host: 'github.example.test', apiUrl, + }), /must match|HTTPS URL without credentials|query or fragment/); + } +}); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index e71188c8..7b1f12b1 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -29,6 +29,8 @@ export interface GitHubAppCredentialProviderOptions { installationId: string; privateKeyPath: string; apiUrl?: string; + /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ + host?: string; fetch?: typeof globalThis.fetch; now?: () => Date; platform?: NodeJS.Platform; @@ -95,6 +97,7 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private cached?: GitHubCredential; + private readonly apiUrl: string; constructor(private readonly options: GitHubAppCredentialProviderOptions) { if ((options.platform ?? process.platform) === 'win32') { @@ -107,14 +110,23 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { 'GitHub App installation ID', options.installationId, ); - if (options.apiUrl != null) { - const apiUrl = new URL(options.apiUrl); - if (apiUrl.protocol !== 'https:' || apiUrl.username || apiUrl.password) { - throw new Error( - 'GitHub API URL must be an HTTPS URL without credentials', - ); - } + const host = options.host == null ? undefined : normalizeGitHubHost(options.host); + const apiUrl = new URL(options.apiUrl ?? ( + host != null && host !== 'github.com' + ? `https://${host}/api/v3` + : 'https://api.github.com' + )); + if (apiUrl.protocol !== 'https:' || apiUrl.username || apiUrl.password) { + throw new Error('GitHub API URL must be an HTTPS URL without credentials'); } + if (/[?#]/.test(apiUrl.href)) { + throw new Error('GitHub API URL must not contain a query or fragment'); + } + const apiHost = apiUrl.hostname === 'api.github.com' ? 'github.com' : apiUrl.hostname; + if (host != null && host !== apiHost) { + throw new Error('LIBRECHAT_CODE_GITHUB_HOST must match the GitHub App API hostname'); + } + this.apiUrl = apiUrl.href.replace(/\/+$/, ''); } async getCredential(signal?: AbortSignal): Promise { @@ -127,15 +139,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { } const privateKey = await readPrivateKey(this.options.privateKeyPath); const jwt = createAppJwt(this.options.appId, privateKey, now); - const apiUrl = (this.options.apiUrl ?? 'https://api.github.com').replace( - /\/+$/, - '', - ); const request = this.options.fetch ?? globalThis.fetch; const response = await request( - `${apiUrl}/app/installations/${this.options.installationId}/access_tokens`, + `${this.apiUrl}/app/installations/${this.options.installationId}/access_tokens`, { method: 'POST', + redirect: 'error', headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${jwt}`, From 5fdeeebec551f2e8525f865ac81e82572e7ea03b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 23:28:03 -0400 Subject: [PATCH 060/116] fix(code): support declared Node engine range (#151) --- .github/workflows/ci.yml | 10 ++++++---- packages/code/src/worker.ts | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1a4b6e3..9c7b75d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,8 +125,12 @@ jobs: run: bun run test code-package-tests: - name: Code Package Tests + name: Code Package Tests (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ['20.11.0', '22.21.0', '24.16.0'] defaults: run: working-directory: packages/code @@ -135,9 +139,7 @@ jobs: - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - # The suite fails 47 worker tests on Node 22 despite the package's - # ">=20.11" engines range, so CI pins the version it is green on. - node-version: 24.16.0 + node-version: ${{ matrix.node-version }} cache: npm cache-dependency-path: packages/code/package-lock.json diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 657ba726..1fda70f7 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -1647,7 +1647,6 @@ export class BridgeWorker { signal?.addEventListener('abort', abortRequest, { once: true }); } const timeout = setTimeout(abortRequest, timeoutMs); - timeout.unref?.(); try { return await this.request(url, body, controller.signal); } finally { From 725f79900bf06298653668a029eefd69460d900e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 7 Sep 2026 23:28:32 -0400 Subject: [PATCH 061/116] fix: use runner DNS for KVM artifact service discovery (#152) * fix: use runner DNS for KVM artifact service discovery * fix: initialize KVM DNS before every guest executable * fix: pass only guest arguments to libkrun exec --- .github/workflows/ci.yml | 3 + README.md | 17 ++++ api/Dockerfile | 7 +- api/src/guest-dns.sh | 49 +++++++++++ docker/Dockerfile.worker-sandbox | 7 +- launcher/Dockerfile | 4 +- launcher/entrypoint.sh | 62 ++------------ launcher/src/main.rs | 12 ++- tests/kvm_guest_dns.sh | 138 +++++++++++++++++++++++++++++++ 9 files changed, 239 insertions(+), 60 deletions(-) create mode 100644 api/src/guest-dns.sh create mode 100755 tests/kvm_guest_dns.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c7b75d2..6f79328e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: - name: Block-root package delivery run: tests/block_root_package_delivery.sh + - name: KVM guest resolver handoff + run: tests/kvm_guest_dns.sh + - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh diff --git a/README.md b/README.md index c2eeb7eb..1e6432f7 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,23 @@ virtio-fs mount. The first image build takes longer because it compiles the language runtimes, but package-heavy workloads do not accumulate host file descriptors in the launcher. +KVM guests use the runner container's `/etc/resolv.conf`, including Docker's +embedded resolver or Kubernetes nameservers and search domains. The launcher +preserves service hostnames instead of pinning their startup IP addresses. +Both baked and directory rootfs images contain a resolver symlink whose target +is populated by a guest wrapper in private `/run` runtime storage before any +`LAUNCHER_EXEC` executable starts; the +read-only root disk does not need modification at boot. Rebuild the runner +image to pick up this layout change. A missing resolver handoff fails startup +rather than leaving the guest with an unrelated public DNS server. + +To validate a deployment, execute code that creates a file in `/mnt/data`, +confirm the response includes its file reference, and download it. Recreate the +egress gateway with a different container IP while leaving the runner alive, +then repeat after DNS caches expire. The file must still upload and download; +`artifact_delivery` must not report a failure. `tests/kvm_guest_dns.sh` checks +the resolver handoff and rootfs assembly without requiring KVM. + Setting `KVM_ENABLED=false` still selects the directory-root target and the host package mount automatically for direct NsJail development. diff --git a/api/Dockerfile b/api/Dockerfile index 26877009..66db9e3d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -148,6 +148,7 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \ 2>/dev/null || true COPY api/src/entrypoint.sh ./entrypoint.sh +COPY api/src/guest-dns.sh ./guest-dns.sh COPY api/src/hosted-app-launcher.sh /usr/local/bin/codeapi-hosted-app-launcher RUN chmod +x ./entrypoint.sh /usr/local/bin/codeapi-hosted-app-launcher @@ -266,7 +267,8 @@ COPY --from=sandbox-build / /sandbox-rootfs/ COPY --from=package-builder /pkgs /sandbox-rootfs/pkgs COPY docker/build-rootfs-image.sh /usr/local/bin/build-rootfs-image.sh -RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg +RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \ + && bash /sandbox-rootfs/sandbox_api/guest-dns.sh --prepare-rootfs /sandbox-rootfs RUN chmod +x /usr/local/bin/build-rootfs-image.sh \ && /usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img @@ -284,7 +286,8 @@ FROM sandbox-runner-base AS sandbox-runner COPY --from=sandbox-build / /sandbox-rootfs/ -RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg +RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \ + && bash /sandbox-rootfs/sandbox_api/guest-dns.sh --prepare-rootfs /sandbox-rootfs RUN mkdir -p /host-packages diff --git a/api/src/guest-dns.sh b/api/src/guest-dns.sh new file mode 100644 index 00000000..9795ca7f --- /dev/null +++ b/api/src/guest-dns.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# The guest root may be read-only. Bake the link, populate its target only +# after /run is mounted, and leave direct NsJail/Lambda resolvers untouched. + +prepare_guest_dns() { + local root="$1" + mkdir -p "$root/run" + rm -f "$root/etc/resolv.conf" + ln -s ../run/codeapi-resolver/resolv.conf "$root/etc/resolv.conf" +} + +configure_guest_dns() { + local root="${1:-}" + local target="$root/run/codeapi-resolver" + if [ ! -L "$root/etc/resolv.conf" ] || \ + [ "$(readlink "$root/etc/resolv.conf")" != '../run/codeapi-resolver/resolv.conf' ]; then + return 0 + fi + if ! printf '%s\n' "${SANDBOX_RESOLV_CONF:-}" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then + echo 'ERROR: KVM guest requires resolver configuration from launcher-entrypoint.sh' >&2 + return 1 + fi + # A fresh, root-owned directory prevents a sandbox UID from replacing DNS + # configuration in the runtime mount. Never reuse a pre-existing entry. + (umask 077; mkdir "$target") || return 1 + printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$target/resolv.conf" || return 1 + chmod 600 "$target/resolv.conf" || return 1 + unset SANDBOX_RESOLV_CONF +} + +run_guest_command() { + local root="$1" + shift + # This runs for every LAUNCHER_EXEC, before the selected executable. Keep + # DNS separate from /tmp, which the normal API entrypoint mounts later. + mount -t tmpfs -o size=1m,mode=0755 tmpfs "$root/run" || return 1 + configure_guest_dns "$root" || return 1 + exec -- "$@" +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + set -e + case "${1:-}" in + --prepare-rootfs) prepare_guest_dns "${2:?rootfs path required}" ;; + --configure) configure_guest_dns "${2:-}" ;; + --exec) run_guest_command "" "${2:?guest executable required}" ;; + *) echo 'usage: guest-dns.sh --prepare-rootfs ROOTFS | --configure [ROOTFS] | --exec EXECUTABLE' >&2; exit 2 ;; + esac +fi diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index c18eab82..7156719c 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -152,6 +152,7 @@ RUN bun install --frozen-lockfile --production COPY --from=sandbox-builder /app/.build ./.build COPY api/config ./config COPY api/src/entrypoint.sh ./entrypoint.sh +COPY api/src/guest-dns.sh ./guest-dns.sh RUN chmod +x ./entrypoint.sh RUN mkdir -p /pkgs /tmp/sandbox @@ -200,7 +201,8 @@ COPY --from=sandbox-rootfs / /sandbox-rootfs/ COPY --from=package-builder /pkgs /sandbox-rootfs/pkgs COPY docker/build-rootfs-image.sh /usr/local/bin/build-rootfs-image.sh -RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg +RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \ + && bash /sandbox-rootfs/sandbox_api/guest-dns.sh --prepare-rootfs /sandbox-rootfs RUN chmod +x /usr/local/bin/build-rootfs-image.sh \ && /usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img @@ -236,7 +238,7 @@ ENV PATH="/root/.bun/bin:${PATH}" COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup -# --- Launcher entrypoint (DNS resolution + socat relay before VM boot) --- +# --- Launcher entrypoint (resolver configuration before VM boot) --- COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh @@ -268,6 +270,7 @@ FROM worker-sandbox-base AS worker-sandbox-legacy COPY --from=sandbox-rootfs / /sandbox-rootfs/ RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \ + && bash /sandbox-rootfs/sandbox_api/guest-dns.sh --prepare-rootfs /sandbox-rootfs \ && mkdir -p /host-packages # KVM production default. The package tree is part of the read-only block root, diff --git a/launcher/Dockerfile b/launcher/Dockerfile index 1a077e12..b2b02386 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -93,6 +93,7 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \ 2>/dev/null || true COPY api/src/entrypoint.sh ./entrypoint.sh +COPY api/src/guest-dns.sh ./guest-dns.sh RUN chmod +x ./entrypoint.sh # ============================================================================ @@ -132,7 +133,8 @@ COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-s COPY --from=sandbox-build / /sandbox-rootfs/ -RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg +RUN sed -i '/^cgroup_mem_swap_max/d' /sandbox-rootfs/sandbox_api/config/sandbox.cfg \ + && bash /sandbox-rootfs/sandbox_api/guest-dns.sh --prepare-rootfs /sandbox-rootfs RUN mkdir -p /host-packages diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index a6db4cf5..0369ec2a 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -1,59 +1,15 @@ #!/bin/bash set -e -# Resolve Docker Compose service names to IPs before entering the microVM. -# libkrun's TSI networking doesn't have access to Docker's embedded DNS (127.0.0.11), -# so DNS-based service discovery won't work inside the guest. - -resolve_url() { - local var_name="$1" - local url="${!var_name}" - [ -z "$url" ] && return - - local proto="${url%%://*}" - local rest="${url#*://}" - local host_port="${rest%%/*}" - local path="/${rest#*/}" - [ "$rest" = "$host_port" ] && path="" - local host="${host_port%%:*}" - local port="${host_port#*:}" - [ "$host" = "$port" ] && port="" - - # Skip if already an IP - echo "$host" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' && return - - local ip - ip=$(getent hosts "$host" 2>/dev/null | awk '{print $1}' | head -1) - if [ -n "$ip" ]; then - local new_url="${proto}://${ip}" - [ -n "$port" ] && new_url="${new_url}:${port}" - new_url="${new_url}${path}" - export "$var_name"="$new_url" - echo "[entrypoint] ${var_name}: ${host} -> ${ip}" - fi -} - -resolve_host_port() { - local var_name="$1" - local val="${!var_name}" - [ -z "$val" ] && return - - local host="${val%%:*}" - local port="${val#*:}" - - echo "$host" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' && return - - local ip - ip=$(getent hosts "$host" 2>/dev/null | awk '{print $1}' | head -1) - if [ -n "$ip" ]; then - export "$var_name"="${ip}:${port}" - echo "[entrypoint] ${var_name}: ${host} -> ${ip}" - fi -} - -resolve_url EGRESS_GATEWAY_URL -resolve_url FILE_SERVER_URL -resolve_host_port SANDBOX_FORWARD_TARGET +# TSI opens guest sockets in this container's network namespace. Keep service +# names intact so new connections can resolve replacements after a restart. +# Forward the resolver and search domains supplied by Docker or Kubernetes, +# rather than pinning endpoint IPs or baking a deployment-specific nameserver. +export SANDBOX_RESOLV_CONF="$(cat /etc/resolv.conf)" +if ! printf '%s\n' "$SANDBOX_RESOLV_CONF" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then + echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2 + exit 1 +fi if [ "${LAUNCHER_FILTER_VSOCK_ENOTCONN:-true}" = "true" ]; then # libkrun can emit this benign TSI/vsock teardown line after the guest has diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 5376b07c..cbc4b5fc 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -424,6 +424,7 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_EXECUTE_BODY_LIMIT", "SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY", "SANDBOX_FORWARD_TARGET", + "SANDBOX_RESOLV_CONF", "SANDBOX_LIMIT_OVERRIDES", "SANDBOX_LOG_LEVEL", "SANDBOX_MAX_CONCURRENT_JOBS", @@ -518,7 +519,8 @@ fn main() { let root_device_c = cstr(&root_device); let root_fstype_c = cstr(&root_fstype); let root_options_c = cstr(&root_options); - let exec_c = cstr(&exec_path); + // Always initialize guest DNS, including when LAUNCHER_EXEC overrides the API. + let exec_c = cstr("/bin/bash"); let port_map_strs = vec![cstr("2000:2000")]; let port_map_ptrs = null_term(&port_map_strs); @@ -533,7 +535,12 @@ fn main() { .collect(); let env_ptrs = null_term(&env_strs); - let argv_strs: Vec = vec![cstr(&exec_path)]; + // krun_set_exec supplies argv[0]; this array contains arguments only. + let argv_strs: Vec = vec![ + cstr("/sandbox_api/guest-dns.sh"), + cstr("--exec"), + cstr(&exec_path), + ]; let argv_ptrs = null_term(&argv_strs); let rlimit_strs: Vec = vec![guest_nofile_rlimit(nofile_target)]; @@ -645,6 +652,7 @@ mod tests { "SANDBOX_DISABLE_NETWORKING", "SANDBOX_ALLOWED_LOCAL_NETWORK_PORT", "SANDBOX_FORWARD_TARGET", + "SANDBOX_RESOLV_CONF", "SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY", "SANDBOX_RUN_TIMEOUT", "NSJAIL_CONFIG", diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh new file mode 100755 index 00000000..e00e3e38 --- /dev/null +++ b/tests/kvm_guest_dns.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_DIR="$(mktemp -d)" +trap 'chmod -R u+w "$TEST_DIR"; rm -rf "$TEST_DIR"' EXIT +source "$ROOT/api/src/guest-dns.sh" + +# Configure the baked link while writable, then only the runtime /run target. +mkdir -p "$TEST_DIR/guest/etc" "$TEST_DIR/guest/run" +printf 'nameserver 1.1.1.1\n' > "$TEST_DIR/guest/etc/resolv.conf" +prepare_guest_dns "$TEST_DIR/guest" +[[ "$(readlink "$TEST_DIR/guest/etc/resolv.conf")" == '../run/codeapi-resolver/resolv.conf' ]] +chmod 555 "$TEST_DIR/guest/etc" +SANDBOX_RESOLV_CONF=$'nameserver 127.0.0.11\noptions ndots:0' +configure_guest_dns "$TEST_DIR/guest" +printf 'nameserver 127.0.0.11\noptions ndots:0\n' > "$TEST_DIR/expected" +cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +[[ ! -v SANDBOX_RESOLV_CONF ]] +# Ownership protection: no group/other permissions on the runtime directory. +[[ "$(ls -ld "$TEST_DIR/guest/run/codeapi-resolver" | cut -c1-10)" == 'drwx------' ]] + +# A fresh boot can use Kubernetes DNS/search paths without rebuilding the root. +rm -rf "$TEST_DIR/guest/run/codeapi-resolver" +SANDBOX_RESOLV_CONF=$'nameserver 10.96.0.10\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5' +printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_DIR/expected" +configure_guest_dns "$TEST_DIR/guest" +cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" + +# Never reuse a stale directory or follow an attacker-controlled runtime link. +SANDBOX_RESOLV_CONF='nameserver 127.0.0.11' +if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then + echo 'accepted pre-existing runtime DNS directory' >&2; exit 1 +fi +rm -rf "$TEST_DIR/guest/run/codeapi-resolver" +mkdir "$TEST_DIR/foreign" +ln -s "$TEST_DIR/foreign" "$TEST_DIR/guest/run/codeapi-resolver" +if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then + echo 'accepted runtime DNS symlink' >&2; exit 1 +fi +[[ ! -e "$TEST_DIR/foreign/resolv.conf" ]] +rm "$TEST_DIR/guest/run/codeapi-resolver" +unset SANDBOX_RESOLV_CONF +if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then + echo 'accepted missing guest resolver' >&2; exit 1 +fi +SANDBOX_RESOLV_CONF='# no nameserver' +if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then + echo 'accepted empty guest resolver' >&2; exit 1 +fi + +# Direct NsJail and Lambda retain the resolver managed by their container. +mkdir -p "$TEST_DIR/direct/etc" +printf 'nameserver 192.0.2.53\n' > "$TEST_DIR/direct/etc/resolv.conf" +cp "$TEST_DIR/direct/etc/resolv.conf" "$TEST_DIR/expected" +configure_guest_dns "$TEST_DIR/direct" +cmp "$TEST_DIR/expected" "$TEST_DIR/direct/etc/resolv.conf" + +# Exercise the actual launcher script up to exec, substituting only its binary. +# Service names (including HTTPS authority and IPv6) must never be rewritten. +mkdir "$TEST_DIR/bin" +cat > "$TEST_DIR/bin/launcher" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +[[ "$EGRESS_GATEWAY_URL" == 'https://egress_gateway:3190/base' ]] +[[ "$FILE_SERVER_URL" == 'http://[::1]:3000/base' ]] +[[ "$SANDBOX_FORWARD_TARGET" == 'tool_call_server:3033' ]] +printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_RESOLVER_OUTPUT" +STUB +cat > "$TEST_DIR/bin/getent" <<'STUB' +#!/usr/bin/env bash +printf '192.0.2.99 stale-address\n' +STUB +chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" +sed "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" +PATH="$TEST_DIR/bin:$PATH" \ +EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ +FILE_SERVER_URL='http://[::1]:3000/base' \ +SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ +LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ +TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ +bash "$TEST_DIR/entrypoint.sh" +printf '%s\n' "$(cat /etc/resolv.conf)" > "$TEST_DIR/expected" +cmp "$TEST_DIR/expected" "$TEST_DIR/forwarded" + +# Every rootfs assembly path must prepare DNS after COPY, before disk creation. +python3 - "$ROOT" <<'PY' +from pathlib import Path +import sys +root = Path(sys.argv[1]) +for name, count in [('api/Dockerfile', 2), ('docker/Dockerfile.worker-sandbox', 2), ('launcher/Dockerfile', 1)]: + text = (root / name).read_text() + assert text.count('--prepare-rootfs /sandbox-rootfs') == count, name + assert 'COPY api/src/guest-dns.sh ./guest-dns.sh' in text, name + for stage in text.split('\nFROM '): + if 'COPY --from=sandbox-' in stage and ' / /sandbox-rootfs/' in stage: + assert stage.index(' / /sandbox-rootfs/') < stage.index('--prepare-rootfs /sandbox-rootfs'), name + if '/usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img' in stage: + assert stage.index('--prepare-rootfs /sandbox-rootfs') < stage.index('/usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img'), name +text = (root / 'launcher/src/main.rs').read_text() +assert '"SANDBOX_RESOLV_CONF"' in text.split('const ALLOW_EXACT:')[1].split('];')[0] +argv = text.split('let argv_strs:')[1].split('let argv_ptrs:')[0] +# libkrun init supplies argv[0]. Repeating the binary here makes Bash try to +# interpret /bin/bash itself as a shell script instead of the DNS wrapper. +assert 'cstr("/bin/bash")' not in argv +assert 'cstr("/sandbox_api/guest-dns.sh")' in argv +assert 'cstr("--exec")' in argv and 'cstr(&exec_path)' in argv +assert 'let exec_c = cstr("/bin/bash")' in text +PY +# The wrapper configures DNS before a custom guest executable, independently +# of the normal API entrypoint and its later /tmp mount. +rm -rf "$TEST_DIR/guest/run/codeapi-resolver" +cat > "$TEST_DIR/bin/mount" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +[[ "$*" == "-t tmpfs -o size=1m,mode=0755 tmpfs $TEST_GUEST_ROOT/run" ]] +[[ "${TEST_MOUNT_FAIL:-false}" != true ]] +STUB +cat > "$TEST_DIR/bin/custom-guest" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +[[ "$(cat "$TEST_GUEST_ROOT/etc/resolv.conf")" == 'nameserver 127.0.0.11' ]] +[[ ! -v SANDBOX_RESOLV_CONF ]] +echo 'custom guest DNS ready' +STUB +chmod +x "$TEST_DIR/bin/mount" "$TEST_DIR/bin/custom-guest" +PATH="$TEST_DIR/bin:$PATH" TEST_GUEST_ROOT="$TEST_DIR/guest" \ +SANDBOX_RESOLV_CONF='nameserver 127.0.0.11' \ +bash -c 'source "$1"; run_guest_command "$2" "$3"' -- \ + "$ROOT/api/src/guest-dns.sh" "$TEST_DIR/guest" "$TEST_DIR/bin/custom-guest" + +if PATH="$TEST_DIR/bin:$PATH" TEST_GUEST_ROOT="$TEST_DIR/guest" \ +TEST_MOUNT_FAIL=true SANDBOX_RESOLV_CONF='nameserver 127.0.0.11' \ +bash -c 'source "$1"; run_guest_command "$2" "$3"' -- \ + "$ROOT/api/src/guest-dns.sh" "$TEST_DIR/guest" "$TEST_DIR/bin/custom-guest" > "$TEST_DIR/failed-boot"; then + echo 'started custom guest despite failed runtime mount' >&2; exit 1 +fi +[[ ! -s "$TEST_DIR/failed-boot" ]] +printf 'KVM guest DNS checks passed\n' From d50e9bb9c645f99597aac441f5ddcca0d9336960 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 08:36:54 -0400 Subject: [PATCH 062/116] fix: preserve preview auth on first navigation (#153) --- docs/lambda-microvm/README.md | 5 +- .../src/hosted-app/preview-gateway.test.ts | 50 +++++++++++++++++++ service/src/hosted-app/preview-gateway.ts | 45 +++++++++++++---- 3 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 service/src/hosted-app/preview-gateway.test.ts diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 4ec87d94..80e5e468 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -232,7 +232,10 @@ Content-Type: application/json ``` `GET /v1/hosted-apps/:app_id?runtime_session_hint=...` returns status and a -fresh five-minute `preview_url`; `DELETE` on the same resource terminates the +fresh five-minute `preview_url`; the authorization response loads a minimal +same-origin handoff page before opening the app so the first request includes +the host-only `SameSite=Strict` preview cookie even when LibreChat is on another +site. `DELETE` on the same resource terminates the lease. A revision is immutable. Retrying the identical spec reasserts the resident process; changing code or launch settings requires a new revision and captures a new exact checkpoint. An ambiguous provider launch is replayed only diff --git a/service/src/hosted-app/preview-gateway.test.ts b/service/src/hosted-app/preview-gateway.test.ts new file mode 100644 index 00000000..1dcf7839 --- /dev/null +++ b/service/src/hosted-app/preview-gateway.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'bun:test'; +import type { Response } from 'express'; +import { sendHostedAppPreviewAuthorizationHandoff } from './preview-gateway'; + +describe('hosted app preview authorization handoff', () => { + test('ends the cross-site redirect chain before starting a same-origin navigation', () => { + const headers = new Map(); + let status: number | undefined; + let type: string | undefined; + let body: string | undefined; + let redirects = 0; + const response = { + setHeader(name: string, value: string) { + headers.set(name.toLowerCase(), value); + return this; + }, + status(value: number) { + status = value; + return this; + }, + type(value: string) { + type = value; + return this; + }, + send(value: string) { + body = value; + return this; + }, + redirect() { + redirects += 1; + return this; + }, + } as unknown as Response; + + sendHostedAppPreviewAuthorizationHandoff(response, 'signed.token/value', 300); + + expect(status).toBe(200); + expect(type).toBe('html'); + expect(redirects).toBe(0); + expect(headers.get('cache-control')).toBe('no-store'); + expect(headers.has('location')).toBe(false); + expect(headers.get('set-cookie')).toBe( + '__Host-codeapi-app=signed.token%2Fvalue; Path=/; HttpOnly; Secure; SameSite=Strict; Max-Age=300', + ); + expect(body).toContain(''); + expect(body).toContain(''); + expect(body).not.toContain('__codeapi/authorize'); + expect(body).not.toContain('token'); + }); +}); diff --git a/service/src/hosted-app/preview-gateway.ts b/service/src/hosted-app/preview-gateway.ts index 4ce6ed5d..9c076cf0 100644 --- a/service/src/hosted-app/preview-gateway.ts +++ b/service/src/hosted-app/preview-gateway.ts @@ -49,6 +49,40 @@ function reject(res: Response, status: number, message: string): Response { return res.status(status).type('text/plain').send(message); } +/** + * End the cross-site navigation before loading the app. A SameSite=Strict + * cookie set on a cross-site HTTP redirect can remain excluded for the whole + * redirect chain. Loading this small document first makes its navigation to + * `/` originate from the preview site while keeping the cookie Strict. + */ +export function sendHostedAppPreviewAuthorizationHandoff( + res: Response, + sessionToken: string, + maxAge: number, +): Response { + res.setHeader('Set-Cookie', [ + `${COOKIE_NAME}=${encodeURIComponent(sessionToken)}`, + 'Path=/', + 'HttpOnly', + 'Secure', + 'SameSite=Strict', + `Max-Age=${maxAge}`, + ].join('; ')); + res.setHeader('Cache-Control', 'no-store'); + return res.status(200).type('html').send([ + '', + '', + '', + '', + '', + 'Opening preview', + '', + '', + '

Continue to preview

', + '', + ].join('')); +} + export async function hostedAppPreviewGateway( req: Request, res: Response, @@ -111,16 +145,7 @@ export async function hostedAppPreviewGateway( expiresAt, }, previewKey()); const maxAge = Math.max(1, Math.floor((expiresAt - Date.now()) / 1_000)); - res.setHeader('Set-Cookie', [ - `${COOKIE_NAME}=${encodeURIComponent(sessionToken)}`, - 'Path=/', - 'HttpOnly', - 'Secure', - 'SameSite=Strict', - `Max-Age=${maxAge}`, - ].join('; ')); - res.setHeader('Cache-Control', 'no-store'); - res.redirect(303, '/'); + sendHostedAppPreviewAuthorizationHandoff(res, sessionToken, maxAge); return; } From 5ee769e4db40c413b0f89ba0fe9a347082756980 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 08:37:32 -0400 Subject: [PATCH 063/116] =?UTF-8?q?=F0=9F=A7=BE=20fix:=20Log=20Workspace?= =?UTF-8?q?=20Tool=20HTTP=20Outcomes=20(#154)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: log workspace tool HTTP outcomes * fix: observe workspace outcomes across middleware and settlement * test: handle overloaded logger arguments in timing assertion * fix: classify workspace outcomes across middleware boundaries * fix: scope workspace logging to the exact POST endpoint --- service/src/api-server.ts | 2 + service/src/hosted-app/preview-access.ts | 9 + service/src/hosted-app/preview-gateway.ts | 13 +- service/src/middleware/execution-profile.ts | 2 + service/src/service-api.ts | 2 + service/src/workspace-tools/outcome.ts | 79 ++++ service/src/workspace-tools/router.test.ts | 401 +++++++++++++++++++- service/src/workspace-tools/router.ts | 30 +- 8 files changed, 521 insertions(+), 17 deletions(-) create mode 100644 service/src/workspace-tools/outcome.ts diff --git a/service/src/api-server.ts b/service/src/api-server.ts index fd77b55d..fc93aebf 100644 --- a/service/src/api-server.ts +++ b/service/src/api-server.ts @@ -20,6 +20,7 @@ import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; import bridgeRouter from './bridge'; import workspaceToolsRouter from './workspace-tools'; +import { workspaceToolOutcomeLogging } from './workspace-tools/outcome'; import { connection } from './queue'; import { metricsHandler } from './metrics'; import { httpMetricsMiddleware } from './middleware/httpMetrics'; @@ -33,6 +34,7 @@ import { hostedAppPreviewGateway } from './hosted-app/preview-gateway'; const { LOCAL_MODE: isLocalMode } = env; const app = express(); +app.post('/v1/workspace-tools/execute', workspaceToolOutcomeLogging); app.disable('x-powered-by'); app.set('trust proxy', 1); app.use(traceHttpRequest('codeapi.api.request')); diff --git a/service/src/hosted-app/preview-access.ts b/service/src/hosted-app/preview-access.ts index 374b34ea..35abe7af 100644 --- a/service/src/hosted-app/preview-access.ts +++ b/service/src/hosted-app/preview-access.ts @@ -142,3 +142,12 @@ export function hostedAppPreviewAuthorizeUrl( origin.searchParams.set('token', token); return origin.toString(); } + +export function hostedAppRequestHostname(host: string | undefined): string | undefined { + if (host == null || host.length === 0 || /[\s/@\\]/.test(host)) return undefined; + try { + return new URL(`http://${host}`).hostname; + } catch { + return undefined; + } +} diff --git a/service/src/hosted-app/preview-gateway.ts b/service/src/hosted-app/preview-gateway.ts index 9c076cf0..301d47ad 100644 --- a/service/src/hosted-app/preview-gateway.ts +++ b/service/src/hosted-app/preview-gateway.ts @@ -5,6 +5,7 @@ import { readRuntimeSessionRecord } from '../runtime-session/registry'; import { HostedAppControlPlaneError } from './control-plane'; import { hostedAppRuntimeIdFromHostname, + hostedAppRequestHostname, HostedAppPreviewAccessError, hostedAppPreviewOwnerBinding, signHostedAppPreviewAccess, @@ -16,16 +17,6 @@ import { applyHostedAppPreviewSecurityHeaders } from './proxy-policy'; const COOKIE_NAME = '__Host-codeapi-app'; const PREVIEW_COOKIE_TTL_MS = 60 * 60_000; -function rawHostname(req: Request): string | undefined { - const host = req.headers.host; - if (!host || /[\s/@\\]/.test(host)) return undefined; - try { - return new URL(`http://${host}`).hostname; - } catch { - return undefined; - } -} - function cookie(req: Request, name: string): string | undefined { for (const item of (req.headers.cookie ?? '').split(';')) { const separator = item.indexOf('='); @@ -89,7 +80,7 @@ export async function hostedAppPreviewGateway( next: NextFunction, ): Promise { if (!env.HOSTED_APPS_ENABLED || !env.HOSTED_APP_PREVIEW_ORIGIN) return next(); - const hostname = rawHostname(req); + const hostname = hostedAppRequestHostname(req.headers.host); const runtimeId = hostname ? hostedAppRuntimeIdFromHostname(hostname, env.HOSTED_APP_PREVIEW_ORIGIN) : undefined; diff --git a/service/src/middleware/execution-profile.ts b/service/src/middleware/execution-profile.ts index 38512dc4..37b8ba96 100644 --- a/service/src/middleware/execution-profile.ts +++ b/service/src/middleware/execution-profile.ts @@ -1,5 +1,6 @@ import type { NextFunction, Request, Response } from 'express'; import { env } from '../config'; +import { recordWorkspaceToolRejection } from '../workspace-tools/outcome'; import { checkExecutionProfileExpectation, EXECUTION_PROFILE_HEADER, @@ -48,5 +49,6 @@ export function executionProfileMiddleware( ? 'mismatch' : 'invalid', }); + recordWorkspaceToolRejection(res, expectation.body.error); res.status(expectation.status).json(expectation.body); } diff --git a/service/src/service-api.ts b/service/src/service-api.ts index 49516a29..6f9a87ed 100644 --- a/service/src/service-api.ts +++ b/service/src/service-api.ts @@ -7,6 +7,7 @@ import serviceRouter from './service/router'; import programmaticRouter from './service/programmatic-router'; import bridgeRouter from './bridge'; import workspaceToolsRouter from './workspace-tools'; +import { workspaceToolOutcomeLogging } from './workspace-tools/outcome'; import { connection } from './queue'; import { env } from './config'; import logger from './logger'; @@ -14,6 +15,7 @@ import hostedAppRouter from './hosted-app/router'; import { hostedAppPreviewGateway } from './hosted-app/preview-gateway'; const app = express(); +app.post('/v1/workspace-tools/execute', workspaceToolOutcomeLogging); app.disable('x-powered-by'); app.set('trust proxy', 1); app.use(executionProfileMiddleware); diff --git a/service/src/workspace-tools/outcome.ts b/service/src/workspace-tools/outcome.ts new file mode 100644 index 00000000..d965d515 --- /dev/null +++ b/service/src/workspace-tools/outcome.ts @@ -0,0 +1,79 @@ +import type { RequestHandler, Response } from 'express'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import logger from '../logger'; +import { env } from '../config'; +import { hostedAppRequestHostname, hostedAppRuntimeIdFromHostname } from '../hosted-app/preview-access'; + +interface WorkspaceToolOutcome { + operation?: WorkspaceToolRequest['operation']; + workerId?: string; + errorCode?: string; + deadlineBudgetMs?: number; + dispatchDurationMs?: number; + dispatchPending: boolean; + flush: () => void; +} + +const outcomes = new WeakMap(); +const earlyErrorCodes: Record = { + 400: 'INVALID_REQUEST', + 401: 'UNAUTHENTICATED', + 403: 'AUTHORIZATION_REJECTED', + 413: 'REQUEST_TOO_LARGE', + 415: 'UNSUPPORTED_MEDIA_TYPE', + 429: 'RATE_LIMITED', + 500: 'INTERNAL_ERROR', +}; + +export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { + const existing = outcomes.get(res); + if (existing != null) return existing; + const startedAt = performance.now(); + let responseEndedAt: number | undefined; + let logged = false; + const outcome: WorkspaceToolOutcome = { + dispatchPending: false, + flush: (): void => { + if (logged || responseEndedAt == null || outcome.dispatchPending) return; + logged = true; + outcomes.delete(res); + const finished = res.writableFinished; + logger.log(finished && res.statusCode < 400 ? 'info' : 'warn', 'Workspace tool request completed', { + route: '/workspace-tools/execute', + operation: outcome.operation, + workerId: outcome.workerId, + status: finished ? res.statusCode : undefined, + outcome: finished ? 'completed' : 'disconnected', + errorCode: outcome.errorCode ?? (finished ? earlyErrorCodes[res.statusCode] : undefined), + durationMs: Math.round(responseEndedAt - startedAt), + dispatchDurationMs: outcome.dispatchDurationMs, + deadlineBudgetMs: outcome.deadlineBudgetMs, + }); + }, + }; + const onResponseEnd = (): void => { + res.removeListener('finish', onResponseEnd); + res.removeListener('close', onResponseEnd); + responseEndedAt = performance.now(); + outcome.flush(); + }; + outcomes.set(res, outcome); + res.once('finish', onResponseEnd); + res.once('close', onResponseEnd); + return outcome; +} + +export function recordWorkspaceToolRejection(res: Response, errorCode: string): void { + const outcome = outcomes.get(res); + if (outcome != null) outcome.errorCode = errorCode; +} + +/** Classify raw Host like the preview gateway without moving the earlier profile guard. */ +export const workspaceToolOutcomeLogging: RequestHandler = (req, res, next): void => { + if (req.method !== 'POST') return next(); + const hostname = hostedAppRequestHostname(req.headers.host); + if (env.HOSTED_APPS_ENABLED && env.HOSTED_APP_PREVIEW_ORIGIN && hostname != null && + hostedAppRuntimeIdFromHostname(hostname, env.HOSTED_APP_PREVIEW_ORIGIN) != null) return next(); + getWorkspaceToolOutcome(res); + next(); +}; diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 5688d739..21aa86c7 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -1,18 +1,33 @@ import { createServer } from 'node:http'; import type { Server } from 'node:http'; -import { afterEach, expect, test } from 'bun:test'; +import { afterEach, beforeEach, expect, spyOn, test } from 'bun:test'; import express, { json } from 'express'; +import rateLimitFactory from 'express-rate-limit'; +import logger from '../logger'; +import { env } from '../config'; +import { apiKeyAuth } from '../middleware/auth'; +import { workspaceToolOutcomeLogging } from './outcome'; +import { executionProfileMiddleware } from '../middleware/execution-profile'; +import { hostedAppPreviewGateway } from '../hosted-app/preview-gateway'; import { applyPrincipal } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; import { bridgeStoreStatus, createWorkspaceToolsRouter } from './router'; let server: Server | undefined; +let logCompleted: ReturnType>; +let logSpy: ReturnType>; + +beforeEach(() => { + logCompleted = Promise.withResolvers(); + logSpy = spyOn(logger, 'log').mockImplementation(() => { logCompleted.resolve(); return logger; }); +}); afterEach(() => { server?.close(); server = undefined; + logSpy.mockRestore(); }); test('maps invalid worker results to an upstream failure', () => { @@ -66,6 +81,16 @@ test('rejects new workspace dispatches while the service is shutting down', asyn expect(response.status).toBe(503); expect(dispatched).toBe(false); + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith( + 'warn', + 'Workspace tool request completed', + expect.objectContaining({ + status: 503, + errorCode: 'SERVICE_SHUTTING_DOWN', + outcome: 'completed', + }), + ); }); test.each([ @@ -131,13 +156,33 @@ test.each([ }); expect(response.status).toBe(expectedStatus); - await expect(response.json()).resolves.toMatchObject({ code: errorCode }); + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith( + 'warn', + 'Workspace tool request completed', + expect.objectContaining({ + status: expectedStatus, + errorCode, + operation: 'search_text', + workerId: 'user-worker', + dispatchDurationMs: expect.any(Number), + deadlineBudgetMs: 30_000, + }), + ); + await expect(response.json()).resolves.toMatchObject({ + code: errorCode, + }); }); test('dispatches an authenticated workspace tool request to the principal-bound worker', async () => { let dispatchArgs: Record | undefined; const app = express(); app.use(json()); + app.use((_req, res, next) => { + const send = res.json.bind(res); + res.json = (body): typeof res => { setTimeout(() => send(body), 120); return res; }; + next(); + }); app.use((req, _res, next) => { applyPrincipal(req, { userId: 'user-1', @@ -199,6 +244,21 @@ test('dispatches an authenticated workspace tool request to the principal-bound }); expect(response.status).toBe(200); + const timing = logSpy.mock.calls[0].at(-1) as { durationMs: number; dispatchDurationMs: number }; + expect(timing.durationMs - timing.dispatchDurationMs).toBeGreaterThanOrEqual(100); + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith( + 'info', + 'Workspace tool request completed', + expect.objectContaining({ + status: 200, + operation: 'read_file', + workerId: 'user-worker', + outcome: 'completed', + }), + ); + expect(JSON.stringify(logSpy.mock.calls)).not.toContain('# LibreChat'); + expect(JSON.stringify(logSpy.mock.calls)).not.toContain('tenant-1'); await expect(response.json()).resolves.toMatchObject({ operation: 'read_file', content: '# LibreChat', @@ -210,3 +270,340 @@ test('dispatches an authenticated workspace tool request to the principal-bound request, }); }); + +test.each([ + ['WORKER_UNAUTHORIZED', 403], + ['ASSIGNMENT_INVALID', 400], + ['RESULT_INVALID', 502], + ['ASSIGNMENT_EXPIRED', 504], + ['WORKER_OFFLINE', 503], + ['WORKER_BUSY', 503], + ['WORKER_MISMATCH', 409], +] as const)('logs store rejection %s with actual HTTP %i', async (errorCode, expectedStatus) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { + userId: 'user-1', + tenantId: 'tenant-1', + principalSource: 'librechat_jwt', + codeWorkerId: 'user-worker', + }); + next(); + }); + app.use( + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'user-worker', + dynamicWorkers: true, + timeoutMs: 300_000, + store: { + async dispatchWorkspaceTool() { + throw new BridgeStoreError(errorCode, 'private diagnostic details'); + }, + }, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + }), + }); + expect(response.status).toBe(expectedStatus); + await response.text(); + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith( + 'warn', + 'Workspace tool request completed', + expect.objectContaining({ + operation: 'list_files', + workerId: 'user-worker', + status: expectedStatus, + errorCode, + outcome: 'completed', + deadlineBudgetMs: 300_000, + dispatchDurationMs: expect.any(Number), + }), + ); + expect(JSON.stringify(logSpy.mock.calls)).not.toContain('private diagnostic details'); +}); + +test.each([ + ['unauthenticated', 401, 'UNAUTHENTICATED'], + ['invalid request', 400, 'INVALID_WORKSPACE_TOOL_REQUEST'], + ['selection denied', 403, 'WORKER_SELECTION_REJECTED'], + ['invalid worker', 400, 'WORKER_SELECTION_REJECTED'], + ['no backend', 503, 'WORKSPACE_BACKEND_UNAVAILABLE'], +] as const)('logs early %s without dispatching', async (scenario, status, errorCode) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + const workerId = scenario === 'invalid worker' ? 'bad/worker' : 'user-worker'; + if (scenario !== 'unauthenticated') + applyPrincipal(req, { + userId: 'user-1', + tenantId: 'tenant-1', + principalSource: 'librechat_jwt', + codeWorkerId: + scenario === 'no backend' ? undefined : workerId, + }); + next(); + }); + app.use( + createWorkspaceToolsRouter({ + backend: scenario === 'no backend' ? 'http' : 'remote-bridge', + configuredWorkerId: 'user-worker', + dynamicWorkers: true, + store: { + async dispatchWorkspaceTool() { + throw new Error('Must not dispatch'); + }, + }, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(scenario === 'selection denied' ? { 'X-LibreChat-Code-Worker-ID': 'forged-worker' } : {}), + }, + body: JSON.stringify( + scenario === 'invalid request' + ? { operation: 'private-untrusted-operation' } + : { + protocolVersion: 1, + operation: 'list_files', + workspaceId: 'primary', + }, + ), + }); + expect(response.status).toBe(status); + await response.text(); + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith( + 'warn', + 'Workspace tool request completed', + expect.objectContaining({ + status, + errorCode, + dispatchDurationMs: undefined, + }), + ); + expect(JSON.stringify(logSpy.mock.calls)).not.toContain('forged-worker'); + expect(JSON.stringify(logSpy.mock.calls)).not.toContain('private-untrusted-operation'); +}); + +test('logs a disconnected dispatch once without inventing HTTP 200', async () => { + const app = express(); + const started = Promise.withResolvers(); + const closed = Promise.withResolvers(); + const settlementGate = Promise.withResolvers(); + let dispatchAborted = false; + let closeConnection = (): void => { throw new Error('connection not ready'); }; + app.use(json()); + app.use((req, res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + closeConnection = (): void => { res.destroy(); }; + res.once('close', () => closed.resolve()); + next(); + }); + app.use( + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'user-worker', + dynamicWorkers: true, + store: { + async dispatchWorkspaceTool({ signal }) { + started.resolve(); + return await new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + dispatchAborted = true; + void settlementGate.promise.then(() => reject(new BridgeStoreError('ASSIGNMENT_EXPIRED', 'caller left'))); + }, + { once: true }, + ); + }); + }, + }, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const controller = new AbortController(); + const response = fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }), + }); + await started.promise; + closeConnection(); + await expect(response).rejects.toThrow(); + await closed.promise; + expect(dispatchAborted).toBe(true); + expect(logSpy).not.toHaveBeenCalled(); + settlementGate.resolve(); + await logCompleted.promise; + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith( + 'warn', + 'Workspace tool request completed', + expect.objectContaining({ + outcome: 'disconnected', + errorCode: 'ASSIGNMENT_EXPIRED', + status: undefined, + operation: 'list_files', + workerId: 'user-worker', + }), + ); +}); + +test.each(['auth', 'limit'] as const)('logs requests rejected by upstream %s middleware', async (stage) => { + const originalLocalMode = env.LOCAL_MODE; + const originalProvider = process.env.CODEAPI_AUTH_PROVIDER; + env.LOCAL_MODE = false; + process.env.CODEAPI_AUTH_PROVIDER = 'librechat-jwt'; + try { + const app = express(); + app.post('/v1/workspace-tools/execute', workspaceToolOutcomeLogging); + app.use(json()); + if (stage === 'auth') app.use(apiKeyAuth); + else app.use(rateLimitFactory({ windowMs: 60_000, max: 1 })); + let reachedHandler = false; + app.post('/v1/workspace-tools/execute', (_req, res) => { + reachedHandler = true; + res.json({ ok: true }); + }); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const request = (): Promise => fetch(`http://127.0.0.1:${address.port}/v1/workspace-tools/execute`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}', + }); + if (stage === 'limit') { + await (await request()).text(); + logSpy.mockClear(); + reachedHandler = false; + } + const response = await request(); + await response.text(); + expect(response.status).toBe(stage === 'auth' ? 401 : 429); + expect(reachedHandler).toBe(false); + expect(logSpy.mock.calls.filter(([, message]) => message === 'Workspace tool request completed')).toHaveLength(1); + expect(logSpy).toHaveBeenCalledWith('warn', 'Workspace tool request completed', expect.objectContaining({ + status: response.status, errorCode: stage === 'auth' ? 'UNAUTHENTICATED' : 'RATE_LIMITED', + dispatchDurationMs: undefined, outcome: 'completed', + })); + } finally { + env.LOCAL_MODE = originalLocalMode; + if (originalProvider == null) delete process.env.CODEAPI_AUTH_PROVIDER; + else process.env.CODEAPI_AUTH_PROVIDER = originalProvider; + } +}); + +test.each([ + ['preview', undefined, 401, undefined], + ['preview', 'stateful', 409, undefined], + ['api', 'stateful', 409, 'execution_profile_mismatch'], + ['api', 'invalid', 400, 'invalid_execution_profile'], +] as const)('classifies %s host traffic with expected profile %s', async (hostKind, expectedProfile, status, errorCode) => { + const saved = { enabled: env.HOSTED_APPS_ENABLED, origin: env.HOSTED_APP_PREVIEW_ORIGIN, profile: env.EXECUTION_PROFILE }; + env.HOSTED_APPS_ENABLED = true; + env.HOSTED_APP_PREVIEW_ORIGIN = 'https://apps.example.test'; + env.EXECUTION_PROFILE = 'default'; + try { + const app = express(); + app.post('/v1/workspace-tools/execute', workspaceToolOutcomeLogging); + app.use(executionProfileMiddleware); + app.use(hostedAppPreviewGateway); + app.use(json()); + app.post('/v1/workspace-tools/execute', (_req, res) => { res.sendStatus(200); }); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/workspace-tools/execute`, { + method: 'POST', headers: { + 'Content-Type': 'application/json', + Host: hostKind === 'preview' ? `happ-${'a'.repeat(40)}.apps.example.test` : 'api.example.test', + ...(expectedProfile == null ? {} : { 'X-CodeAPI-Expected-Profile': expectedProfile }), + }, body: '{}', + }); + await response.text(); + expect(response.status).toBe(status); + if (hostKind === 'preview') { + expect(logSpy).not.toHaveBeenCalled(); + } else { + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith('warn', 'Workspace tool request completed', expect.objectContaining({ + status, errorCode, dispatchDurationMs: undefined, + })); + } + } finally { + env.HOSTED_APPS_ENABLED = saved.enabled; + env.HOSTED_APP_PREVIEW_ORIGIN = saved.origin; + env.EXECUTION_PROFILE = saved.profile; + } +}); + +test.each([ + ['POST', '/v1/workspace-tools/execute', true], + ['POST', '/v1/workspace-tools/execute/?attempt=1', true], + ['POST', '/v1/workspace-tools/execute/unknown', false], + ['POST', '/v1/workspace-tools/execute-extra', false], + ['GET', '/v1/workspace-tools/execute', false], +] as const)('logs only workspace endpoint traffic: %s %s', async (method, path, shouldLog) => { + const app = express(); + app.post('/v1/workspace-tools/execute', workspaceToolOutcomeLogging); + app.use((_req, res) => { res.sendStatus(401); }); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}${path}`, { method }); + await response.text(); + expect(response.status).toBe(401); + expect(logSpy).toHaveBeenCalledTimes(shouldLog ? 1 : 0); +}); + +test.each([ + ['unsupported encoding', 'application/json', 'unsupported', '{}', 415, 'UNSUPPORTED_MEDIA_TYPE'], + ['unsupported charset', 'application/json; charset=iso-8859-1', 'identity', '{}', 415, 'UNSUPPORTED_MEDIA_TYPE'], + ['invalid json', 'application/json', 'identity', '{', 400, 'INVALID_REQUEST'], + ['oversized json', 'application/json', 'identity', JSON.stringify({ content: 'x'.repeat(100) }), 413, 'REQUEST_TOO_LARGE'], +] as const)('classifies parser rejection: %s', async (_scenario, contentType, encoding, body, status, errorCode) => { + const app = express(); + app.post('/v1/workspace-tools/execute', workspaceToolOutcomeLogging); + app.use(json({ limit: 32 })); + app.post('/v1/workspace-tools/execute', (_req, res) => { res.sendStatus(200); }); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/workspace-tools/execute`, { + method: 'POST', headers: { 'Content-Type': contentType, 'Content-Encoding': encoding }, body, + }); + await response.text(); + expect(response.status).toBe(status); + expect(logSpy).toHaveBeenCalledTimes(1); + expect(logSpy).toHaveBeenCalledWith('warn', 'Workspace tool request completed', expect.objectContaining({ + status, errorCode, dispatchDurationMs: undefined, outcome: 'completed', + })); +}); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 7f563891..b2b56e06 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -4,6 +4,7 @@ import type { RequestHandler, Response } from 'express'; import type { AuthenticatedRequest } from '../types'; import type { RedisBridgeStore } from '../bridge/store'; +import { getWorkspaceToolOutcome } from './outcome'; import { getPrincipalOrReject } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; import { checkServiceShutDown } from '../lifecycle'; @@ -46,18 +47,27 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) router.post( '/workspace-tools/execute', asyncRoute(async (req, res) => { + const outcome = getWorkspaceToolOutcome(res); + const deadlineBudgetMs = Math.max(1, options.timeoutMs ?? 30_000); + outcome.deadlineBudgetMs = deadlineBudgetMs; const principal = getPrincipalOrReject(req, res); - if (!principal) return; + if (!principal) { + outcome.errorCode = 'UNAUTHENTICATED'; + return; + } if ((options.isShuttingDown ?? checkServiceShutDown)()) { + outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; res.status(503).json({ error: 'Service is shutting down' }); return; } if (!isWorkspaceToolRequest(req.body)) { + outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; res.status(400).json({ error: 'Invalid workspace tool request', }); return; } + outcome.operation = req.body.operation; let selection: { workerId: string; explicit: boolean } | undefined; try { @@ -70,36 +80,44 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) }); } catch (error) { if (error instanceof BridgeWorkerSelectionError) { + outcome.errorCode = 'WORKER_SELECTION_REJECTED'; res.status(error.status).json({ error: error.message }); return; } throw error; } if (selection == null) { + outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; res.status(503).json({ error: 'Workspace tools require the remote-bridge backend', }); return; } + outcome.workerId = selection.workerId; const controller = new AbortController(); - const abort = () => controller.abort(); + const abort = (): void => controller.abort(); req.once('aborted', abort); - const abortClosedResponse = () => { + const abortClosedResponse = (): void => { if (!res.writableEnded) abort(); }; res.once('close', abortClosedResponse); try { + outcome.dispatchPending = true; + const dispatchStartedAt = performance.now(); const settlement = await options.store.dispatchWorkspaceTool({ workerId: selection.workerId, tenantId: principal.tenantId, requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), request: req.body, - deadlineAtMs: Date.now() + Math.max(1, options.timeoutMs ?? 30_000), + deadlineAtMs: Date.now() + deadlineBudgetMs, signal: controller.signal, + }).finally(() => { + outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); }); if (settlement.status === 'rejected') { + outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; let status = 422; if ( settlement.errorCode === 'SEARCH_TIMEOUT' || @@ -129,14 +147,18 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) res.status(200).json(settlement.result); } catch (error) { if (error instanceof BridgeStoreError) { + outcome.errorCode = error.code; res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code, }); return; } + outcome.errorCode = 'INTERNAL_ERROR'; throw error; } finally { + outcome.dispatchPending = false; + outcome.flush(); req.removeListener('aborted', abort); res.removeListener('close', abortClosedResponse); } From b243c93f092baa0c1bf42f3f9ae6f74b3213d6b7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 08:39:26 -0400 Subject: [PATCH 064/116] =?UTF-8?q?=F0=9F=9B=9F=20fix:=20Recover=20Hosted?= =?UTF-8?q?=20App=20Stop=20Failures=20(#155)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: recover hosted app stop failures * fix: clear recovered stop error status * test: preserve startup failure details --- api/src/hosted-app.test.ts | 58 +++++++++++++++++++++++- api/src/hosted-app.ts | 93 +++++++++++++++++++++----------------- 2 files changed, 108 insertions(+), 43 deletions(-) diff --git a/api/src/hosted-app.test.ts b/api/src/hosted-app.test.ts index 727803d1..2d4ae9b1 100644 --- a/api/src/hosted-app.test.ts +++ b/api/src/hosted-app.test.ts @@ -337,7 +337,9 @@ describe('HostedAppSupervisor', () => { expect(error).toBeInstanceOf(HostedAppError); expect(error.code).toBe('hosted_app_start_failed'); + expect(error.message).toBe('hosted app exited'); expect(supervisor.status()?.state).toBe('failed'); + expect(supervisor.status()?.message).toBe('hosted app exited'); }); test('serializes quiesced workspace access and rejects it while an app is running', async () => { @@ -424,9 +426,63 @@ describe('HostedAppSupervisor', () => { expect(error).toBeInstanceOf(HostedAppError); expect(error.code).toBe('hosted_app_cleanup_failed'); expect(error.status).toBe(503); + expect(supervisor.status()).toMatchObject({ + state: 'failed', + message: 'hosted app cleanup failed', + }); permitCleanup = true; - await supervisor.shutdown(); + const recovered = await supervisor.stop(); + expect(recovered).toMatchObject({ state: 'stopped' }); + expect(recovered).not.toHaveProperty('message'); + }); + + test('allows checkpoint and restore to retry cleanup after stop fails', async () => { + const root = await workspace(); + let permitCleanup = false; + const fixture = dependencies(root, { + killCgroup: async () => { + if (!permitCleanup) throw new Error('cgroup remains populated'); + }, + }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + await expect(supervisor.stop()).rejects.toMatchObject({ + code: 'hosted_app_cleanup_failed', + status: 503, + }); + const operations: string[] = []; + + await expect(supervisor.withQuiescedWorkspace(async () => { + operations.push('unsafe checkpoint'); + })).rejects.toMatchObject({ + code: 'hosted_app_cleanup_failed', + status: 503, + }); + expect(operations).toEqual([]); + + permitCleanup = true; + await supervisor.withQuiescedWorkspace(async () => { operations.push('checkpoint'); }); + await supervisor.withQuiescedWorkspace(async () => { operations.push('restore'); }); + + expect(operations).toEqual(['checkpoint', 'restore']); + expect(fixture.cgroupKills).toHaveLength(3); + }); + + test('surfaces replacement cleanup failure as retryable without spawning', async () => { + const root = await workspace(); + const fixture = dependencies(root, { + killCgroup: async () => { throw new Error('cgroup remains populated'); }, + }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + const error = await supervisor.start(request({ revision: 'rev-2' })).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error).toMatchObject({ code: 'hosted_app_cleanup_failed', status: 503 }); + expect(supervisor.status()?.state).toBe('failed'); + expect(fixture.spawns).toHaveLength(1); }); test('fails workspace mutation closed until a failed app cgroup is drained', async () => { diff --git a/api/src/hosted-app.ts b/api/src/hosted-app.ts index d8cbfc50..64a5aab8 100644 --- a/api/src/hosted-app.ts +++ b/api/src/hosted-app.ts @@ -458,18 +458,7 @@ export class HostedAppSupervisor { } async stop(): Promise { - return this.serialize(async () => { - try { - return await this.stopImpl(); - } catch (error) { - logger.error({ err: error }, 'Hosted-app stop cleanup failed'); - throw new HostedAppError( - 'hosted_app_cleanup_failed', - 'the hosted app could not be stopped safely', - 503, - ); - } - }); + return this.serialize(() => this.stopImpl()); } async shutdown(): Promise { @@ -729,42 +718,62 @@ export class HostedAppSupervisor { private async stopImpl(preserveActive = false): Promise { const active = this.active; if (!active) return undefined; - const child = active.process; - if (!child?.pid) { + try { + const child = active.process; + if (!child?.pid) { + await this.deps.killCgroup(); + active.cgroupDrained = true; + active.status.state = 'stopped'; + if (!preserveActive) delete active.status.message; + active.status.exited_at ??= this.deps.now().toISOString(); + if (!preserveActive) this.active = undefined; + return publicStatus(active); + } + + active.status.state = 'stopping'; + this.deps.killProcessGroup(child.pid, 'SIGTERM'); + const exited = new Promise(resolve => child.once('exit', () => resolve(true))); + let timer: ReturnType | undefined; + const timedOut = new Promise(resolve => { + timer = setTimeout(() => resolve(false), config.hosted_app_stop_timeout_ms); + timer.unref?.(); + }); + const stopped = await Promise.race([exited, timedOut]); + if (timer) clearTimeout(timer); + if (!stopped && active.process?.pid) { + await this.deps.killCgroup(); + await Promise.race([ + new Promise(resolve => child.once('exit', () => resolve())), + new Promise(resolve => setTimeout(resolve, config.hosted_app_stop_timeout_ms)), + ]); + } + /* Always sweep the cgroup: the tracked parent may have exited cleanly + * while a daemonized descendant stayed alive in a different process group. */ await this.deps.killCgroup(); active.cgroupDrained = true; active.status.state = 'stopped'; + if (!preserveActive) delete active.status.message; active.status.exited_at ??= this.deps.now().toISOString(); + const status = publicStatus(active); if (!preserveActive) this.active = undefined; - return publicStatus(active); - } - - active.status.state = 'stopping'; - this.deps.killProcessGroup(child.pid, 'SIGTERM'); - const exited = new Promise(resolve => child.once('exit', () => resolve(true))); - let timer: ReturnType | undefined; - const timedOut = new Promise(resolve => { - timer = setTimeout(() => resolve(false), config.hosted_app_stop_timeout_ms); - timer.unref?.(); - }); - const stopped = await Promise.race([exited, timedOut]); - if (timer) clearTimeout(timer); - if (!stopped && active.process?.pid) { - await this.deps.killCgroup(); - await Promise.race([ - new Promise(resolve => child.once('exit', () => resolve())), - new Promise(resolve => setTimeout(resolve, config.hosted_app_stop_timeout_ms)), - ]); + return status; + } catch (error) { + /* Keep the failed record so checkpoint/restore can retry the cgroup + * sweep. A `stopping` record would permanently reject those operations + * before they reach the recoverable cleanup path. */ + active.cgroupDrained = false; + active.status.state = 'failed'; + active.status.message = 'hosted app cleanup failed'; + logger.error( + { err: error, appId: active.request.app_id }, + 'Hosted-app stop cleanup failed', + ); + throw new HostedAppError( + 'hosted_app_cleanup_failed', + 'the hosted app could not be stopped safely', + 503, + ); } - /* Always sweep the cgroup: the tracked parent may have exited cleanly - * while a daemonized descendant stayed alive in a different process group. */ - await this.deps.killCgroup(); - active.cgroupDrained = true; - active.status.state = 'stopped'; - active.status.exited_at ??= this.deps.now().toISOString(); - const status = publicStatus(active); - if (!preserveActive) this.active = undefined; - return status; } } From 1d556c023e51a49f9a37e05da6cb6d952d5f1d5e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 08:39:37 -0400 Subject: [PATCH 065/116] fix: encode the KVM resolver handoff for the kernel command line (#157) libkrun appends every guest environment entry to the kernel command line, which linux-loader restricts to single-line printable ASCII. Since #152 the launcher entrypoint exported the runner's whole /etc/resolv.conf, so any Docker-generated file (comments, blank lines, options) made krun_start_enter panic with InvalidAscii and sandbox-runner restart-looped. Forward only the resolver directives, whitespace-normalized and joined by '|', and expand them back into lines in the guest wrapper. Validate every forwarded variable in the launcher against the command line's charset, the kernel's quote handling and its 2048-byte size limit, failing with a named error before libkrun can panic. Run the launcher's unit tests in CI. --- .github/workflows/ci.yml | 18 ++++ README.md | 9 ++ api/src/guest-dns.sh | 10 +- launcher/entrypoint.sh | 31 +++++- launcher/src/main.rs | 199 +++++++++++++++++++++++++++++++++++++-- tests/kvm_guest_dns.sh | 110 +++++++++++++++++++--- 6 files changed, 351 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f79328e..71536caa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,24 @@ jobs: docker buildx build --check -f api/Dockerfile . docker buildx build --check -f docker/Dockerfile.worker-sandbox . + launcher-unit-tests: + name: Launcher Unit Tests + runs-on: ubuntu-latest + container: fedora:43 + defaults: + run: + working-directory: launcher + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Install Rust and libkrun + # Mirrors launcher/Dockerfile's builder stage; libkrun is only packaged + # for Fedora, and the guest-environment checks link against it. + run: dnf install -y --setopt=install_weak_deps=False rust cargo libkrun-devel gcc + + - name: Cargo tests + run: cargo test + api-unit-tests: name: API Unit Tests runs-on: ubuntu-latest diff --git a/README.md b/README.md index 1e6432f7..7bff76fb 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,15 @@ read-only root disk does not need modification at boot. Rebuild the runner image to pick up this layout change. A missing resolver handoff fails startup rather than leaving the guest with an unrelated public DNS server. +libkrun delivers the guest environment on the kernel command line, which only +carries single-line printable ASCII and is capped at 2048 bytes by the guest +kernel. The launcher entrypoint therefore forwards only the `nameserver`, +`search`, `domain`, `options` and `sortlist` directives, joined by `|`, and the +guest wrapper expands them back into `/etc/resolv.conf` lines. The launcher +rejects any forwarded variable that would not survive that trip (control +characters, non-ASCII bytes, quoting the kernel would split, or an oversized +environment) with a named error instead of a libkrun panic and restart loop. + To validate a deployment, execute code that creates a file in `/mnt/data`, confirm the response includes its file reference, and download it. Recreate the egress gateway with a different container IP while leaving the runner alive, diff --git a/api/src/guest-dns.sh b/api/src/guest-dns.sh index 9795ca7f..d8371444 100644 --- a/api/src/guest-dns.sh +++ b/api/src/guest-dns.sh @@ -2,6 +2,10 @@ # The guest root may be read-only. Bake the link, populate its target only # after /run is mounted, and leave direct NsJail/Lambda resolvers untouched. +# launcher/entrypoint.sh joins resolver directives with this separator because +# the handoff rides the guest kernel command line, which cannot carry newlines. +RESOLV_FIELD_SEPARATOR='|' + prepare_guest_dns() { local root="$1" mkdir -p "$root/run" @@ -12,18 +16,20 @@ prepare_guest_dns() { configure_guest_dns() { local root="${1:-}" local target="$root/run/codeapi-resolver" + local resolv_conf="${SANDBOX_RESOLV_CONF:-}" + resolv_conf="${resolv_conf//"$RESOLV_FIELD_SEPARATOR"/$'\n'}" if [ ! -L "$root/etc/resolv.conf" ] || \ [ "$(readlink "$root/etc/resolv.conf")" != '../run/codeapi-resolver/resolv.conf' ]; then return 0 fi - if ! printf '%s\n' "${SANDBOX_RESOLV_CONF:-}" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then + if ! printf '%s\n' "$resolv_conf" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then echo 'ERROR: KVM guest requires resolver configuration from launcher-entrypoint.sh' >&2 return 1 fi # A fresh, root-owned directory prevents a sandbox UID from replacing DNS # configuration in the runtime mount. Never reuse a pre-existing entry. (umask 077; mkdir "$target") || return 1 - printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$target/resolv.conf" || return 1 + printf '%s\n' "$resolv_conf" > "$target/resolv.conf" || return 1 chmod 600 "$target/resolv.conf" || return 1 unset SANDBOX_RESOLV_CONF } diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 0369ec2a..304858e4 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -5,8 +5,35 @@ set -e # names intact so new connections can resolve replacements after a restart. # Forward the resolver and search domains supplied by Docker or Kubernetes, # rather than pinning endpoint IPs or baking a deployment-specific nameserver. -export SANDBOX_RESOLV_CONF="$(cat /etc/resolv.conf)" -if ! printf '%s\n' "$SANDBOX_RESOLV_CONF" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then +# +# libkrun places every guest environment entry on the kernel command line, +# which accepts only single-line printable ASCII and is truncated by the guest +# kernel past 2048 bytes. Keep the resolver directives alone, one per field, +# joined by a separator that api/src/guest-dns.sh expands back into lines. +RESOLV_FIELD_SEPARATOR='|' + +encode_resolv_conf() { + local LC_ALL=C + local line words encoded='' + while IFS= read -r line || [ -n "$line" ]; do + line="${line%$'\r'}" + if [[ ! "$line" =~ ^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]] ]]; then + continue + fi + read -ra words <<< "$line" + line="${words[*]}" + if [[ "$line" == *[!' '-'~']* || "$line" == *[\"$RESOLV_FIELD_SEPARATOR]* ]]; then + echo "ERROR: runner /etc/resolv.conf line cannot cross the kernel command line: $line" >&2 + return 1 + fi + encoded+="${encoded:+$RESOLV_FIELD_SEPARATOR}$line" + done + printf '%s' "$encoded" +} + +SANDBOX_RESOLV_CONF="$(encode_resolv_conf < /etc/resolv.conf)" +export SANDBOX_RESOLV_CONF +if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATOR}nameserver "[!\#]* ]]; then echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2 exit 1 fi diff --git a/launcher/src/main.rs b/launcher/src/main.rs index cbc4b5fc..02d26418 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -479,6 +479,92 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { false } +/// The guest kernel keeps at most this many command-line bytes (`COMMAND_LINE_SIZE` +/// on x86-64 and aarch64). libkrun on x86-64 assembles a longer line without +/// complaint and the kernel silently drops the tail, which carries the guest +/// environment and the `-- ` epilog. +const GUEST_CMDLINE_LIMIT: usize = 2048; +/// libkrun's own entries ahead of the environment: its default kernel +/// parameters, `init=/init.krun`, `KRUN_INIT`, the block-root and rlimit +/// entries and `tsi_hijack`. About 260 bytes for this launcher; the reserve +/// leaves headroom for libkrun to grow. +const LIBKRUN_CMDLINE_RESERVE: usize = 384; + +/// libkrun wraps each entry in double quotes and joins them with spaces. +fn quoted_cmdline_len<'a>(items: impl Iterator) -> usize { + items.map(|item| item.len() + 3).sum() +} + +/// libkrun hands the guest its environment on the kernel command line, one +/// double-quoted `KEY=VALUE` token per entry. linux-loader rejects anything +/// outside printable ASCII with an `InvalidAscii` panic before boot. The guest +/// kernel's `next_arg()` then toggles quoting on every double quote inside the +/// token, ends it at the first unquoted space, and strips a leading quote from +/// the value, so an entry only survives when its quotes balance, no space is +/// left unquoted, and the value does not open with a quote. +fn guest_env_entry_problem(key: &str, value: &str) -> Option { + let mut in_quote = true; + for (offset, byte) in value.bytes().enumerate() { + match byte { + b'"' => in_quote = !in_quote, + b' ' if !in_quote => { + return Some(format!( + "{key} has a space at offset {offset} outside balanced double quotes; the guest kernel splits the entry there" + )); + } + b' '..=b'~' => {} + _ => { + return Some(format!( + "{key} contains byte 0x{byte:02x} at offset {offset}; guest environment entries travel on the kernel command line as single-line printable ASCII" + )); + } + } + } + if !in_quote { + return Some(format!( + "{key} has unbalanced double quotes; the guest kernel merges the next entry into it" + )); + } + if value.starts_with('"') { + return Some(format!( + "{key} starts with a double quote, which the guest kernel strips from the value" + )); + } + None +} + +fn guest_cmdline_problem(env: &[(String, String)], args: &[String]) -> Option { + if let Some(problem) = env + .iter() + .find_map(|(key, value)| guest_env_entry_problem(key, value)) + { + return Some(problem); + } + + let entries: Vec = env.iter().map(|(key, value)| format!("{key}={value}")).collect(); + let env_bytes = quoted_cmdline_len(entries.iter().map(String::as_str)); + let args_bytes = " -- ".len() + quoted_cmdline_len(args.iter().map(String::as_str)); + let budget = GUEST_CMDLINE_LIMIT.saturating_sub(LIBKRUN_CMDLINE_RESERVE + args_bytes); + if env_bytes <= budget { + return None; + } + + let mut sizes: Vec<(&str, usize)> = env + .iter() + .map(|(key, value)| (key.as_str(), value.len())) + .collect(); + sizes.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); + let largest: Vec = sizes + .iter() + .take(3) + .map(|(key, len)| format!("{key} ({len} bytes)")) + .collect(); + Some(format!( + "guest environment needs {env_bytes} bytes on the kernel command line but only {budget} fit under the {GUEST_CMDLINE_LIMIT}-byte kernel limit; largest entries: {}", + largest.join(", ") + )) +} + fn main() { let vcpus: u8 = env::var("LAUNCHER_VCPUS") .ok() @@ -528,19 +614,28 @@ fn main() { let egress_gateway_enabled = env::var("EGRESS_GATEWAY_URL") .map(|value| !value.trim().is_empty()) .unwrap_or(false); - let env_strs: Vec = env::vars() + let guest_env: Vec<(String, String)> = env::vars() .filter(|(k, _)| !k.starts_with("LAUNCHER_")) .filter(|(k, _)| is_allowed_guest_env_key(k, egress_gateway_enabled)) + .collect(); + // krun_set_exec supplies argv[0]; this array contains arguments only. + let guest_args: Vec = vec![ + "/sandbox_api/guest-dns.sh".into(), + "--exec".into(), + exec_path.clone(), + ]; + if let Some(problem) = guest_cmdline_problem(&guest_env, &guest_args) { + eprintln!("[launcher] ERROR: {problem}"); + process::exit(1); + } + + let env_strs: Vec = guest_env + .iter() .map(|(k, v)| cstr(&format!("{k}={v}"))) .collect(); let env_ptrs = null_term(&env_strs); - // krun_set_exec supplies argv[0]; this array contains arguments only. - let argv_strs: Vec = vec![ - cstr("/sandbox_api/guest-dns.sh"), - cstr("--exec"), - cstr(&exec_path), - ]; + let argv_strs: Vec = guest_args.iter().map(|arg| cstr(arg)).collect(); let argv_ptrs = null_term(&argv_strs); let rlimit_strs: Vec = vec![guest_nofile_rlimit(nofile_target)]; @@ -618,7 +713,95 @@ fn main() { #[cfg(test)] mod tests { - use super::{desired_nofile_soft_limit, guest_nofile_rlimit, is_allowed_guest_env_key}; + use super::{ + desired_nofile_soft_limit, guest_cmdline_problem, guest_nofile_rlimit, + is_allowed_guest_env_key, GUEST_CMDLINE_LIMIT, LIBKRUN_CMDLINE_RESERVE, + }; + + fn guest_args() -> Vec { + vec![ + "/sandbox_api/guest-dns.sh".into(), + "--exec".into(), + "/sandbox_api/entrypoint.sh".into(), + ] + } + + fn env(entries: &[(&str, &str)]) -> Vec<(String, String)> { + entries + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect() + } + + #[test] + fn guest_cmdline_accepts_single_line_printable_env() { + let env = env(&[ + ("SANDBOX_RESOLV_CONF", "nameserver 127.0.0.11|options ndots:0"), + ("EGRESS_GATEWAY_URL", "http://egress_gateway:3190"), + ("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin"), + ]); + assert_eq!(guest_cmdline_problem(&env, &guest_args()), None); + } + + #[test] + fn guest_cmdline_rejects_multi_line_resolv_conf_before_libkrun() { + let env = env(&[("SANDBOX_RESOLV_CONF", "# Generated by Docker Engine.\nnameserver 127.0.0.11\noptions ndots:0")]); + let problem = guest_cmdline_problem(&env, &guest_args()).expect("newline must be rejected"); + assert!(problem.starts_with("SANDBOX_RESOLV_CONF contains byte 0x0a at offset "), "{problem}"); + } + + #[test] + fn guest_cmdline_rejects_control_and_non_ascii_bytes() { + for value in ["a\tb", "caf\u{e9}", "\u{7f}", "a\rb", "\u{1b}[0m"] { + let env = env(&[("SANDBOX_LIMIT_OVERRIDES", value)]); + let problem = guest_cmdline_problem(&env, &guest_args()).expect("non-printable bytes must be rejected"); + assert!(problem.starts_with("SANDBOX_LIMIT_OVERRIDES contains byte 0x"), "{problem}"); + } + } + + #[test] + fn guest_cmdline_keeps_quotes_the_kernel_parser_preserves() { + for value in ["{\"python\":{\"run_timeout\":30}}", "{\"python\": {\"run_timeout\": 30}}", "plain words with spaces"] { + let env = env(&[("SANDBOX_LIMIT_OVERRIDES", value)]); + assert_eq!(guest_cmdline_problem(&env, &guest_args()), None, "{value}"); + } + } + + #[test] + fn guest_cmdline_rejects_quotes_that_split_or_merge_kernel_tokens() { + let cases = [ + ("{\"a b\":1}", "SANDBOX_LIMIT_OVERRIDES has a space at offset 3 outside balanced double quotes;"), + ("say \"hi", "SANDBOX_LIMIT_OVERRIDES has unbalanced double quotes;"), + ("\"quoted\"", "SANDBOX_LIMIT_OVERRIDES starts with a double quote,"), + ]; + for (value, expected) in cases { + let env = env(&[("SANDBOX_LIMIT_OVERRIDES", value)]); + let problem = guest_cmdline_problem(&env, &guest_args()).expect(value); + assert!(problem.starts_with(expected), "{problem}"); + } + } + + #[test] + fn guest_cmdline_rejects_env_that_overflows_the_kernel_limit() { + let key = "SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY"; + let big = "A".repeat(1_700); + let env = env(&[(key, big.as_str()), ("SANDBOX_RESOLV_CONF", "nameserver 127.0.0.11")]); + let problem = guest_cmdline_problem(&env, &guest_args()).expect("oversized env must be rejected"); + assert!(problem.contains(&format!("under the {GUEST_CMDLINE_LIMIT}-byte kernel limit")), "{problem}"); + assert!(problem.contains(&format!("largest entries: {key} (1700 bytes), SANDBOX_RESOLV_CONF (21 bytes)")), "{problem}"); + } + + #[test] + fn guest_cmdline_budget_accounts_for_libkrun_reserve_and_args() { + let args = guest_args(); + let args_bytes = " -- ".len() + args.iter().map(|arg| arg.len() + 3).sum::(); + let budget = GUEST_CMDLINE_LIMIT - LIBKRUN_CMDLINE_RESERVE - args_bytes; + let key = "SANDBOX_LIMIT_OVERRIDES"; + let exact = "A".repeat(budget - key.len() - "=".len() - 3); + assert_eq!(guest_cmdline_problem(&env(&[(key, exact.as_str())]), &args), None); + let over = format!("{exact}A"); + assert!(guest_cmdline_problem(&env(&[(key, over.as_str())]), &args).is_some()); + } #[test] fn guest_env_allowlist_blocks_control_plane_and_secret_vars() { diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index e00e3e38..4c073de2 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -26,6 +26,14 @@ printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_DIR/expected" configure_guest_dns "$TEST_DIR/guest" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +# The launcher entrypoint joins directives with a separator so the handoff +# survives the kernel command line; the guest expands it back into lines. +rm -rf "$TEST_DIR/guest/run/codeapi-resolver" +SANDBOX_RESOLV_CONF='nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' +configure_guest_dns "$TEST_DIR/guest" +cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +[[ ! -v SANDBOX_RESOLV_CONF ]] + # Never reuse a stale directory or follow an attacker-controlled runtime link. SANDBOX_RESOLV_CONF='nameserver 127.0.0.11' if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then @@ -47,6 +55,10 @@ SANDBOX_RESOLV_CONF='# no nameserver' if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then echo 'accepted empty guest resolver' >&2; exit 1 fi +SANDBOX_RESOLV_CONF='search example.com|options ndots:0' +if configure_guest_dns "$TEST_DIR/guest" 2>/dev/null; then + echo 'accepted encoded guest resolver without nameserver' >&2; exit 1 +fi # Direct NsJail and Lambda retain the resolver managed by their container. mkdir -p "$TEST_DIR/direct/etc" @@ -71,16 +83,81 @@ cat > "$TEST_DIR/bin/getent" <<'STUB' printf '192.0.2.99 stale-address\n' STUB chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" -sed "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" -PATH="$TEST_DIR/bin:$PATH" \ -EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ -FILE_SERVER_URL='http://[::1]:3000/base' \ -SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ -LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ -TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ -bash "$TEST_DIR/entrypoint.sh" -printf '%s\n' "$(cat /etc/resolv.conf)" > "$TEST_DIR/expected" -cmp "$TEST_DIR/expected" "$TEST_DIR/forwarded" +# libkrun appends every guest environment entry to the kernel command line, +# which linux-loader limits to single-line printable ASCII. Docker's generated +# resolv.conf is multi-line with comments, so the entrypoint must forward only +# the resolver directives, joined by the separator guest-dns.sh expands. +run_entrypoint() { + local resolv_conf="$1" + sed -e "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" \ + -e "s|/etc/resolv.conf|$resolv_conf|g" \ + "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" + rm -f "$TEST_DIR/forwarded" + PATH="$TEST_DIR/bin:$PATH" \ + EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ + FILE_SERVER_URL='http://[::1]:3000/base' \ + SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ + LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ + TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ + bash "$TEST_DIR/entrypoint.sh" +} +# Independent reference for the expected handoff: directives only, whitespace +# collapsed, joined by the separator. +encoded_reference() { + LC_ALL=C awk '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1; print }' "$1" | paste -sd '|' - +} +cat > "$TEST_DIR/docker-resolv.conf" <<'RESOLV' +# Generated by Docker Engine. +# This file can be edited; Docker Engine will not make further changes once it +# has been modified. + +nameserver 127.0.0.11 +options ndots:0 + +# Based on host file: '/etc/resolv.conf' (internal resolver) +# ExtServers: [host(10.255.255.254)] +# Overrides: [] +# Option ndots from: internal +RESOLV +run_entrypoint "$TEST_DIR/docker-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 127.0.0.11|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/docker-resolv.conf")" ]] +[[ "$(wc -l < "$TEST_DIR/forwarded")" == 1 ]] +if LC_ALL=C grep -q '[^ -~]' "$TEST_DIR/forwarded"; then + echo 'forwarded resolver contains bytes the kernel command line rejects' >&2; exit 1 +fi +# The guest restores exactly the directives Docker supplied. +rm -rf "$TEST_DIR/guest/run/codeapi-resolver" +SANDBOX_RESOLV_CONF="$(cat "$TEST_DIR/forwarded")" configure_guest_dns "$TEST_DIR/guest" +printf 'nameserver 127.0.0.11\noptions ndots:0\n' > "$TEST_DIR/expected" +cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" + +# Kubernetes resolvers: tabs, CRLF, trailing spaces, and unknown keywords are +# normalized away; search domains and options survive intact. +printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" +run_entrypoint "$TEST_DIR/k8s-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] + +# The runner's own resolver must round-trip through the same reference. +run_entrypoint /etc/resolv.conf +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference /etc/resolv.conf)" ]] + +# Content that cannot cross the kernel command line fails before the launcher +# starts, instead of a libkrun InvalidAscii panic and a restart loop. +for bad in $'nameserver 1.1.1.1\nsearch caf\xc3\xa9.example\n' $'nameserver 1.1.1.1\nsearch a|b\n' $'nameserver 1.1.1.1\noptions "ndots:1"\n'; do + printf '%s' "$bad" > "$TEST_DIR/bad-resolv.conf" + if run_entrypoint "$TEST_DIR/bad-resolv.conf" 2> "$TEST_DIR/entrypoint-error"; then + echo 'forwarded resolver content the kernel command line cannot carry' >&2; exit 1 + fi + grep -q 'cannot cross the kernel command line' "$TEST_DIR/entrypoint-error" + [[ ! -e "$TEST_DIR/forwarded" ]] +done +printf '# comments only\nsearch example.com\n' > "$TEST_DIR/bad-resolv.conf" +if run_entrypoint "$TEST_DIR/bad-resolv.conf" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a nameserver' >&2; exit 1 +fi +grep -q 'has no nameserver' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" ]] # Every rootfs assembly path must prepare DNS after COPY, before disk creation. python3 - "$ROOT" <<'PY' @@ -98,13 +175,18 @@ for name, count in [('api/Dockerfile', 2), ('docker/Dockerfile.worker-sandbox', assert stage.index('--prepare-rootfs /sandbox-rootfs') < stage.index('/usr/local/bin/build-rootfs-image.sh /sandbox-rootfs /sandbox-rootfs.img'), name text = (root / 'launcher/src/main.rs').read_text() assert '"SANDBOX_RESOLV_CONF"' in text.split('const ALLOW_EXACT:')[1].split('];')[0] -argv = text.split('let argv_strs:')[1].split('let argv_ptrs:')[0] +argv = text.split('let guest_args:')[1].split('let env_strs:')[0] # libkrun init supplies argv[0]. Repeating the binary here makes Bash try to # interpret /bin/bash itself as a shell script instead of the DNS wrapper. -assert 'cstr("/bin/bash")' not in argv -assert 'cstr("/sandbox_api/guest-dns.sh")' in argv -assert 'cstr("--exec")' in argv and 'cstr(&exec_path)' in argv +assert '"/bin/bash"' not in argv +assert '"/sandbox_api/guest-dns.sh".into()' in argv +assert '"--exec".into()' in argv and 'exec_path.clone()' in argv assert 'let exec_c = cstr("/bin/bash")' in text +assert 'guest_args.iter().map(|arg| cstr(arg))' in text +# Every forwarded entry is checked against the kernel command line's charset, +# quoting and size rules before libkrun can panic on it. +assert text.index('guest_cmdline_problem(&guest_env, &guest_args)') < text.index('ffi::krun_set_exec(') +assert text.index('is_allowed_guest_env_key(k, egress_gateway_enabled)') < text.index('guest_cmdline_problem(&guest_env, &guest_args)') PY # The wrapper configures DNS before a custom guest executable, independently # of the normal API entrypoint and its later /tmp mount. From 3842a7b83c071cef58e3fd06cf103da8bb0f75c9 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 15:37:25 -0400 Subject: [PATCH 066/116] =?UTF-8?q?=F0=9F=9A=90=20fix:=20Queue=20Contended?= =?UTF-8?q?=20BYOM=20Workspace=20Requests=20(#159)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/byom-worker-admission.md | 26 ++++ service/src/bridge/admission.test.ts | 37 +++++ service/src/bridge/admission.ts | 90 +++++++++++ service/src/bridge/store.ts | 79 ++++++++-- service/src/bridge/worker-admission.test.ts | 144 ++++++++++++++++++ .../src/sandbox-backend/remote-bridge.test.ts | 1 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/workspace-tools/router.test.ts | 1 + service/src/workspace-tools/router.ts | 1 + 9 files changed, 370 insertions(+), 10 deletions(-) create mode 100644 docs/byom-worker-admission.md create mode 100644 service/src/bridge/admission.test.ts create mode 100644 service/src/bridge/admission.ts create mode 100644 service/src/bridge/worker-admission.test.ts diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md new file mode 100644 index 00000000..c88d5669 --- /dev/null +++ b/docs/byom-worker-admission.md @@ -0,0 +1,26 @@ +# BYOM worker admission + +Workspace tool calls to a busy worker wait in a bounded FIFO shared through Redis. +The limit is 32 admitted requests per worker, including the active request. When +the limit is reached, the workspace endpoint returns HTTP 429 with +`WORKER_QUEUE_FULL`. A different worker has an independent admission queue. + +Waiting uses the caller's existing absolute dispatch deadline. It does not reset +or extend execution timeouts. Disconnecting or cancelling removes the waiting +request without cancelling the active assignment. Expired entries are pruned; +Redis key expiry also bounds state left by a crashed API process. + +After admission, the API revalidates the worker incarnation, identity, tenant +binding and workspace operation. A waiting request cannot migrate to a replacement +worker. Existing execution acknowledgement, fencing, settlement and quarantine +rules remain responsible for the active assignment. + +This is compatible with existing workers and requires only a Code API update. +Existing workers still execute one assignment at a time. Parallel execution across +workspaces requires separate lease claims and isolated native sandbox contexts; +this admission change does not advertise that capability. Queue time and execution +time currently share the HTTP request deadline. Separate budgets require a matching +LibreChat client change so that the client does not disconnect while waiting. + +Focused regression coverage lives in `service/src/bridge/admission.test.ts` and +`service/src/bridge/worker-admission.test.ts`. diff --git a/service/src/bridge/admission.test.ts b/service/src/bridge/admission.test.ts new file mode 100644 index 00000000..b2268012 --- /dev/null +++ b/service/src/bridge/admission.test.ts @@ -0,0 +1,37 @@ +import { afterEach, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { BridgeAdmissionQueue } from './admission'; + +const redis = new RedisMock() as unknown as Redis; +afterEach(async () => { + await redis.flushall(); +}); + +test('bounds FIFO admission across API replicas and releases cancelled waiters', async () => { + const firstReplica = new BridgeAdmissionQueue(redis, 2); + const secondReplica = new BridgeAdmissionQueue(redis, 2); + const deadline = Date.now() + 5000; + expect(await firstReplica.enter('worker', 'first', deadline)).toBe(true); + expect(await secondReplica.enter('worker', 'second', deadline)).toBe(true); + expect(await firstReplica.enter('worker', 'third', deadline)).toBe(false); + expect(await secondReplica.isHead('worker', 'second')).toBe(false); + await firstReplica.leave('worker', 'first'); + expect(await secondReplica.isHead('worker', 'second')).toBe(true); + expect(await firstReplica.enter('worker', 'third', deadline)).toBe(true); + expect(await firstReplica.isHead('worker', 'third')).toBe(false); +}); + +test('expired crashed callers cannot strand the next request or consume capacity', async () => { + const queue = new BridgeAdmissionQueue(redis, 1); + expect(await queue.enter('worker', 'expired', Date.now() - 1)).toBe(true); + expect(await queue.enter('worker', 'live', Date.now() + 5000)).toBe(true); + expect(await queue.isHead('worker', 'live')).toBe(true); +}); + +test('different machines do not share admission capacity', async () => { + const queue = new BridgeAdmissionQueue(redis, 1); + expect(await queue.enter('worker-a', 'a', Date.now() + 5000)).toBe(true); + expect(await queue.enter('worker-b', 'b', Date.now() + 5000)).toBe(true); + expect(await queue.isHead('worker-b', 'b')).toBe(true); +}); diff --git a/service/src/bridge/admission.ts b/service/src/bridge/admission.ts new file mode 100644 index 00000000..1e19286e --- /dev/null +++ b/service/src/bridge/admission.ts @@ -0,0 +1,90 @@ +import type Redis from 'ioredis'; + +/** Bounded FIFO admission shared by API replicas. Entries expire after caller deadlines. */ +export class BridgeAdmissionQueue { + constructor( + private readonly redis: Redis, + private readonly capacity = 32, + ) {} + + private keys(workerId: string): [string, string, string] { + const prefix = `codeapi:bridge:v1:worker:${encodeURIComponent(workerId)}:admission`; + return [prefix, `${prefix}:deadlines`, `${prefix}:sequence`]; + } + + async enter( + workerId: string, + id: string, + deadlineAtMs: number, + ): Promise { + return ( + Number( + await this.redis.eval( + [ + "local expired = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', ARGV[2])", + 'for _, id in ipairs(expired) do', + " redis.call('ZREM', KEYS[1], id)", + " redis.call('ZREM', KEYS[2], id)", + 'end', + "if redis.call('ZSCORE', KEYS[1], ARGV[1]) then return 1 end", + "if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[4]) then return 0 end", + "local sequence = redis.call('INCR', KEYS[3])", + "redis.call('ZADD', KEYS[1], sequence, ARGV[1])", + "redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])", + "local latest = redis.call('ZREVRANGE', KEYS[2], 0, 0, 'WITHSCORES')", + 'for _, key in ipairs(KEYS) do', + " redis.call('PEXPIREAT', key, tonumber(latest[2]) + 30000)", + 'end', + 'return 1', + ].join('\n'), + 3, + ...this.keys(workerId), + id, + Date.now(), + deadlineAtMs, + this.capacity, + ), + ) === 1 + ); + } + + async isHead(workerId: string, id: string): Promise { + const [order, deadlines] = this.keys(workerId); + return ( + Number( + await this.redis.eval( + [ + "local expired = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', ARGV[2])", + 'for _, id in ipairs(expired) do', + " redis.call('ZREM', KEYS[1], id)", + " redis.call('ZREM', KEYS[2], id)", + 'end', + "local head = redis.call('ZRANGE', KEYS[1], 0, 0)", + 'if head[1] == ARGV[1] then return 1 end', + 'return 0', + ].join('\n'), + 2, + order, + deadlines, + id, + Date.now(), + ), + ) === 1 + ); + } + + async leave(workerId: string, id: string): Promise { + const [order, deadlines] = this.keys(workerId); + await this.redis.eval( + [ + "redis.call('ZREM', KEYS[1], ARGV[1])", + "redis.call('ZREM', KEYS[2], ARGV[1])", + 'return 1', + ].join('\n'), + 2, + order, + deadlines, + id, + ); + } +} diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index de302654..43634c7c 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -18,6 +18,7 @@ import { isWorkspaceToolResult, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; +import { BridgeAdmissionQueue } from './admission'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; @@ -43,6 +44,7 @@ export class BridgeStoreError extends Error { | 'WORKER_OFFLINE' | 'WORKER_UNAUTHORIZED' | 'WORKER_BUSY' + | 'WORKER_QUEUE_FULL' | 'ASSIGNMENT_EXPIRED' | 'ASSIGNMENT_FENCED' | 'ASSIGNMENT_NOT_FOUND' @@ -684,18 +686,42 @@ export class RedisBridgeStore { const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let resultCommitted = false; + const admission = args.workspaceRequest == null + ? undefined + : new BridgeAdmissionQueue(this.redis); try { - const locked = await this.dispatchCommand( - () => - this.acquireLock( - args.workerId, - assignmentId, - lockIncarnationId, - ttlSeconds, - ), + if (admission != null && !(await this.dispatchCommand( + () => admission.enter(args.workerId, assignmentId, args.deadlineAtMs), args, - 'Bridge assignment lock acquisition', - ); + 'Bridge admission enqueue', + ))) { + throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); + } + let locked = false; + do { + if (admission != null && !(await this.dispatchCommand( + () => admission.isHead(args.workerId, assignmentId), + args, + 'Bridge admission position', + ))) { + await delay(Math.min(POLL_INTERVAL_MS, args.deadlineAtMs - Date.now()), args.signal); + continue; + } + locked = await this.dispatchCommand( + () => + this.acquireLock( + args.workerId, + assignmentId, + lockIncarnationId, + ttlSeconds, + ), + args, + 'Bridge assignment lock acquisition', + ); + if (!locked && admission != null) { + await delay(Math.min(POLL_INTERVAL_MS, args.deadlineAtMs - Date.now()), args.signal); + } + } while (!locked && admission != null); if (!locked) { throw new BridgeStoreError( 'WORKER_BUSY', @@ -703,6 +729,25 @@ export class RedisBridgeStore { ); } this.assertDispatchActive(args.signal, args.deadlineAtMs); + if (admission != null) { + // Waiting must not transfer accepted work to a replacement machine or identity. + const current = await this.dispatchCommand( + () => this.dispatchableRegistration(args.workerId), + args, + 'Bridge admitted worker validation', + ); + if ( + current == null || + current.registration.incarnationId !== registration.incarnationId || + current.registration.identityId !== registration.identityId || + current.registration.binding?.tenantId !== registration.binding?.tenantId + ) { + throw new BridgeStoreError('WORKER_OFFLINE', 'Bridge worker changed while the request was waiting'); + } + if (!supportsWorkspaceTool(current.registration, args.workspaceRequest!)) { + throw new BridgeStoreError('WORKER_MISMATCH', 'Bridge worker capabilities changed while the request was waiting'); + } + } const generation = await this.dispatchCommand( () => this.redis.incr(generationKey(args.workerId)), args, @@ -748,6 +793,12 @@ export class RedisBridgeStore { 'Bridge assignment enqueue', ); if (queued) break; + if (admission != null) { + throw new BridgeStoreError( + 'WORKER_FENCED', + 'Bridge worker changed before the waiting request could be dispatched', + ); + } const replacement = await this.dispatchCommand( () => this.dispatchableRegistration(args.workerId), args, @@ -810,6 +861,14 @@ export class RedisBridgeStore { throw error; } } finally { + if (admission != null) { + // Expiry remains the fallback if Redis is unavailable during cancellation. + await boundedCommand( + admission.leave(args.workerId, assignmentId), + this.redisCommandTimeoutMs, + 'Bridge admission cleanup', + ).catch(() => undefined); + } if (resultCommitted) { try { await this.cleanupWithRetry(args.workerId, assignmentId, assignment); diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts new file mode 100644 index 00000000..57ad3d84 --- /dev/null +++ b/service/src/bridge/worker-admission.test.ts @@ -0,0 +1,144 @@ +import { afterEach, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const store = new RedisBridgeStore(redis); +const workerId = 'admission-worker'; +const incarnationId = 'incarnation-00000001'; +afterEach(async () => { + await redis.flushall(); +}); + +async function register( + operations: Array<'read_file' | 'list_files'> = ['read_file'], +): Promise { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations, + workspaces: [{ id: 'primary', name: 'Workspace' }], + }, + }, + }); +} + +function dispatch( + path: string, + controller = new AbortController(), + budgetMs = 5000, +): ReturnType { + return store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: 'primary', + path, + }, + }); +} + +async function settle( + assignment: CodeBridgeAssignment | undefined, +): Promise { + expect(assignment).toBeDefined(); + await store.settle(workerId, assignment!.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment!.generation, + leaseToken: assignment!.leaseToken, + status: 'rejected', + error: 'File not found', + }); +} + +test('a second workspace call waits until the first settles instead of returning WORKER_BUSY', async () => { + await register(); + const first = dispatch('first'); + const assignment = await store.lease(workerId, incarnationId, 1000); + const second = dispatch('second'); + await settle(assignment); + await first; + const next = await store.lease(workerId, incarnationId, 1000); + expect(next?.request).toMatchObject({ path: 'second' }); + await settle(next); + await expect(second).resolves.toMatchObject({ + status: 'rejected', + error: 'File not found', + }); +}); + +test('cancelling a waiting caller does not release or cancel the active assignment', async () => { + await register(); + const first = dispatch('first'); + const assignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const second = dispatch('second', controller); + void second.catch(() => undefined); + const waitDeadline = Date.now() + 1000; + while ( + (await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)) < 2 + ) { + if (Date.now() >= waitDeadline) + throw new Error('Second caller never entered admission'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + controller.abort(); + await expect(second).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBe( + assignment!.assignmentId, + ); + await settle(assignment); + await first; + expect(await store.lease(workerId, incarnationId, 50)).toBeUndefined(); +}); + +test('an expired queued call never reaches the worker and does not strand later calls', async () => { + await register(); + const first = dispatch('first'); + const assignment = await store.lease(workerId, incarnationId, 1000); + await expect( + dispatch('expired', new AbortController(), 25), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + const third = dispatch('third'); + await settle(assignment); + await first; + const next = await store.lease(workerId, incarnationId, 1000); + expect(next?.request).toMatchObject({ path: 'third' }); + await settle(next); + await third; +}); + +test('a queued request is rejected if the worker withdraws its capability', async () => { + await register(); + const first = dispatch('first'); + const assignment = await store.lease(workerId, incarnationId, 1000); + const second = dispatch('second'); + void second.catch(() => undefined); + const deadline = Date.now() + 1000; + while ( + (await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)) < 2 + ) { + if (Date.now() > deadline) + throw new Error('Second caller did not enter admission'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await register(['list_files']); + await settle(assignment); + await first; + await expect(second).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await store.lease(workerId, incarnationId, 50)).toBeUndefined(); +}); diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index b0121d2e..a65bc89e 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -79,6 +79,7 @@ describe('RemoteBridgeSandboxBackend', () => { WORKER_OFFLINE: ['BRIDGE_WORKER_OFFLINE', true, 503, 'Code environment is offline'], WORKER_UNAUTHORIZED: ['BRIDGE_WORKER_UNAUTHORIZED', false, 403, 'Code environment is not authorized for this tenant'], WORKER_BUSY: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], + WORKER_QUEUE_FULL: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], ASSIGNMENT_EXPIRED: ['BRIDGE_DEADLINE_EXCEEDED', false, 504, 'Code environment execution timed out'], ASSIGNMENT_FENCED: ['BRIDGE_ASSIGNMENT_FENCED', false, 409, 'Code environment assignment is fenced; inspect the execution before retrying'], ASSIGNMENT_NOT_FOUND: ['BRIDGE_ASSIGNMENT_NOT_FOUND', false, 409, 'Code environment assignment is no longer available; inspect the execution before retrying'], diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 719e7004..0bee0038 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -18,6 +18,7 @@ const bridgeErrorCodes = { WORKER_OFFLINE: 'BRIDGE_WORKER_OFFLINE', WORKER_UNAUTHORIZED: 'BRIDGE_WORKER_UNAUTHORIZED', WORKER_BUSY: 'BRIDGE_WORKER_BUSY', + WORKER_QUEUE_FULL: 'BRIDGE_WORKER_BUSY', ASSIGNMENT_EXPIRED: 'BRIDGE_DEADLINE_EXCEEDED', ASSIGNMENT_FENCED: 'BRIDGE_ASSIGNMENT_FENCED', ASSIGNMENT_NOT_FOUND: 'BRIDGE_ASSIGNMENT_NOT_FOUND', diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 21aa86c7..b5c38ef8 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -32,6 +32,7 @@ afterEach(() => { test('maps invalid worker results to an upstream failure', () => { expect(bridgeStoreStatus(new BridgeStoreError('RESULT_INVALID', 'invalid worker result'))).toBe(502); + expect(bridgeStoreStatus(new BridgeStoreError('WORKER_QUEUE_FULL', 'queue full'))).toBe(429); }); test('rejects new workspace dispatches while the service is shutting down', async () => { diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index b2b56e06..93d15890 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -31,6 +31,7 @@ function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promi } export function bridgeStoreStatus(error: BridgeStoreError): number { + if (error.code === 'WORKER_QUEUE_FULL') return 429; if (error.code === 'WORKER_UNAUTHORIZED') return 403; if (error.code === 'ASSIGNMENT_INVALID') return 400; if (error.code === 'RESULT_INVALID') return 502; From 01a190507c8d718a5c475132af1978849b41e646 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 15:52:24 -0400 Subject: [PATCH 067/116] fix: Isolate Native Sandbox Scratch Storage (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔒 fix: Isolate Native Sandbox Scratch Storage * 📝 docs: Explain Shared Scratch Denial * 🛡️ fix: Preserve Windows SRT Temp Isolation * 🧪 test: Synchronize Sandbox Cancellation * 🔒 fix: Harden Scratch Lifecycle Boundaries --- packages/code/README.md | 15 +- packages/code/src/native-sandbox.test.ts | 189 +++++++++++++++++++- packages/code/src/native-sandbox.ts | 217 +++++++++++++++++++++-- 3 files changed, 404 insertions(+), 17 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 9dd3019a..78cab322 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -89,9 +89,18 @@ Windows. Startup fails before worker registration when the platform or its dependencies are unavailable. There is no unsandboxed command fallback. The bridge worker remains outside the sandbox so it can maintain its outbound -Code API connection. Each command and its descendants run inside SRT with: - -- write access restricted to the one canonical registered workspace; +Code API connection. On macOS and Linux, each worker process creates an +owner-only scratch directory and grants SRT access to that exact directory +without opening the host temporary-directory root. Commands receive it through +`TMPDIR`, and orderly worker shutdown removes it. SRT's shared compatibility +scratch path is explicitly denied. Windows uses the restricted SRT account's +isolated profile and temporary directory instead. A workspace registration is +rejected if it sits inside SRT's shared scratch path or is broad enough to +contain worker scratch storage. Each command and its descendants run inside +SRT with: + +- write access restricted to the one canonical registered workspace and the + worker's private scratch directory; - read access denied to the worker's home directory except for that workspace; - paired identity and mutation-quarantine files explicitly denied; - `LIBRECHAT_CODE_*` and nonessential inherited environment variables removed; diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 5ee411fb..8fe20225 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; import { EventEmitter } from 'node:events'; import { access, @@ -6,6 +7,7 @@ import { mkdir, realpath, rm, + stat, writeFile, } from 'node:fs/promises'; import { tmpdir, homedir } from 'node:os'; @@ -34,6 +36,7 @@ function fakeManager( beforeWrap?: () => Promise; appendGitSafeDirectory?: boolean; inheritedGitEnvironment?: Record; + initializeError?: Error; wrappedEnvironment?: NodeJS.ProcessEnv; } = {}, ) { @@ -41,6 +44,7 @@ function fakeManager( let reset = false; let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; + let scratchSelectorSeenDuringWrap: string | undefined; const manager = { isSupportedPlatform: () => true, async checkDependenciesAsync() { @@ -48,11 +52,13 @@ function fakeManager( }, async initialize(value: SandboxRuntimeConfig) { config = value; + if (options.initializeError) throw options.initializeError; }, async wrapWithSandboxArgv(command: string) { await options.beforeWrap?.(); credentialSeenDuringWrap = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; gitLfsRequiredSeenDuringWrap = process.env.GIT_CONFIG_VALUE_3; + scratchSelectorSeenDuringWrap = process.env.CLAUDE_CODE_TMPDIR; const ambientGitEnvironment = Object.fromEntries( Object.entries(process.env).filter( ([name, value]) => name.startsWith('GIT_CONFIG_') && value != null, @@ -105,6 +111,9 @@ function fakeManager( get gitLfsRequiredSeenDuringWrap() { return gitLfsRequiredSeenDuringWrap; }, + get scratchSelectorSeenDuringWrap() { + return scratchSelectorSeenDuringWrap; + }, }; } @@ -133,23 +142,187 @@ test('initializes SRT with a default-deny network and scrubbed worker credential join(await realpath(tmpdir()), 'librechat-code-identity.json'), ); const canonicalHome = await realpath(homedir()); + const scratchDirectory = fake.config?.filesystem.allowWrite[1]; + assert.equal(typeof scratchDirectory, 'string'); assert.deepEqual(fake.config?.network.allowedDomains, []); assert.equal(fake.config?.network.strictAllowlist, true); assert.equal(fake.config?.network.allowAllUnixSockets, false); - assert.deepEqual(fake.config?.filesystem.allowRead, [canonicalRoot]); - assert.deepEqual(fake.config?.filesystem.allowWrite, [canonicalRoot]); + assert.deepEqual(fake.config?.filesystem.allowRead, [ + canonicalRoot, + scratchDirectory, + ]); + assert.deepEqual(fake.config?.filesystem.allowWrite, [ + canonicalRoot, + scratchDirectory, + ]); + assert.equal((await stat(scratchDirectory!)).mode & 0o777, 0o700); assert.ok(fake.config?.filesystem.denyRead.includes(canonicalHome)); assert.ok(fake.config?.filesystem.denyWrite.includes(canonicalIdentity)); + assert.ok( + fake.config?.filesystem.denyWrite.some((path) => + path.endsWith('/tmp/claude'), + ), + ); const denied = fake.config?.credentials?.envVars?.map(({ name }) => name); assert.ok(denied?.includes('LIBRECHAT_CODE_WORKER_TOKEN')); assert.ok(denied?.includes('AWS_SECRET_ACCESS_KEY')); assert.ok(!denied?.includes('PATH')); assert.ok(denied?.includes('Path')); assert.ok(denied?.includes('lc_api_token')); + assert.ok(denied?.includes('CLAUDE_CODE_TMPDIR')); + assert.ok(denied?.includes('CLAUDE_TMPDIR')); + await sandbox.close(); + assert.equal(fake.reset, true); + await assert.rejects(access(scratchDirectory!)); +}); + +test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const originalTmpdir = process.env.TMPDIR; + const originalSrtTmpdir = process.env.CLAUDE_CODE_TMPDIR; + const originalLegacySrtTmpdir = process.env.CLAUDE_TMPDIR; + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + + const result = await sandbox.execute({ + ...request, + maxOutputBytes: 1_024, + command: 'touch "$TMPDIR/probe" && printf %s "$TMPDIR"', + }); + + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /librechat-code-srt-/); + await access(join(result.stdout, 'probe')); + assert.equal(fake.scratchSelectorSeenDuringWrap, result.stdout); + assert.equal(process.env.TMPDIR, originalTmpdir); + assert.equal(process.env.CLAUDE_CODE_TMPDIR, originalSrtTmpdir); + assert.equal(process.env.CLAUDE_TMPDIR, originalLegacySrtTmpdir); await sandbox.close(); + await assert.rejects(access(result.stdout)); +}); + +test('removes scratch storage when SRT initialization fails', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager({ initializeError: new Error('init failed') }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + + await assert.rejects(sandbox.prepare(), /init failed/); + const scratchDirectory = fake.config?.filesystem.allowWrite[1]; + assert.equal(typeof scratchDirectory, 'string'); + await assert.rejects(access(scratchDirectory!)); assert.equal(fake.reset, true); }); +test('rejects workspaces nested inside SRT shared scratch storage', async (t) => { + if (process.platform === 'win32') return; + const sharedRoot = '/tmp/claude'; + await mkdir(sharedRoot, { recursive: true }); + const root = await mkdtemp(join(sharedRoot, 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + await assert.rejects( + sandbox.prepare(), + (error: WorkspaceToolError) => + error.code === 'REGISTRATION_INVALID' && + /inherited writable path/.test(error.message), + ); +}); + +test('rejects a workspace that contains worker scratch storage', async () => { + if (process.platform === 'win32') return; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: tmpdir(), + manager: fakeManager().manager, + }); + + await assert.rejects( + sandbox.prepare(), + (error: WorkspaceToolError) => + error.code === 'REGISTRATION_INVALID' && + /contain worker scratch storage/.test(error.message), + ); + await sandbox.close(); +}); + +test('keeps concurrent sandbox scratch directories independent', async (t) => { + const firstRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + const secondRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(firstRoot, { recursive: true, force: true })); + t.after(() => rm(secondRoot, { recursive: true, force: true })); + let releaseWrap!: () => void; + let wrapStarted!: () => void; + const wrapStartedPromise = new Promise((resolve) => { + wrapStarted = resolve; + }); + const holdWrap = new Promise((resolve) => { + releaseWrap = resolve; + }); + const firstFake = fakeManager({ + async beforeWrap() { + wrapStarted(); + await holdWrap; + }, + }); + const secondFake = fakeManager(); + const firstSandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: firstRoot, + manager: firstFake.manager, + }); + const secondSandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: secondRoot, + manager: secondFake.manager, + }); + const firstExecution = firstSandbox.execute({ + ...request, + command: 'printf first', + }); + await wrapStartedPromise; + await secondSandbox.prepare(); + const firstScratch = firstFake.config?.filesystem.allowWrite[1]; + const secondScratch = secondFake.config?.filesystem.allowWrite[1]; + assert.equal(typeof firstScratch, 'string'); + assert.equal(typeof secondScratch, 'string'); + assert.notEqual(firstScratch, secondScratch); + assert.ok(!secondScratch!.startsWith(`${firstScratch}/`)); + releaseWrap(); + await firstExecution; + await firstSandbox.close(); + await access(secondScratch!); + await secondSandbox.close(); +}); + +test('removes scratch storage after a command revokes traversal permissions', async (t) => { + if (process.platform === 'win32') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + + const result = await sandbox.execute({ + ...request, + command: + 'printf %s "$TMPDIR"; mkdir "$TMPDIR/locked"; touch "$TMPDIR/locked/file"; chmod 000 "$TMPDIR/locked" "$TMPDIR"', + }); + + assert.equal(result.exitCode, 0); + await sandbox.close(); + await assert.rejects(access(result.stdout)); +}); + const proxyEnvironment = { HTTP_PROXY: 'http://upstream.invalid:8080', HTTPS_PROXY: 'http://upstream.invalid:8080', @@ -586,16 +759,26 @@ test('terminates detached command descendants before returning', async (t) => { test('reports cancellation after command start as a potentially committed mutation', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); + let commandStarted!: () => void; + const commandStartedPromise = new Promise((resolve) => { + commandStarted = resolve; + }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: root, manager: fakeManager().manager, + spawnCommand(command, args, options) { + const child = spawn(command, [...args], options); + commandStarted(); + return child; + }, }); const controller = new AbortController(); const execution = sandbox.execute( { ...request, command: 'sleep 30' }, controller.signal, ); - setTimeout(() => controller.abort(), 25); + await commandStartedPromise; + controller.abort(); await assert.rejects( execution, diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 9743ad76..47510f68 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -1,6 +1,6 @@ import { spawn } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { homedir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { basename, dirname, @@ -11,7 +11,17 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { access, realpath, stat } from 'node:fs/promises'; +import { + access, + chmod, + lstat, + mkdtemp, + open, + readdir, + realpath, + rm, + stat, +} from 'node:fs/promises'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -21,6 +31,11 @@ import { BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, isWorkspaceToolRequest, } from './protocol.js'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, + removePrivateStorageAcl, +} from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; import type { @@ -96,6 +111,17 @@ const { ...TRUSTED_GIT_CONFIG_ENTRIES } = TRUSTED_GIT_ENVIRONMENT; +const NATIVE_SANDBOX_SCRATCH_PREFIX = 'librechat-code-srt-'; +// SRT grants these shared compatibility paths by default. A worker-specific +// TMPDIR must also deny them or separate worker processes can exchange files. +const SRT_SHARED_SCRATCH_PATHS = ['/tmp/claude', '/private/tmp/claude']; +const SRT_SCRATCH_SELECTOR_NAMES = [ + 'CLAUDE_CODE_TMPDIR', + 'CLAUDE_TMPDIR', +] as const; +// Capture this before any command wrapper can temporarily mutate process.env. +const HOST_TEMPORARY_ROOT = tmpdir(); + interface NativeSandboxManager { isSupportedPlatform(): boolean; checkDependenciesAsync(): Promise<{ warnings: string[]; errors: string[] }>; @@ -211,6 +237,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + private scratchDirectory?: string; constructor( private readonly options: NativeSrtWorkspaceCommandSandboxOptions, @@ -230,6 +257,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox if (this.initialized) return this.initialized; this.initialized = this.initializeOnce().catch(async (error) => { await this.manager.reset().catch(() => undefined); + await this.removeScratchDirectory().catch(() => undefined); this.initialized = undefined; throw error; }); @@ -266,6 +294,26 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + const sharedScratchPaths = await Promise.all( + (this.platform === 'win32' ? [] : SRT_SHARED_SCRATCH_PATHS).map( + canonicalPath, + ), + ); + const inheritedWritablePaths = [ + ...sharedScratchPaths, + ...(await Promise.all( + [join(home, '.npm', '_logs'), join(home, '.claude', 'debug')].map( + canonicalPath, + ), + )), + ]; + const deniedInheritedWritablePaths = [...new Set(inheritedWritablePaths)]; + if (deniedInheritedWritablePaths.some((path) => isWithin(path, root))) { + throw new WorkspaceToolError( + 'Native sandbox workspace cannot be inside an inherited writable path', + 'REGISTRATION_INVALID', + ); + } const dependencies = await this.manager.checkDependenciesAsync(); if (dependencies.errors.length > 0) { throw new WorkspaceToolError( @@ -283,6 +331,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } } + const canonicalScratchDirectory = + await this.createScratchDirectory(sharedScratchPaths); + if ( + canonicalScratchDirectory && + isWithin(root, canonicalScratchDirectory) + ) { + throw new WorkspaceToolError( + 'Native sandbox workspace cannot contain worker scratch storage', + 'REGISTRATION_INVALID', + ); + } const config: SandboxRuntimeConfig = { network: { allowedDomains: [...(this.options.allowedDomains ?? [])], @@ -293,10 +352,21 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), }, filesystem: { - denyRead: [home], - allowRead: [root], - allowWrite: [root], - denyWrite: protectedPaths, + denyRead: [ + home, + ...sharedScratchPaths.filter((path) => + deniedInheritedWritablePaths.includes(path), + ), + ], + allowRead: [ + root, + ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ], + allowWrite: [ + root, + ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ], + denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], allowGitConfig: false, }, credentials: { @@ -305,7 +375,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox mode: 'deny' as const, })), envVars: [ - ...deniedEnvironmentNames(this.environment, this.platform) + ...deniedEnvironmentNames( + { + ...this.environment, + CLAUDE_CODE_TMPDIR: '', + CLAUDE_TMPDIR: '', + }, + this.platform, + ) .filter((name) => { const normalized = normalizedEnvironmentName( name, @@ -388,6 +465,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { ...TRUSTED_GIT_ENVIRONMENT, ...(credentialEnvironment ?? {}), + ...this.scratchSelectorEnvironment(), }, () => this.manager.wrapWithSandboxArgv( @@ -476,6 +554,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd, env: { ...wrapped.env, + ...this.scratchEnvironment(), ...TRUSTED_GIT_CONFIG_ENTRIES, GIT_CONFIG_COUNT: wrapped.env.GIT_CONFIG_COUNT ?? TRUSTED_GIT_CONFIG_COUNT, @@ -618,10 +697,126 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : 1; } + private async createScratchDirectory( + sharedScratchPaths: string[], + ): Promise { + // Windows SRT supplies the restricted account's private TEMP directory. + if (this.platform === 'win32') return undefined; + const canonicalTemporaryRoot = await canonicalPath(HOST_TEMPORARY_ROOT); + const sharedScratchRoot = sharedScratchPaths.find((path) => + isWithin(path, canonicalTemporaryRoot), + ); + const scratchDirectory = await mkdtemp( + join( + sharedScratchRoot + ? dirname(sharedScratchRoot) + : canonicalTemporaryRoot, + NATIVE_SANDBOX_SCRATCH_PREFIX, + ), + ); + try { + await assertPrivateStorageAncestors(scratchDirectory); + const scratchHandle = await open(scratchDirectory, 'r'); + try { + await removePrivateStorageAcl(scratchHandle, scratchDirectory); + await scratchHandle.chmod(0o700); + await assertPrivateStorageAcl( + scratchHandle, + scratchDirectory, + true, + ); + if (((await scratchHandle.stat()).mode & 0o777) !== 0o700) { + throw new Error('Native sandbox scratch directory is not private'); + } + } finally { + await scratchHandle.close(); + } + this.scratchDirectory = await realpath(scratchDirectory); + return this.scratchDirectory; + } catch (error) { + await rm(scratchDirectory, { recursive: true, force: true }).catch( + () => undefined, + ); + throw error; + } + } + + private scratchEnvironment(): NodeJS.ProcessEnv { + const scratchDirectory = this.scratchDirectory; + if (!scratchDirectory) return {}; + return this.platform === 'win32' + ? { + TMPDIR: scratchDirectory, + TEMP: scratchDirectory, + TMP: scratchDirectory, + } + : { TMPDIR: scratchDirectory }; + } + + private scratchSelectorEnvironment(): NodeJS.ProcessEnv { + const scratchDirectory = this.scratchDirectory; + if (!scratchDirectory) return {}; + return Object.fromEntries( + SRT_SCRATCH_SELECTOR_NAMES.map((name) => [name, scratchDirectory]), + ); + } + + private async removeScratchDirectory(): Promise { + const scratchDirectory = this.scratchDirectory; + if (!scratchDirectory) return; + try { + await rm(scratchDirectory, { recursive: true, force: true }); + } catch { + await this.restoreScratchTraversal(scratchDirectory); + await rm(scratchDirectory, { recursive: true, force: true }); + } + this.scratchDirectory = undefined; + } + + private async restoreScratchTraversal(root: string): Promise { + const pending = [root]; + for (let index = 0; index < pending.length; index += 1) { + const directory = pending[index]; + const metadata = await lstat(directory).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return undefined; + throw error; + }, + ); + if (!metadata?.isDirectory()) continue; + // Commands own their scratch contents and may remove all directory mode + // bits. Restore traversal before opening the directory with O_NOFOLLOW. + await chmod(directory, 0o700); + let handle; + try { + handle = await open( + directory, + fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW, + ); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (['ENOENT', 'ELOOP', 'ENOTDIR'].includes(code ?? '')) continue; + throw error; + } + try { + if (!(await handle.stat()).isDirectory()) continue; + } finally { + await handle.close(); + } + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.isDirectory()) pending.push(join(directory, entry.name)); + } + } + } + async close(): Promise { - if (!this.initialized) return; - await this.manager.reset(); - this.initialized = undefined; - this.canonicalRoot = undefined; + if (!this.initialized && !this.scratchDirectory) return; + try { + if (this.initialized) await this.manager.reset(); + } finally { + this.initialized = undefined; + this.canonicalRoot = undefined; + await this.removeScratchDirectory(); + } } } From 1e32bda1c6e8772c4ec0f8e13c2edfdb671fe2f8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 16:49:32 -0400 Subject: [PATCH 068/116] =?UTF-8?q?fix:=20Separate=20BYOM=20Admission=20an?= =?UTF-8?q?d=20Execution=20Deadlines=20=E2=8F=B1=EF=B8=8F=20(#161)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/byom-worker-admission.md | 24 ++++++++--- service/src/bridge/store.ts | 16 +++++++- service/src/bridge/worker-admission.test.ts | 30 +++++++++++++- service/src/workspace-tools/router.test.ts | 45 ++++++++++++++++++++- service/src/workspace-tools/router.ts | 33 ++++++++++++--- 5 files changed, 133 insertions(+), 15 deletions(-) diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md index c88d5669..2c2b354b 100644 --- a/docs/byom-worker-admission.md +++ b/docs/byom-worker-admission.md @@ -5,8 +5,12 @@ The limit is 32 admitted requests per worker, including the active request. When the limit is reached, the workspace endpoint returns HTTP 429 with `WORKER_QUEUE_FULL`. A different worker has an independent admission queue. -Waiting uses the caller's existing absolute dispatch deadline. It does not reset -or extend execution timeouts. Disconnecting or cancelling removes the waiting +The workspace HTTP endpoint allows at most 30 seconds for admission. After +admission and worker validation, a separate execution deadline starts. Commands +receive their requested timeout (30 seconds by default, up to five minutes), +capped by the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. +Read/search/list operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. +Disconnecting or cancelling removes the waiting request without cancelling the active assignment. Expired entries are pruned; Redis key expiry also bounds state left by a crashed API process. @@ -15,12 +19,20 @@ binding and workspace operation. A waiting request cannot migrate to a replaceme worker. Existing execution acknowledgement, fencing, settlement and quarantine rules remain responsible for the active assignment. -This is compatible with existing workers and requires only a Code API update. +This is compatible with existing workers: assignments retain the same absolute +deadline and server-relative timing fields. Store callers that omit the new +internal `executionTimeoutMs` argument retain their existing absolute-deadline behavior. Existing workers still execute one assignment at a time. Parallel execution across workspaces requires separate lease claims and isolated native sandbox contexts; -this admission change does not advertise that capability. Queue time and execution -time currently share the HTTP request deadline. Separate budgets require a matching -LibreChat client change so that the client does not disconnect while waiting. +this admission change does not advertise that capability. + +LibreChat must allow queue time plus execution/settlement time and five seconds +for HTTP delivery: 65 seconds for reads, 70 seconds for default commands, and +340 seconds for five-minute commands. Either side can be upgraded first. Older +clients still cancel at their earlier deadline; newer clients preserve errors from +older servers without retrying mutations. Both updates are needed for the full +waiting budget. Any reverse proxy request timeout must accommodate these totals. +The worker package does not need an update for the deadline change. Focused regression coverage lives in `service/src/bridge/admission.test.ts` and `service/src/bridge/worker-admission.test.ts`. diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 43634c7c..2fa67336 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -573,6 +573,7 @@ export class RedisBridgeStore { requireTenantBinding?: boolean; request: WorkspaceToolRequest; deadlineAtMs: number; + executionTimeoutMs?: number; signal: AbortSignal; }): Promise { if (!isWorkspaceToolRequest(args.request)) { @@ -614,12 +615,20 @@ export class RedisBridgeStore { workspaceRequest?: WorkspaceToolRequest; runtimeSessionId?: string; deadlineAtMs: number; + executionTimeoutMs?: number; signal: AbortSignal; finalize?: ( settlement: CodeBridgeSettlement, registration: RegisteredBridgeWorker, ) => Promise; }): Promise { + if (args.executionTimeoutMs !== undefined && ( + args.workspaceRequest == null || + !Number.isSafeInteger(args.executionTimeoutMs) || + args.executionTimeoutMs < 1 || args.executionTimeoutMs > 305_000 + )) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid workspace execution budget'); + } this.assertDispatchActive(args.signal, args.deadlineAtMs); const dispatchable = await this.dispatchCommand( () => this.dispatchableRegistration(args.workerId), @@ -682,7 +691,8 @@ export class RedisBridgeStore { const assignmentId = randomBytes(18).toString('base64url'); const leaseToken = randomBytes(32).toString('base64url'); - const ttlSeconds = assignmentTtlSeconds(args.deadlineAtMs); + // The lock is acquired before admission finishes; it must outlive the later execution deadline. + const ttlSeconds = assignmentTtlSeconds(args.deadlineAtMs + (args.executionTimeoutMs ?? 0)); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let resultCommitted = false; @@ -748,6 +758,10 @@ export class RedisBridgeStore { throw new BridgeStoreError('WORKER_MISMATCH', 'Bridge worker capabilities changed while the request was waiting'); } } + this.assertDispatchActive(args.signal, args.deadlineAtMs); + if (args.executionTimeoutMs !== undefined) { + args = { ...args, deadlineAtMs: Date.now() + args.executionTimeoutMs }; + } const generation = await this.dispatchCommand( () => this.redis.incr(generationKey(args.workerId)), args, diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index 57ad3d84..db2e5956 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -37,11 +37,13 @@ function dispatch( path: string, controller = new AbortController(), budgetMs = 5000, + executionTimeoutMs?: number, ): ReturnType { return store.dispatchWorkspaceTool({ workerId, signal: controller.signal, deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs, request: { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'read_file', @@ -111,7 +113,7 @@ test('an expired queued call never reaches the worker and does not strand later const first = dispatch('first'); const assignment = await store.lease(workerId, incarnationId, 1000); await expect( - dispatch('expired', new AbortController(), 25), + dispatch('expired', new AbortController(), 25, 1000), ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); const third = dispatch('third'); await settle(assignment); @@ -142,3 +144,29 @@ test('a queued request is rejected if the worker withdraws its capability', asyn await expect(second).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); expect(await store.lease(workerId, incarnationId, 50)).toBeUndefined(); }); + +test('execution receives a fresh budget after waiting and the lock covers long commands', async () => { + await register(); + const first = dispatch('first'); + const active = await store.lease(workerId, incarnationId, 1000); + const second = dispatch('second', new AbortController(), 1000, 305_000); + await new Promise(resolve => setTimeout(resolve, 150)); + await settle(active); + await first; + const next = await store.lease(workerId, incarnationId, 1000); + expect(next).toBeDefined(); + expect(Date.parse(next!.expiresAt) - Date.now()).toBeGreaterThan(304_000); + expect(await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeGreaterThan(305_000); + await settle(next); + await second; +}); + +test('execution expires independently of an unused queue allowance', async () => { + await register(); + const completion = dispatch('short', new AbortController(), 5000, 150); + void completion.catch(() => undefined); + const assignment = await store.lease(workerId, incarnationId, 1000); + expect(assignment).toBeDefined(); + expect(Date.parse(assignment!.expiresAt) - Date.now()).toBeLessThanOrEqual(150); + await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); +}); diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index b5c38ef8..ae291a4b 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -14,6 +14,7 @@ import { hostedAppPreviewGateway } from '../hosted-app/preview-gateway'; import { applyPrincipal } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; import { bridgeStoreStatus, createWorkspaceToolsRouter } from './router'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; let server: Server | undefined; let logCompleted: ReturnType>; @@ -35,6 +36,46 @@ test('maps invalid worker results to an upstream failure', () => { expect(bridgeStoreStatus(new BridgeStoreError('WORKER_QUEUE_FULL', 'queue full'))).toBe(429); }); +test.each<[WorkspaceToolRequest, number, number?]>([ + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000], +])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + let executionBudget: number | undefined; + let queueRemaining: number | undefined; + let commandTimeout: number | undefined; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + timeoutMs: ceiling, + store: { async dispatchWorkspaceTool(args) { + executionBudget = args.executionTimeoutMs; + if (args.request.operation === 'execute_command') commandTimeout = args.request.timeoutMs; + queueRemaining = args.deadlineAtMs - Date.now(); + return { protocolVersion: 1, generation: 1, leaseToken: 'lease', incarnationId: 'incarnation', status: 'rejected', error: 'fixture' }; + } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(request), + }); + await response.json(); + expect(executionBudget).toBe(expectedExecution); + if (request.operation === 'execute_command') expect(commandTimeout).toBe(expectedExecution - 5000); + expect(queueRemaining).toBeGreaterThan(29_000); + expect(queueRemaining).toBeLessThanOrEqual(30_000); +}); + test('rejects new workspace dispatches while the service is shutting down', async () => { let dispatched = false; const app = express(); @@ -167,7 +208,7 @@ test.each([ operation: 'search_text', workerId: 'user-worker', dispatchDurationMs: expect.any(Number), - deadlineBudgetMs: 30_000, + deadlineBudgetMs: 60_000, }), ); await expect(response.json()).resolves.toMatchObject({ @@ -330,7 +371,7 @@ test.each([ status: expectedStatus, errorCode, outcome: 'completed', - deadlineBudgetMs: 300_000, + deadlineBudgetMs: 60_000, dispatchDurationMs: expect.any(Number), }), ); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 93d15890..17085963 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -3,12 +3,16 @@ import { Router } from 'express'; import type { RequestHandler, Response } from 'express'; import type { AuthenticatedRequest } from '../types'; import type { RedisBridgeStore } from '../bridge/store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; import { getWorkspaceToolOutcome } from './outcome'; import { getPrincipalOrReject } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; import { checkServiceShutDown } from '../lifecycle'; -import { isWorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import { + isWorkspaceToolRequest, + BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, +} from '../../../packages/code/src/protocol'; import { CODEAPI_BRIDGE_WORKER_HEADER, BridgeWorkerSelectionError, @@ -21,6 +25,7 @@ interface WorkspaceToolsRouterOptions { configuredWorkerId: string; dynamicWorkers: boolean; timeoutMs?: number; + queueTimeoutMs?: number; isShuttingDown?: () => boolean; } @@ -43,14 +48,21 @@ export function bridgeStoreStatus(error: BridgeStoreError): number { } export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions): Router { + const queueBudgetMs = options.queueTimeoutMs ?? 30_000; + if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > 30_000) { + throw new RangeError('Workspace queue timeout must be between 1 and 30000 milliseconds'); + } + if (options.timeoutMs !== undefined && ( + !Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1 + )) { + throw new RangeError('Workspace execution timeout must be a positive safe integer'); + } const router = Router(); router.post( '/workspace-tools/execute', asyncRoute(async (req, res) => { const outcome = getWorkspaceToolOutcome(res); - const deadlineBudgetMs = Math.max(1, options.timeoutMs ?? 30_000); - outcome.deadlineBudgetMs = deadlineBudgetMs; const principal = getPrincipalOrReject(req, res); if (!principal) { outcome.errorCode = 'UNAUTHENTICATED'; @@ -69,6 +81,16 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) return; } outcome.operation = req.body.operation; + const request: WorkspaceToolRequest = req.body.operation === 'execute_command' + ? { ...req.body, timeoutMs: Math.min( + req.body.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + options.timeoutMs ?? Number.MAX_SAFE_INTEGER, + ) } + : req.body; + const executionBudgetMs = request.operation === 'execute_command' + ? request.timeoutMs! + 5_000 + : Math.min(options.timeoutMs ?? 30_000, 30_000); + outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; let selection: { workerId: string; explicit: boolean } | undefined; try { @@ -111,8 +133,9 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) tenantId: principal.tenantId, requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), - request: req.body, - deadlineAtMs: Date.now() + deadlineBudgetMs, + request, + deadlineAtMs: Date.now() + queueBudgetMs, + executionTimeoutMs: executionBudgetMs, signal: controller.signal, }).finally(() => { outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); From 499fb58723b1a0b31f0cff0e11049bbaee16b699 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 17:13:33 -0400 Subject: [PATCH 069/116] fix: Fence Native SRT Lifecycle Ownership (#162) --- packages/code/README.md | 10 ++ packages/code/src/native-sandbox.test.ts | 124 +++++++++++++++++++++++ packages/code/src/native-sandbox.ts | 73 +++++++++++-- 3 files changed, 201 insertions(+), 6 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 78cab322..e3cc5489 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -88,6 +88,16 @@ bubblewrap plus seccomp on Linux, and the SRT restricted-account helper on Windows. Startup fails before worker registration when the platform or its dependencies are unavailable. There is no unsandboxed command fallback. +The native SRT manager owns process-global policy, proxy, and cleanup state. +Only one sandbox instance may own a manager, and that instance accepts one +command at a time. Overlapping calls fail before a second command starts; +they are not queued inside the sandbox. `close()` waits for the active command +and initialization before resetting the manager and removing scratch. A failed +reset keeps ownership fenced until a later `close()` succeeds. Independent +native workspaces need separate worker processes, not multiple instances of +the default manager in one process. This lifecycle guard does not enable +parallel assignments on a single bridge worker. + The bridge worker remains outside the sandbox so it can maintain its outbound Code API connection. On macOS and Linux, each worker process creates an owner-only scratch directory and grants SRT access to that exact directory diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 8fe20225..2d40db00 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -117,6 +117,130 @@ function fakeManager( }; } +test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const first = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + const second = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => first.close()); + t.after(() => second.close()); + await first.prepare(); + await assert.rejects( + second.prepare(), + /already belongs to another workspace/, + ); + await second.close(); + assert.equal( + fake.reset, + false, + 'a rejected owner must not reset the live manager', + ); + assert.equal((await first.execute(request)).stdout, 'hello'); + await first.close(); + await second.prepare(); + assert.equal((await second.execute(request)).stdout, 'hello'); +}); + +test('exclusive lifecycle rejects overlapping commands and waits before resetting', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + let entered!: () => void; + const wrapping = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const fake = fakeManager({ + beforeWrap: async () => { + entered(); + await gate; + }, + }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + const execution = sandbox.execute(request); + await wrapping; + await assert.rejects(sandbox.execute(request), /active command/); + const closing = sandbox.close(); + const secondClose = sandbox.close(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(fake.reset, false); + await assert.rejects(sandbox.prepare(), /closing/); + release(); + assert.equal((await execution).stdout, 'hello'); + await Promise.all([closing, secondClose]); + assert.equal(fake.reset, true); +}); + +test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + let failReset = true; + fake.manager.reset = async () => { + if (failReset) throw new Error('reset failed'); + }; + const first = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + const second = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + await first.prepare(); + await assert.rejects(first.close(), /reset failed/); + await assert.rejects(first.execute(request), /requires cleanup/); + await assert.rejects(second.prepare(), /already belongs/); + failReset = false; + await first.close(); + await second.prepare(); + await second.close(); +}); + +test('exclusive lifecycle waits for initialization before resetting the manager', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + let entered!: () => void; + const initializing = new Promise((resolve) => { + entered = resolve; + }); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + fake.manager.initialize = async () => { + entered(); + await gate; + }; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + const preparing = sandbox.prepare(); + await initializing; + const closing = sandbox.close(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(fake.reset, false); + release(); + await preparing; + await closing; + assert.equal(fake.reset, true); +}); + test('initializes SRT with a default-deny network and scrubbed worker credentials', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const identity = join(tmpdir(), 'librechat-code-identity.json'); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 47510f68..bc570442 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -139,6 +139,13 @@ interface NativeSandboxManager { reset(): Promise; } +// SRT's default manager is process-global, including its policy and cleanup +// state. Distinct workspace objects must not reconfigure the same manager. +const managerOwners = new WeakMap< + NativeSandboxManager, + NativeSrtWorkspaceCommandSandbox +>(); + type SpawnCommand = ( command: string, args: readonly string[], @@ -238,6 +245,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private initialized?: Promise; private canonicalRoot?: string; private scratchDirectory?: string; + private execution?: Promise; + private closing?: Promise; + private resetFailed = false; constructor( private readonly options: NativeSrtWorkspaceCommandSandboxOptions, @@ -254,10 +264,27 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } private async initialize(): Promise { + if (this.closing || this.resetFailed) { + throw new WorkspaceToolError( + 'Native sandbox is closing or requires cleanup', + 'COMMAND_UNAVAILABLE', + ); + } if (this.initialized) return this.initialized; + const owner = managerOwners.get(this.manager); + if (owner && owner !== this) { + throw new WorkspaceToolError( + 'Native sandbox manager already belongs to another workspace; use a separate worker process', + 'COMMAND_UNAVAILABLE', + ); + } + managerOwners.set(this.manager, this); this.initialized = this.initializeOnce().catch(async (error) => { - await this.manager.reset().catch(() => undefined); + await this.manager.reset().catch(() => { + this.resetFailed = true; + }); await this.removeScratchDirectory().catch(() => undefined); + if (!this.resetFailed) managerOwners.delete(this.manager); this.initialized = undefined; throw error; }); @@ -419,6 +446,25 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox async execute( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, + ): Promise { + if (this.execution || this.closing) { + throw new WorkspaceToolError( + 'Native sandbox already has an active command or is closing', + 'COMMAND_UNAVAILABLE', + ); + } + const execution = this.executeExclusive(request, signal); + this.execution = execution; + try { + return await execution; + } finally { + this.execution = undefined; + } + } + + private async executeExclusive( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, ): Promise { if ( !isWorkspaceToolRequest(request) || @@ -810,13 +856,28 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } async close(): Promise { - if (!this.initialized && !this.scratchDirectory) return; + if (this.closing) return this.closing; + const closing = this.closeExclusive(); + this.closing = closing; try { - if (this.initialized) await this.manager.reset(); + await closing; } finally { - this.initialized = undefined; - this.canonicalRoot = undefined; - await this.removeScratchDirectory(); + this.closing = undefined; + } + } + + private async closeExclusive(): Promise { + // Never reset proxy/credential state or remove scratch beneath a live child. + await this.execution?.catch(() => undefined); + await this.initialized?.catch(() => undefined); + if (managerOwners.get(this.manager) === this) { + this.resetFailed = true; + await this.manager.reset(); + this.resetFailed = false; + managerOwners.delete(this.manager); } + this.initialized = undefined; + this.canonicalRoot = undefined; + await this.removeScratchDirectory(); } } From c888fec10c0bd01ef3395bb23f8cc911ffa67629 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 18:01:20 -0400 Subject: [PATCH 070/116] feat: Isolate Native SRT Executor Processes (#163) * feat: Isolate Native SRT Executor Processes * fix: Preserve Executor Compatibility And Bound Shutdown * fix: Preserve Pre-Dispatch Atomicity And Platform Environment Rules * fix: Terminate Executors After Failed Preparation * fix: Drain Native Executors On Terminal Signals --- packages/code/README.md | 16 + packages/code/src/cli.ts | 5 +- packages/code/src/index.ts | 1 + .../code/src/native-process-child.test.ts | 39 ++ packages/code/src/native-process-child.ts | 111 +++++ packages/code/src/native-process.test.ts | 380 ++++++++++++++++++ packages/code/src/native-process.ts | 357 ++++++++++++++++ 7 files changed, 907 insertions(+), 2 deletions(-) create mode 100644 packages/code/src/native-process-child.test.ts create mode 100644 packages/code/src/native-process-child.ts create mode 100644 packages/code/src/native-process.test.ts create mode 100644 packages/code/src/native-process.ts diff --git a/packages/code/README.md b/packages/code/README.md index e3cc5489..7e7c0aaf 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -98,6 +98,22 @@ native workspaces need separate worker processes, not multiple instances of the default manager in one process. This lifecycle guard does not enable parallel assignments on a single bridge worker. +The CLI hosts the native manager in a persistent, dedicated Node executor +process. It does not inherit the bridge credential, arbitrary host environment, +or Node loader/debugger options. Workspace policy and per-command masked +credentials travel over private parent/child IPC, never command-line arguments. +The bridge retains pairing and GitHub App identity management. Cancellation is +addressed to the active command; executor loss after dispatch is treated as an +uncertain mutation and is never automatically replayed. Restarting a worker +still requires its existing quarantine checks. Native platform limitations on +hard descendant teardown continue to apply. + +Embedding applications can use `NativeProcessWorkspaceCommandSandbox` from +`@librechat/code` for separate native managers in one host application, with +`prepare()`, `execute()`, and `close()`. Each instance is serial and must be +closed by its owner. The bridge scheduler remains serial until negotiated +execution slots and workspace-scoped quarantine are supported end to end. + The bridge worker remains outside the sandbox so it can maintain its outbound Code API connection. On macOS and Linux, each worker process creates an owner-only scratch directory and grants SRT access to that exact directory diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 28103891..67ec396c 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -27,7 +27,7 @@ import { EndpointRuntimeSupervisor, } from './runtime.js'; import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; -import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { GITHUB_ALLOWED_DOMAINS, GITHUB_CREDENTIAL_ENV_NAME, @@ -659,7 +659,7 @@ async function run( }); const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? new NativeSrtWorkspaceCommandSandbox({ + ? new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: canonicalWorkerDirectory!, protectedPaths: [ identityPath, @@ -738,6 +738,7 @@ async function run( await github.provider?.getCredential(controller.signal); await nativeCommandSandbox?.prepare(); } catch (error) { + await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); throw error; } diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 74ed37d4..712e7487 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -6,5 +6,6 @@ export * from './runtime.js'; export * from './workspace.js'; export * from './workspace-runtime.js'; export * from './native-sandbox.js'; +export * from './native-process.js'; export * from './github.js'; export * from './worker.js'; diff --git a/packages/code/src/native-process-child.test.ts b/packages/code/src/native-process-child.test.ts new file mode 100644 index 00000000..1d4866ae --- /dev/null +++ b/packages/code/src/native-process-child.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { fork } from 'node:child_process'; +import test from 'node:test'; +import { nativeExecutorEnvironment } from './native-process.js'; + +for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM'] as const) { + test( + `executor routes ${signal} through shutdown rather than default signal exit`, + { skip: process.platform === 'win32', timeout: 10_000 }, + async (t) => { + const child = fork( + new URL('./native-process-child.js', import.meta.url), + [], + { + execArgv: [], + env: nativeExecutorEnvironment(process.env), + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + }, + ); + t.after(() => { + child.kill('SIGKILL'); + }); + const exited = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve, reject) => { + child.once('error', reject); + child.once('exit', (code, exitSignal) => + resolve({ code, signal: exitSignal }), + ); + }); + // An invalid-state reply proves module initialization and signal-handler + // installation finished without requiring platform SRT dependencies. + child.once('message', () => child.kill(signal)); + child.send({ id: 'startup-probe', type: 'probe' }); + assert.deepEqual(await exited, { code: 1, signal: null }); + }, + ); +} diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts new file mode 100644 index 00000000..164147e3 --- /dev/null +++ b/packages/code/src/native-process-child.ts @@ -0,0 +1,111 @@ +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { WorkspaceToolError } from './workspace.js'; +import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; +import type { WorkspaceExecuteCommandRequest } from './protocol.js'; + +// This entrypoint is private to a forked trusted executor. No HTTP listener, +// argv credentials, bridge token, or persisted pairing material is required. +let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; +let active: { id: string; controller: AbortController } | undefined; +let busy = false; +let credentials: Record = {}; +let wrappedCommand: string | undefined; + +if (!process.send) throw new Error('Native executor requires IPC'); +function reply(message: object): void { + if (!process.connected) return; + try { + process.send?.({ ...message, fatal: shuttingDown }, () => undefined); + } catch { + /* Parent was lost. */ + } +} +let shuttingDown = false; +const shutdown = () => { + if (shuttingDown) return; + shuttingDown = true; + active?.controller.abort(); + void (sandbox?.close() ?? Promise.resolve()).finally(() => process.exit(1)); + setTimeout(() => process.exit(1), 5000); +}; +process.on('disconnect', shutdown); +process.on('SIGTERM', shutdown); +process.on('SIGINT', shutdown); +process.on('SIGHUP', shutdown); +process.on('message', async (raw: unknown) => { + if (shuttingDown) return; + const message = raw as { + id: string; + type: string; + options: Omit< + NativeSrtWorkspaceCommandSandboxOptions, + 'maskedEnvironment' + > & { + variables?: NonNullable< + NativeSrtWorkspaceCommandSandboxOptions['maskedEnvironment'] + >['variables']; + }; + request: WorkspaceExecuteCommandRequest; + credentials?: Record; + wrappedCommand?: string; + }; + if (!message || typeof message.id !== 'string') return; + if (message.type === 'cancel') { + if (active?.id === message.id) active.controller.abort(); + return; + } + if (busy) return; + busy = true; + try { + let result: unknown; + if (message.type === 'prepare' && !sandbox) { + const { variables, ...options } = message.options; + sandbox = new NativeSrtWorkspaceCommandSandbox({ + ...options, + ...(variables + ? { + maskedEnvironment: { + variables, + async resolve() { + return credentials; + }, + wrapCommand(command) { + return wrappedCommand ?? command; + }, + }, + } + : {}), + }); + await sandbox.prepare(); + } else if (message.type === 'execute' && sandbox) { + active = { id: message.id, controller: new AbortController() }; + credentials = message.credentials ?? {}; + wrappedCommand = message.wrappedCommand; + result = await sandbox.execute(message.request, active.controller.signal); + } else if (message.type === 'close' && sandbox) { + await sandbox.close(); + } else throw new Error('Invalid executor state'); + reply({ id: message.id, ok: true, result }); + } catch (error) { + reply({ + id: message.id, + ok: false, + code: + error instanceof WorkspaceToolError + ? error.code + : 'COMMAND_UNAVAILABLE', + ...(error instanceof WorkspaceToolError + ? { errorMessage: error.message.slice(0, 1024) } + : {}), + mutation: + error instanceof WorkspaceToolError + ? error.mutationMayHaveCommitted + : true, + }); + } finally { + active = undefined; + credentials = {}; + wrappedCommand = undefined; + busy = false; + } +}); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts new file mode 100644 index 00000000..ffb068e4 --- /dev/null +++ b/packages/code/src/native-process.test.ts @@ -0,0 +1,380 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import test from 'node:test'; +import type { ChildProcess, ForkOptions } from 'node:child_process'; +import { + NativeProcessWorkspaceCommandSandbox, + nativeExecutorEnvironment, +} from './native-process.js'; +import { WorkspaceToolError } from './workspace.js'; + +const request = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'printf ok', + timeoutMs: 1000, + maxOutputBytes: 64, +}; +const result = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + stdout: 'ok', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, +}; + +function fixture( + execute?: (child: EventEmitter, message: Record) => void, + prepare?: (child: EventEmitter, message: Record) => void, +) { + const child = new EventEmitter() as ChildProcess; + let options: ForkOptions | undefined; + let killCalls = 0; + const messages: Record[] = []; + Object.assign(child, { + connected: true, + send(message: Record, callback: (error: null) => void) { + messages.push(message); + callback(null); + queueMicrotask(() => { + if (message.type === 'prepare' && prepare) + return prepare(child, message); + if (message.type === 'execute' && execute) + return execute(child, message); + if (message.type === 'cancel') return; + child.emit('message', { + id: message.id, + ok: true, + ...(message.type === 'execute' ? { result } : {}), + }); + }); + return true; + }, + kill() { + killCalls += 1; + child.emit('exit', 1); + return true; + }, + }); + return { + get killCalls() { + return killCalls; + }, + child, + messages, + get options() { + return options; + }, + fork(_path: URL, args: string[], value: ForkOptions) { + assert.deepEqual(args, []); + options = value; + return child; + }, + }; +} + +test('executor bootstrap excludes bridge credentials and Node injection variables', async () => { + assert.deepEqual( + nativeExecutorEnvironment({ + PATH: '/bin', + HOME: '/home/user', + NODE_OPTIONS: '--require bad.js', + LIBRECHAT_CODE_WORKER_TOKEN: 'secret', + GITHUB_TOKEN: 'secret', + AWS_SECRET_ACCESS_KEY: 'secret', + }), + { PATH: '/bin', HOME: '/home/user' }, + ); + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + environment: { + PATH: '/bin', + NODE_OPTIONS: 'secret', + LIBRECHAT_CODE_WORKER_TOKEN: 'secret', + }, + }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.options?.execArgv, []); + assert.deepEqual(fake.options?.env, { PATH: '/bin' }); + assert.equal(JSON.stringify(fake.messages).includes('secret'), false); + await sandbox.close(); +}); + +test('executor hands credentials over IPC only for the current command', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + maskedEnvironment: { + variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], + async resolve() { + return { TOKEN: 'per-command-secret' }; + }, + wrapCommand(command) { + return `wrapped ${command}`; + }, + }, + }, + fake.fork, + ); + assert.deepEqual(await sandbox.execute(request), result); + assert.equal( + JSON.stringify(fake.options).includes('per-command-secret'), + false, + ); + assert.equal( + JSON.stringify(fake.messages[0]).includes('per-command-secret'), + false, + ); + assert.deepEqual(fake.messages[1].credentials, { + TOKEN: 'per-command-secret', + }); + assert.equal(fake.messages[1].wrappedCommand, 'wrapped printf ok'); + await sandbox.close(); +}); + +test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { + const fake = fixture((child) => child.emit('exit', 1)); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + ); + await assert.rejects(sandbox.execute(request), /unavailable/); + assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + await sandbox.close(); +}); + +test('executor cancellation targets the active request and preserves mutation certainty', async () => { + let dispatched!: () => void; + const dispatch = new Promise((resolve) => { + dispatched = resolve; + }); + const fake = fixture(() => dispatched()); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + const controller = new AbortController(); + const execution = sandbox.execute(request, controller.signal); + await dispatch; + await assert.rejects(sandbox.execute(request), /unavailable/); + controller.abort(); + const command = fake.messages.find((m) => m.type === 'execute')!; + assert.deepEqual(fake.messages.at(-1), { type: 'cancel', id: command.id }); + fake.child.emit('message', { + id: command.id, + ok: false, + code: 'EXECUTION_ABORTED', + mutation: true, + }); + await assert.rejects( + execution, + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + error.mutationMayHaveCommitted, + ); + await sandbox.close(); +}); + +test('executor rejects mismatched results as uncertain and fences subsequent commands', async () => { + const fake = fixture((child, message) => + child.emit('message', { + id: message.id, + ok: true, + result: { ...result, workspaceId: 'another-workspace' }, + }), + ); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + ); + await assert.rejects(sandbox.execute(request), /unavailable/); + await sandbox.close(); +}); + +test('executor close drains an active command before closing IPC', async () => { + let dispatched!: () => void; + const dispatch = new Promise((resolve) => { + dispatched = resolve; + }); + const fake = fixture(() => dispatched()); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + const execution = sandbox.execute(request); + await dispatch; + const closing = sandbox.close(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal( + fake.messages.some((m) => m.type === 'close'), + false, + ); + const command = fake.messages.find((m) => m.type === 'execute')!; + fake.child.emit('message', { id: command.id, ok: true, result }); + assert.deepEqual(await execution, result); + await closing; + assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor startup loss is not reported as an applied mutation', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + (path, args, options) => { + const child = fake.fork(path, args, options); + queueMicrotask(() => child.emit('error', new Error('startup failed'))); + return child; + }, + ); + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted, + ); + assert.equal( + fake.messages.some((m) => m.type === 'execute'), + false, + ); + await sandbox.close(); +}); + +test('executor shutdown receipt fences reuse before the OS exit event', async () => { + const fake = fixture((child, message) => + child.emit('message', { + id: message.id, + ok: false, + fatal: true, + mutation: true, + code: 'EXECUTION_ABORTED', + }), + ); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await assert.rejects(sandbox.execute(request)); + await assert.rejects(sandbox.execute(request), /unavailable/); + assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + await sandbox.close(); +}); + +test('executor preserves bounded startup diagnostics and conventional host settings', async () => { + assert.deepEqual( + nativeExecutorEnvironment( + { + HTTPS_PROXY: 'http://proxy:8080', + PATHEXT: '.EXE', + NODE_OPTIONS: 'unsafe', + }, + 'win32', + ), + { HTTPS_PROXY: 'http://proxy:8080', PATHEXT: '.EXE' }, + ); + const fake = fixture(undefined, (child, message) => + child.emit('message', { + id: message.id, + ok: false, + mutation: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'Native sandbox dependencies are unavailable: bubblewrap', + }), + ); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await assert.rejects( + sandbox.prepare(), + /dependencies are unavailable: bubblewrap/, + ); + assert.equal( + fake.killCalls, + 1, + 'failed prepare must terminate without caller cleanup', + ); + await sandbox.close(); +}); + +test('executor matches POSIX names exactly and folds names only on Windows', () => { + const env = { + PATH: '/bin', + Path: 'private', + home: 'private', + Temp: 'private', + PATHEXT: 'private', + https_proxy: 'http://proxy:8080', + custom_PROXY: 'private', + }; + assert.deepEqual(nativeExecutorEnvironment(env, 'linux'), { + PATH: '/bin', + https_proxy: 'http://proxy:8080', + }); + assert.deepEqual( + nativeExecutorEnvironment({ Path: 'C:\\bin', Temp: 'C:\\temp' }, 'win32'), + { Path: 'C:\\bin', Temp: 'C:\\temp' }, + ); +}); + +test('executor classifies every pre-dispatch setup failure as mutation-atomic', async () => { + for (const failure of ['fork', 'credential', 'wrapper', 'abort'] as const) { + const fake = fixture(); + const controller = new AbortController(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + maskedEnvironment: { + variables: [], + async resolve() { + if (failure === 'abort') controller.abort(); + if (failure === 'credential' || failure === 'abort') + throw new Error('private provider error'); + return {}; + }, + wrapCommand(command) { + if (failure === 'wrapper') throw new Error('wrapper failed'); + return command; + }, + }, + }, + failure === 'fork' + ? () => { + throw new Error('fork failed'); + } + : fake.fork, + ); + await assert.rejects( + sandbox.execute(request, controller.signal), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + error.code === + (failure === 'abort' ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE'), + ); + assert.equal( + fake.messages.some((m) => m.type === 'execute'), + false, + ); + await sandbox.close(); + } +}); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts new file mode 100644 index 00000000..fc44e3d6 --- /dev/null +++ b/packages/code/src/native-process.ts @@ -0,0 +1,357 @@ +import { fork } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { WorkspaceToolError } from './workspace.js'; +import { isWorkspaceToolRequest, isWorkspaceToolResult } from './protocol.js'; +import type { ChildProcess, ForkOptions } from 'node:child_process'; +import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; +import type { WorkspaceCommandSandbox } from './workspace.js'; +import type { + WorkspaceExecuteCommandRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; + +export type NativeProcessSandboxOptions = Omit< + NativeSrtWorkspaceCommandSandboxOptions, + 'manager' | 'spawnCommand' | 'platform' +>; + +/** Only OS discovery and conventional proxy settings cross into the executor. + * In particular, never inherit NODE_OPTIONS, bridge identity, or app secrets. */ +export function nativeExecutorEnvironment( + source: NodeJS.ProcessEnv, + platform: NodeJS.Platform = process.platform, +): NodeJS.ProcessEnv { + const allowed = new Set([ + 'PATH', + 'HOME', + 'TMPDIR', + 'LANG', + 'LC_ALL', + 'LC_CTYPE', + 'LOGNAME', + 'USER', + 'SHELL', + 'TERM', + 'COLORTERM', + 'NO_COLOR', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'ALL_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'all_proxy', + 'no_proxy', + ]); + if (platform === 'win32') { + for (const name of [ + 'USERPROFILE', + 'SYSTEMROOT', + 'WINDIR', + 'COMSPEC', + 'TEMP', + 'TMP', + 'LOCALAPPDATA', + 'APPDATA', + 'PROGRAMDATA', + 'PROGRAMFILES', + 'PROGRAMFILES(X86)', + 'SYSTEMDRIVE', + 'PATHEXT', + 'HOMEDRIVE', + 'HOMEPATH', + ]) { + allowed.add(name); + } + } + return Object.fromEntries( + Object.entries(source).filter( + ([name, value]) => + value != null && + allowed.has(platform === 'win32' ? name.toUpperCase() : name), + ), + ); +} + +/** One persistent, process-isolated SRT manager per workspace. No automatic + * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ +export class NativeProcessWorkspaceCommandSandbox + implements WorkspaceCommandSandbox +{ + readonly mutationFailuresAreAtomic = true as const; + private child?: ChildProcess; + private ready?: Promise; + private active?: Promise; + private closing?: Promise; + private failed = false; + private terminationTimer?: ReturnType; + private pending?: { + id: string; + resolve(value: unknown): void; + reject(error: Error): void; + mutation: boolean; + }; + + constructor( + private readonly options: NativeProcessSandboxOptions, + private readonly forkExecutor: ( + path: URL, + args: string[], + options: ForkOptions, + ) => ChildProcess = fork, + ) {} + + async prepare(): Promise { + if (this.failed || this.closing) throw this.unavailable(false); + if (this.ready) return this.ready; + this.ready = this.start(); + return this.ready; + } + + private unavailable(mutation: boolean): WorkspaceToolError { + return new WorkspaceToolError( + 'Native executor is unavailable', + 'COMMAND_UNAVAILABLE', + mutation, + ); + } + + private async start(): Promise { + const child = this.forkExecutor( + new URL('./native-process-child.js', import.meta.url), + [], + { + execArgv: [], + env: nativeExecutorEnvironment(this.options.environment ?? process.env), + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + serialization: 'json', + }, + ); + this.child = child; + child.on('message', (raw: unknown) => { + const message = raw as { + id?: unknown; + ok?: unknown; + result?: unknown; + mutation?: unknown; + code?: unknown; + errorMessage?: unknown; + fatal?: unknown; + }; + if ( + !message || + typeof message !== 'object' || + message.id !== this.pending?.id + ) + return; + const pending = this.pending; + if (!pending) return; + if (message.fatal === true) this.failed = true; + if (message.ok === true) pending.resolve(message.result); + else { + const code = + message.code === 'INVALID_PATH' || + message.code === 'INVALID_REQUEST' || + message.code === 'EXECUTION_ABORTED' || + message.code === 'REGISTRATION_INVALID' + ? message.code + : 'COMMAND_UNAVAILABLE'; + pending.reject( + new WorkspaceToolError( + typeof message.errorMessage === 'string' && + message.errorMessage.length <= 1024 + ? message.errorMessage + : 'Native executor request failed', + code, + pending.mutation && message.mutation !== false, + ), + ); + } + }); + const lost = () => { + this.failed = true; + this.pending?.reject(this.unavailable(this.pending.mutation)); + }; + child.on('error', lost); + child.on('exit', lost); + child.on('disconnect', lost); + const { + workspaceRoot, + protectedPaths, + allowedDomains, + homeDirectory, + shellPath, + } = this.options; + await this.rpc( + 'prepare', + { + options: { + workspaceRoot, + protectedPaths, + allowedDomains, + homeDirectory, + shellPath, + variables: this.options.maskedEnvironment?.variables, + }, + }, + 30_000, + false, + ).catch((error) => { + this.failed = true; + this.terminate(); + throw error; + }); + } + + async execute( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + ): Promise { + if ( + !isWorkspaceToolRequest(request) || + request.operation !== 'execute_command' + ) { + throw new WorkspaceToolError('Invalid native command', 'INVALID_REQUEST'); + } + if (this.active || this.closing || this.failed) + throw this.unavailable(false); + const active = this.executeOnce(request, signal); + this.active = active; + try { + return await active; + } finally { + this.active = undefined; + } + } + + private async executeOnce( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) + throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + let credentials: Record | undefined; + let wrappedCommand: string | undefined; + try { + await this.prepare(); + if (signal?.aborted) throw new Error('aborted'); + credentials = await this.options.maskedEnvironment?.resolve(signal); + if (signal?.aborted) throw new Error('aborted'); + wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( + request.command, + process.platform, + ); + if (signal?.aborted) throw new Error('aborted'); + } catch (error) { + // No execute RPC has been sent: setup, token refresh and wrapping cannot + // have mutated the workspace. Do not quarantine it for setup failures. + if (signal?.aborted) + throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError(error.message, error.code, false) + : new WorkspaceToolError( + 'Native executor setup failed before dispatch', + 'COMMAND_UNAVAILABLE', + ); + } + const result = await this.rpc( + 'execute', + { request, credentials, wrappedCommand }, + (request.timeoutMs ?? 30_000) + 5_000, + true, + signal, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + true, + ); + } + if (!isWorkspaceToolResult(request, result)) { + this.failed = true; + this.terminate(); + throw this.unavailable(true); + } + return result as WorkspaceExecuteCommandResult; + } + + private async rpc( + type: string, + payload: object, + timeoutMs: number, + mutation: boolean, + signal?: AbortSignal, + ): Promise { + if (this.pending || !this.child?.connected || this.failed) + throw this.unavailable(false); + const id = randomUUID(); + const child = this.child; + let timer: ReturnType; + const abort = () => { + try { + if (child.connected) + child.send({ type: 'cancel', id }, () => undefined); + } catch { + this.failed = true; + this.terminate(); + } + }; + try { + return await new Promise((resolve, reject) => { + this.pending = { id, resolve, reject, mutation }; + timer = setTimeout(() => { + this.failed = true; + this.terminate(); + reject(this.unavailable(mutation)); + }, timeoutMs); + signal?.addEventListener('abort', abort, { once: true }); + const sendFailed = () => { + this.failed = true; + this.terminate(); + reject(this.unavailable(mutation)); + }; + try { + child.send({ type, id, ...payload }, (error) => { + if (error) sendFailed(); + }); + } catch { + sendFailed(); + } + if (signal?.aborted) abort(); + }); + } finally { + clearTimeout(timer!); + signal?.removeEventListener('abort', abort); + this.pending = undefined; + } + } + + async close(): Promise { + if (this.closing) return this.closing; + this.closing = this.stop(); + return this.closing; + } + + private async stop(): Promise { + await this.active?.catch(() => undefined); + await this.ready?.catch(() => undefined); + try { + if (this.child?.connected && !this.failed) + await this.rpc('close', {}, 10_000, false); + } finally { + this.failed = true; + this.terminate(); + } + } + + private terminate(): void { + const child = this.child; + if (!child || this.terminationTimer) return; + // Give SRT time to abort/reap its command, then bound executor shutdown. + this.terminationTimer = setTimeout(() => child.kill('SIGKILL'), 6000); + this.terminationTimer.unref(); + child.once('exit', () => clearTimeout(this.terminationTimer)); + child.kill('SIGTERM'); + } +} From feb9ed52f469c156e07fe0b59b10d02c1a1df7af Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 22:14:45 -0400 Subject: [PATCH 071/116] fix: Preserve Replacement Bridge Lease Ownership (#165) * fix: Preserve Replacement Bridge Lease Ownership * test: Cover Stateful Cleanup Ownership --- service/src/bridge/cleanup-ownership.test.ts | 82 ++++++++++++++++++++ service/src/bridge/store.ts | 6 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 service/src/bridge/cleanup-ownership.test.ts diff --git a/service/src/bridge/cleanup-ownership.test.ts b/service/src/bridge/cleanup-ownership.test.ts new file mode 100644 index 00000000..e9941c24 --- /dev/null +++ b/service/src/bridge/cleanup-ownership.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; + +const redisUrl = process.env.BRIDGE_TEST_REDIS_URL; +for (const backend of ['mock', 'redis'] as const) { + const suite = backend === 'redis' && !redisUrl ? describe.skip : describe; + suite(`lease cleanup ownership (${backend})`, () => { + let redis: Redis; + let admin: Redis | undefined; + let prefix: string; + afterEach(async () => { + if (admin) { + const keys = await admin.keys(`${prefix}*`); + if (keys.length) await admin.del(...keys); + admin.disconnect(); + admin = undefined; + } else { + await redis.flushall(); + } + redis.disconnect(); + }); + + for (const stateful of [false, true]) { + for (const acknowledged of [false, true]) { + test(`delayed cleanup preserves replacement lease (acknowledged=${acknowledged}, stateful=${stateful})`, async () => { + prefix = `cleanup-test:${randomUUID()}:`; + if (backend === 'redis') { + admin = new Redis(redisUrl!); + redis = new Redis(redisUrl!, { keyPrefix: prefix }); + } else { + redis = new RedisMock() as unknown as Redis; + } + const store = new RedisBridgeStore(redis); + const workerId = 'cleanup-owner'; + const incarnationId = 'cleanup-incarnation'; + const claim = `codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:lease-claim`; + const ack = `codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:lease-ack`; + const assignmentId = 'old-assignment'; + const replacementId = 'replacement-assignment'; + const lock = `codeapi:bridge:v1:worker:${workerId}:lock`; + const runtimeSessionId = stateful ? 'old-workspace' : undefined; + const marker = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash( + 'sha256', + ) + .update(runtimeSessionId ?? '') + .digest('hex')}:quarantined`; + await redis.set(claim, replacementId); + if (acknowledged) await redis.set(ack, replacementId); + await redis.set(lock, replacementId); + await redis.set(`codeapi:bridge:v1:assignment:${assignmentId}`, '{}'); + if (stateful) await redis.set(marker, assignmentId); + // Model an already-issued cleanup that resumes after the worker's + // previous lock expired and a replacement assignment claimed it. + const cleanupStore = store as unknown as { + cleanup(assignment: CodeBridgeAssignment): Promise; + }; + await cleanupStore.cleanup({ + workerId, + incarnationId, + assignmentId, + runtimeSessionId, + } as CodeBridgeAssignment); + expect(await redis.get(claim)).toBe(replacementId); + expect(await redis.get(ack)).toBe( + acknowledged ? replacementId : null, + ); + expect(await redis.get(lock)).toBe(replacementId); + expect( + await redis.get(`codeapi:bridge:v1:assignment:${assignmentId}`), + ).toBeNull(); + // Unknown previous execution must remain fenced, independently of + // the newer assignment's claim and lock. + if (stateful) expect(await redis.get(marker)).toBe(assignmentId); + }); + } + } + }); +} diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 2fa67336..a00e4f2b 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -1785,7 +1785,11 @@ export class RedisBridgeStore { 'if queued == 0 and acknowledged and ARGV[2] == "1" and redis.call(\'GET\', KEYS[5]) == ARGV[1] then', ' return -1', 'end', - "return redis.call('DEL', KEYS[1], KEYS[3], KEYS[4])", + // A delayed cleanup can outlive its lock. Never erase the next + // assignment's claim or acknowledgement when that happens. + "if claimed then redis.call('DEL', KEYS[3]) end", + "if acknowledged then redis.call('DEL', KEYS[4]) end", + "return redis.call('DEL', KEYS[1])", ].join('\n'); const cleanupResult = Number( await boundedCommand( From bf466d370f33fdadf448ed9e51bf872918d3835d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:17:52 -0400 Subject: [PATCH 072/116] feat(codeapi): import Isolate Native SRT Executor Processes (#166) Source: ClickHouse/ai@f5be2d5adfc8f04dbe49ae21a5b4053663a43b7d Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: danny-avila <110412045+danny-avila@users.noreply.github.com> --- tests/kvm_guest_dns.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 4c073de2..0e038a12 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -21,7 +21,7 @@ cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" # A fresh boot can use Kubernetes DNS/search paths without rebuilding the root. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" -SANDBOX_RESOLV_CONF=$'nameserver 10.96.0.10\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5' +SANDBOX_RESOLV_CONF=$'nameserver 10.96.0.10\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5' # leak-check:allow printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_DIR/expected" configure_guest_dns "$TEST_DIR/guest" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" @@ -29,7 +29,7 @@ cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" # The launcher entrypoint joins directives with a separator so the handoff # survives the kernel command line; the guest expands it back into lines. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" -SANDBOX_RESOLV_CONF='nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' +SANDBOX_RESOLV_CONF='nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' # leak-check:allow configure_guest_dns "$TEST_DIR/guest" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" [[ ! -v SANDBOX_RESOLV_CONF ]] @@ -134,9 +134,9 @@ cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" # Kubernetes resolvers: tabs, CRLF, trailing spaces, and unknown keywords are # normalized away; search domains and options survive intact. -printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" +printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" # leak-check:allow run_entrypoint "$TEST_DIR/k8s-resolv.conf" -[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] # leak-check:allow # The runner's own resolver must round-trip through the same reference. run_entrypoint /etc/resolv.conf From f3572e042d1ea59334fda1fd97cf232dcac381fd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 8 Sep 2026 23:50:44 -0400 Subject: [PATCH 073/116] fix: Harden Native Scratch Cleanup (#168) * fix: Harden Native Scratch Cleanup * fix: Resolve Portable POSIX Cleanup Symbols * fix: Bound Descriptor-Relative Scratch Recovery * fix: Bound Native Scratch Recovery Work * Revert "fix: Bound Native Scratch Recovery Work" This reverts commit eb761149fb8c6c11a09b4bfa6bd0c3c03182dbd6. * fix: Bound Native Scratch Recovery Work --- packages/code/src/native-sandbox.test.ts | 123 +++++++++++++++- packages/code/src/native-sandbox.ts | 63 +++------ packages/code/src/native-scratch.ts | 172 +++++++++++++++++++++++ packages/code/src/private-storage.ts | 7 +- 4 files changed, 321 insertions(+), 44 deletions(-) create mode 100644 packages/code/src/native-scratch.ts diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 2d40db00..a74f3727 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -3,11 +3,15 @@ import { spawn } from 'node:child_process'; import { EventEmitter } from 'node:events'; import { access, + chmod, mkdtemp, mkdir, + open, realpath, + rename, rm, stat, + symlink, writeFile, } from 'node:fs/promises'; import { tmpdir, homedir } from 'node:os'; @@ -19,6 +23,7 @@ import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; const request = { @@ -439,7 +444,7 @@ test('removes scratch storage after a command revokes traversal permissions', as const result = await sandbox.execute({ ...request, command: - 'printf %s "$TMPDIR"; mkdir "$TMPDIR/locked"; touch "$TMPDIR/locked/file"; chmod 000 "$TMPDIR/locked" "$TMPDIR"', + 'printf %s "$TMPDIR"; mkdir -p "$TMPDIR/locked/deeper"; touch "$TMPDIR/locked/deeper/file"; chmod 000 "$TMPDIR/locked/deeper" "$TMPDIR/locked" "$TMPDIR"', }); assert.equal(result.exitCode, 0); @@ -447,6 +452,122 @@ test('removes scratch storage after a command revokes traversal permissions', as await assert.rejects(access(result.stdout)); }); +test('scratch traversal never follows a descendant replaced after inspection', async (t) => { + if (process.platform === 'win32') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-race-')); + const outside = await mkdtemp(join(tmpdir(), 'librechat-code-outside-')); + const descendant = join(root, 'locked'); + const retired = join(root, 'retired'); + const outsideChild = join(outside, 'child'); + t.after(() => rm(root, { recursive: true, force: true })); + t.after(() => rm(outside, { recursive: true, force: true })); + await mkdir(descendant); + await mkdir(outsideChild); + await chmod(outside, 0o711); + await chmod(outsideChild, 0o711); + const rootHandle = await open(root, 'r'); + t.after(() => rootHandle.close()); + let swapped = false; + + await restoreScratchTraversal(rootHandle, { + async afterEntryInspected(_directoryFd, name) { + if (name !== 'locked' || swapped) return; + swapped = true; + await rename(descendant, retired); + await symlink(outside, descendant, 'dir'); + }, + }); + + assert.equal(swapped, true); + assert.equal((await stat(outside)).mode & 0o777, 0o711); + assert.equal((await stat(outsideChild)).mode & 0o777, 0o711); +}); + +test('scratch traversal removes command-created Darwin ACLs', async (t) => { + if (process.platform !== 'darwin') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + const result = await sandbox.execute({ + ...request, + command: + 'printf %s "$TMPDIR"; mkdir -p "$TMPDIR/locked/deeper"; touch "$TMPDIR/locked/deeper/file"; chmod +a "$USER deny list,search,delete_child" "$TMPDIR/locked" "$TMPDIR"; chmod 000 "$TMPDIR/locked" "$TMPDIR"', + }); + + assert.equal(result.exitCode, 0); + await sandbox.close(); + await assert.rejects(access(result.stdout)); +}); + +test('scratch traversal bounds descriptors and work across a deep tree', async (t) => { + if (process.platform === 'win32') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-')); + t.after(() => rm(root, { recursive: true, force: true })); + const directories = [root]; + for (let depth = 0; depth < 100; depth += 1) { + directories.push(join(directories[directories.length - 1], 'd')); + await mkdir(directories[directories.length - 1]); + } + for (const directory of directories.slice(1).reverse()) { + await chmod(directory, 0o000); + } + const rootHandle = await open(root, 'r'); + t.after(() => rootHandle.close()); + + await restoreScratchTraversal(rootHandle); + + assert.equal((await stat(directories[directories.length - 1])).mode & 0o777, 0o700); +}); + +test('scratch traversal rejects trees beyond its recovery depth limit', async (t) => { + if (process.platform === 'win32') return; + const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-limit-')); + t.after(() => rm(root, { recursive: true, force: true })); + let directory = root; + for (let depth = 0; depth < 129; depth += 1) { + directory = join(directory, 'd'); + await mkdir(directory); + } + const rootHandle = await open(root, 'r'); + t.after(() => rootHandle.close()); + + await assert.rejects( + restoreScratchTraversal(rootHandle), + /scratch cleanup exceeded its depth limit/, + ); +}); + +test('does not replace scratch state while cleanup remains pending', async (t) => { + if (process.platform === 'win32') return; + const workspace = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + const retained = await mkdtemp(join(tmpdir(), 'librechat-code-retained-')); + const retainedHandle = await open(retained, 'r'); + t.after(() => retainedHandle.close()); + t.after(() => rm(retained, { recursive: true, force: true })); + t.after(() => rm(workspace, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: workspace, + manager: fakeManager().manager, + }); + const mutable = sandbox as unknown as { + scratchDirectory?: string; + scratchHandle?: typeof retainedHandle; + createScratchDirectory(paths: string[]): Promise; + }; + mutable.scratchDirectory = retained; + mutable.scratchHandle = retainedHandle; + + await assert.rejects( + mutable.createScratchDirectory([]), + /scratch cleanup is still pending/, + ); + assert.equal(mutable.scratchDirectory, retained); + assert.equal(mutable.scratchHandle, retainedHandle); +}); + const proxyEnvironment = { HTTP_PROXY: 'http://upstream.invalid:8080', HTTPS_PROXY: 'http://upstream.invalid:8080', diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index bc570442..49d269c0 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -13,15 +13,13 @@ import { import { constants as fsConstants } from 'node:fs'; import { access, - chmod, - lstat, mkdtemp, open, - readdir, realpath, rm, stat, } from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -37,6 +35,7 @@ import { removePrivateStorageAcl, } from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; +import { restoreScratchTraversal } from './native-scratch.js'; import type { ChildProcessWithoutNullStreams, @@ -245,6 +244,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private initialized?: Promise; private canonicalRoot?: string; private scratchDirectory?: string; + private scratchHandle?: FileHandle; private execution?: Promise; private closing?: Promise; private resetFailed = false; @@ -748,6 +748,11 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ): Promise { // Windows SRT supplies the restricted account's private TEMP directory. if (this.platform === 'win32') return undefined; + if (this.scratchDirectory || this.scratchHandle) { + throw new Error( + 'Native sandbox scratch cleanup is still pending; close the sandbox before reinitializing', + ); + } const canonicalTemporaryRoot = await canonicalPath(HOST_TEMPORARY_ROOT); const sharedScratchRoot = sharedScratchPaths.find((path) => isWithin(path, canonicalTemporaryRoot), @@ -774,12 +779,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox if (((await scratchHandle.stat()).mode & 0o777) !== 0o700) { throw new Error('Native sandbox scratch directory is not private'); } - } finally { + this.scratchHandle = scratchHandle; + } catch (error) { await scratchHandle.close(); + throw error; } this.scratchDirectory = await realpath(scratchDirectory); return this.scratchDirectory; } catch (error) { + await this.scratchHandle?.close().catch(() => undefined); + this.scratchHandle = undefined; await rm(scratchDirectory, { recursive: true, force: true }).catch( () => undefined, ); @@ -810,51 +819,23 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private async removeScratchDirectory(): Promise { const scratchDirectory = this.scratchDirectory; if (!scratchDirectory) return; + const scratchHandle = this.scratchHandle; + if (!scratchHandle) { + throw new Error('Native sandbox scratch descriptor is unavailable'); + } try { await rm(scratchDirectory, { recursive: true, force: true }); } catch { - await this.restoreScratchTraversal(scratchDirectory); + await restoreScratchTraversal(scratchHandle); await rm(scratchDirectory, { recursive: true, force: true }); } + // Retain both the descriptor and path when cleanup fails so close() can + // retry without falling back to an attacker-replaceable ambient path. + await scratchHandle.close(); + this.scratchHandle = undefined; this.scratchDirectory = undefined; } - private async restoreScratchTraversal(root: string): Promise { - const pending = [root]; - for (let index = 0; index < pending.length; index += 1) { - const directory = pending[index]; - const metadata = await lstat(directory).catch( - (error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') return undefined; - throw error; - }, - ); - if (!metadata?.isDirectory()) continue; - // Commands own their scratch contents and may remove all directory mode - // bits. Restore traversal before opening the directory with O_NOFOLLOW. - await chmod(directory, 0o700); - let handle; - try { - handle = await open( - directory, - fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW, - ); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (['ENOENT', 'ELOOP', 'ENOTDIR'].includes(code ?? '')) continue; - throw error; - } - try { - if (!(await handle.stat()).isDirectory()) continue; - } finally { - await handle.close(); - } - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (entry.isDirectory()) pending.push(join(directory, entry.name)); - } - } - } - async close(): Promise { if (this.closing) return this.closing; const closing = this.closeExclusive(); diff --git a/packages/code/src/native-scratch.ts b/packages/code/src/native-scratch.ts new file mode 100644 index 00000000..0ecd330e --- /dev/null +++ b/packages/code/src/native-scratch.ts @@ -0,0 +1,172 @@ +import { constants as fsConstants } from 'node:fs'; +import { opendir } from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; + +import koffi from 'koffi'; + +import { removePrivateStorageAcl } from './private-storage.js'; + +const POSIX_PLATFORMS = new Set(['darwin', 'linux']); +const lib = POSIX_PLATFORMS.has(process.platform) ? koffi.load(null) : undefined; +const openat = lib?.func('int openat(int dirfd, const char *path, int flags, uint32_t mode)'); +const closeFd = lib?.func('int close(int fd)'); +const fchmod = lib?.func('int fchmod(int fd, uint32_t mode)'); +const fchmodat = lib?.func( + 'int fchmodat(int dirfd, const char *path, uint32_t mode, int flags)', +); +const AT_SYMLINK_NOFOLLOW = process.platform === 'darwin' ? 0x0020 : 0x0100; +const O_EVTONLY = 0x8000; +// Recovery is a last-resort shutdown path over an attacker-controlled tree. +// Keep both its memory use and its descriptor-relative reopen work bounded. +const MAX_SCRATCH_DIRECTORIES = 10_000; +const MAX_SCRATCH_ENTRIES = 100_000; +const MAX_SCRATCH_DEPTH = 128; +const MAX_SCRATCH_COMPONENT_VISITS = 16_384; +const IGNORED_ENTRY_ERRNOS = new Set([ + koffi.os.errno.ENOENT, + koffi.os.errno.ELOOP, + koffi.os.errno.ENOTDIR, + koffi.os.errno.ENOTSUP, +]); + +export interface ScratchTraversalHooks { + /** Test seam for deterministic replacement-race coverage. */ + afterEntryInspected?(directoryFd: number, name: string): Promise; +} + +function descriptorPath(fd: number): string { + return process.platform === 'linux' ? `/proc/self/fd/${fd}` : `/dev/fd/${fd}`; +} + +function requirePosixBindings(): void { + if (!openat || !closeFd || !fchmod || !fchmodat) { + throw new Error('Descriptor-relative scratch cleanup is unavailable'); + } +} + +function ignoredEntryError(): boolean { + return IGNORED_ENTRY_ERRNOS.has(koffi.errno()); +} + +function restoreEntryMode(directoryFd: number, name: string): boolean { + requirePosixBindings(); + if (fchmodat!(directoryFd, name, 0o700, AT_SYMLINK_NOFOLLOW) === 0) return true; + if (ignoredEntryError()) return false; + throw new Error(`Descriptor-relative scratch chmod failed with errno ${koffi.errno()}`); +} + +function openDirectoryAt(directoryFd: number, name: string): number | undefined { + requirePosixBindings(); + const commonFlags = fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW; + const repairFlags = process.platform === 'darwin' + ? commonFlags | O_EVTONLY + : commonFlags | fsConstants.O_RDONLY; + const fd = openat!(directoryFd, name, repairFlags, 0); + if (fd >= 0) return fd; + if (ignoredEntryError()) return undefined; + throw new Error(`Descriptor-relative scratch open failed with errno ${koffi.errno()}`); +} + +function closeDirectory(fd: number): void { + requirePosixBindings(); + if (closeFd!(fd) !== 0) { + throw new Error(`Descriptor-relative scratch close failed with errno ${koffi.errno()}`); + } +} + +async function repairDirectory(fd: number, label: string): Promise { + requirePosixBindings(); + if (fchmod!(fd, 0o700) !== 0) { + throw new Error(`Descriptor-relative scratch chmod failed with errno ${koffi.errno()}`); + } + await removePrivateStorageAcl({ fd }, label); +} + +async function openRelativeDirectory( + rootFd: number, + components: string[], + hooks: ScratchTraversalHooks, + consumeComponentVisit: () => void, +): Promise { + let currentFd = rootFd; + try { + for (const component of components) { + consumeComponentVisit(); + await hooks.afterEntryInspected?.(currentFd, component); + if (!restoreEntryMode(currentFd, component)) { + if (currentFd !== rootFd) { + const closingFd = currentFd; + currentFd = rootFd; + closeDirectory(closingFd); + } + return undefined; + } + const childFd = openDirectoryAt(currentFd, component); + if (currentFd !== rootFd) { + const closingFd = currentFd; + currentFd = rootFd; + closeDirectory(closingFd); + } + if (childFd === undefined) return undefined; + currentFd = childFd; + await repairDirectory(currentFd, `scratch directory ${components.join('/')}`); + } + return currentFd; + } catch (error) { + if (currentFd !== rootFd) closeDirectory(currentFd); + throw error; + } +} + +/** + * Restores traversal without resolving worker-controlled descendants through + * ambient paths. Relative component lists retain no descriptors; reopening a + * path holds at most two descriptors and refuses replacement symlinks. + */ +export async function restoreScratchTraversal( + root: FileHandle, + hooks: ScratchTraversalHooks = {}, +): Promise { + await root.chmod(0o700); + await removePrivateStorageAcl(root, 'native sandbox scratch root'); + const pending: string[][] = [[]]; + let componentVisits = 0; + const consumeComponentVisit = () => { + componentVisits += 1; + if (componentVisits > MAX_SCRATCH_COMPONENT_VISITS) { + throw new Error('Native sandbox scratch cleanup exceeded its work limit'); + } + }; + let entriesInspected = 0; + for (let index = 0; index < pending.length; index += 1) { + const components = pending[index]; + const directoryFd = components.length === 0 + ? root.fd + : await openRelativeDirectory( + root.fd, + components, + hooks, + consumeComponentVisit, + ); + if (directoryFd === undefined) continue; + try { + const directory = await opendir(descriptorPath(directoryFd)); + for await (const entry of directory) { + entriesInspected += 1; + if (entriesInspected > MAX_SCRATCH_ENTRIES) { + throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + } + if (!entry.isDirectory()) continue; + if (components.length >= MAX_SCRATCH_DEPTH) { + throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); + } + if (pending.length >= MAX_SCRATCH_DIRECTORIES) { + throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); + } + pending.push([...components, entry.name]); + } + } finally { + if (directoryFd !== root.fd) closeDirectory(directoryFd); + } + } +} diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts index af4f185a..25f23a2a 100644 --- a/packages/code/src/private-storage.ts +++ b/packages/code/src/private-storage.ts @@ -25,7 +25,7 @@ async function macOsStorage() { } export async function assertPrivateStorageAcl( - handle: FileHandle, path: string, directory = false, + handle: Pick, path: string, directory = false, ): Promise { if (process.platform === 'darwin') { (await macOsStorage()).verifyMacOsAcl(handle.fd, path, directory); @@ -33,7 +33,10 @@ export async function assertPrivateStorageAcl( } /** Only application-owned files/directories may have their ACLs removed. */ -export async function removePrivateStorageAcl(handle: FileHandle, path: string): Promise { +export async function removePrivateStorageAcl( + handle: Pick, + path: string, +): Promise { if (process.platform === 'darwin') { (await macOsStorage()).removeMacOsAcl(handle.fd, path); } From 71308b5b3a18787dbaf57392e9b03a57eb8d7592 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 9 Sep 2026 00:36:40 -0400 Subject: [PATCH 074/116] feat: Add Bounded Concurrent BYOM Workspace Leases (#167) * feat: Add Bounded Concurrent Native Workspace Leases * fix: Bound Workspace Cleanup Waits and Preserve Legacy Heartbeats * fix: Enforce Replica Slot Ceilings at Dispatch * fix: Fence BYOM Workspaces Independently of Settlement * fix: Retain Workspace Ownership Through Cleanup * fix: Preserve Slot Liveness and Retry Finalization * fix: Close Startup and Executor Recovery Gaps --- docker-compose.yaml | 2 + packages/code/README.md | 60 ++ packages/code/src/cli-slots.test.ts | 143 ++++ packages/code/src/cli.ts | 311 +++++--- packages/code/src/native-pool.test.ts | 154 ++++ packages/code/src/native-pool.ts | 152 ++++ packages/code/src/protocol.ts | 9 + packages/code/src/worker-slots.test.ts | 314 ++++++++ packages/code/src/worker.ts | 590 ++++++++++++++- packages/code/src/workspace-guards.ts | 49 ++ service/src/bridge/admission.ts | 27 +- service/src/bridge/concurrent-store.test.ts | 533 ++++++++++++++ service/src/bridge/concurrent-worker.test.ts | 303 ++++++++ service/src/bridge/index.ts | 7 +- service/src/bridge/router.ts | 23 +- service/src/bridge/slots.test.ts | 93 +++ service/src/bridge/slots.ts | 139 ++++ service/src/bridge/store.ts | 721 +++++++++++++++---- service/src/config.ts | 4 + tests/compose-bridge-config.cjs | 3 + 20 files changed, 3361 insertions(+), 276 deletions(-) create mode 100644 packages/code/src/cli-slots.test.ts create mode 100644 packages/code/src/native-pool.test.ts create mode 100644 packages/code/src/native-pool.ts create mode 100644 packages/code/src/worker-slots.test.ts create mode 100644 packages/code/src/workspace-guards.ts create mode 100644 service/src/bridge/concurrent-store.test.ts create mode 100644 service/src/bridge/concurrent-worker.test.ts create mode 100644 service/src/bridge/slots.test.ts create mode 100644 service/src/bridge/slots.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index 3409554c..ab37fd24 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,6 +13,7 @@ services: - CODEAPI_BRIDGE_TOKEN=${CODEAPI_BRIDGE_TOKEN:-} - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} + - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} @@ -61,6 +62,7 @@ services: - CODEAPI_BRIDGE_TOKEN=${CODEAPI_BRIDGE_TOKEN:-} - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} + - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} diff --git a/packages/code/README.md b/packages/code/README.md index 7e7c0aaf..819d9f5c 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -520,3 +520,63 @@ with `librechat-code reset-workspace `. The command uses the configured worker credentials, registers a fresh incarnation, and only clears the server fence when no assignment is active. Run it while the normal worker process is stopped, then restart the normal worker after the command exits. + +### Opt-in concurrent native workspaces + +Code API defaults to **one execution slot**. To allow independent native roots +to execute concurrently, configure `CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=2` +on every Code API replica and start an updated worker with: + +```sh +librechat-code run \ + --worker-dir /projects/first \ + --workspace second=/projects/second \ + --workspace-lease-slots 2 \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +Keep the existing URL, pairing/identity, and network policy configuration. +The primary root keeps its configured workspace ID (default `primary`). Repeat +`--workspace id=path` to add named roots, up to the protocol's 32-root limit. +Roots must already exist and must not overlap or alias one another. Commands +retain the selected root's sandbox boundary, not a shared parent-directory grant. +The `LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE` single-file override is rejected +when multiple roots are configured; unset it to use separate root-derived markers. + +`LIBRECHAT_CODE_WORKSPACE_LEASE_SLOTS` is the equivalent worker setting. Both +ceilings must be integers from 1 to 8; the lower ceiling wins. An older Code API +without the negotiation receipt keeps the worker on the serial protocol. Deploy +the updated API to all replicas before enabling slots on workers. A capacity +change while work is active fails closed; stop and drain the worker before +changing it. + +Different roots can run concurrently; requests targeting the **same root remain +serialized**, even across chats or agents. This is root-level exclusion, not +file-level locking. Assign separate project/worktree roots for independent work. +The admission queue remains bounded at 32 requests per worker. An idle SRT process +cache is bounded by the local slot setting and evicts only idle executors. Runtime +sandbox assignments continue through the exclusive legacy lane; this does not +enable concurrent Docker/NsJail sessions or bypass any approval/network policy. + +An uncertain mutation or executor failure leaves an assignment-owned local guard +and a server-side fence for that root. Healthy roots can continue. The worker +does not replay the failed command. A guard-cleanup failure after settlement fences +the root independently without replacing the committed result. Expiring ownership +receipts exclude command payloads; explicit reset invalidates old fence requests. +The server releases a root only after result finalization **and** explicit local +cleanup confirmation. Local guard cleanup has a five-second bound; an expired +receipt never implies a clean root. Control receipt delivery retries three times. +If delivery remains unavailable, the root remains fenced while every advertised +lane keeps polling. Capacity becomes reusable when its owned reservation is +released or expires; inspect/reset the affected root before using it again. +Reset-only registration stays unready and cannot attract new assignments. +To recover a quarantined native root: + +1. Stop the worker and inspect or restore the affected directory. +2. Run `librechat-code clear-workspace-quarantine --worker-dir /projects/second --workspace-id second` using the same deployment/identity configuration. +3. Run the normal worker command with all its root/slot options plus `--reset-workspace-quarantine second`. This verifies the local guard is cleared, resets the server fence, then exits. +4. Restart the normal worker command without the reset option. + +The workspace selector in LibreChat must preserve these registered IDs. Adding +roots here does not grant a principal access or change an agent's selected root. diff --git a/packages/code/src/cli-slots.test.ts b/packages/code/src/cli-slots.test.ts new file mode 100644 index 00000000..484f0dc0 --- /dev/null +++ b/packages/code/src/cli-slots.test.ts @@ -0,0 +1,143 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, mkdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +test('CLI rejects aliased and overlapping workspace roots before connecting', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'byom-cli-roots-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, '..nested')); + for (const extra of [root, join(root, '..nested')]) { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + root, + '--workspace', + `second=${extra}`, + ], + { + encoding: 'utf8', + timeout: 3000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1', + LIBRECHAT_CODE_WORKER_TOKEN: 'fixture', + LIBRECHAT_CODE_WORKER_ID: 'fixture-worker', + LIBRECHAT_CODE_COMMAND_SANDBOX: 'native-srt', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /must not overlap or alias/); + } +}); + +test('CLI bounds requested workspace slots before connecting', () => { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--workspace-lease-slots', + '9', + ], + { + encoding: 'utf8', + timeout: 3000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1', + LIBRECHAT_CODE_WORKER_TOKEN: 'fixture', + LIBRECHAT_CODE_WORKER_ID: 'fixture-worker', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /cannot exceed 8/); +}); + +test('CLI rejects one quarantine-file override for multiple roots', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'byom-cli-markers-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'a')); + await mkdir(join(root, 'b')); + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + join(root, 'a'), + '--workspace', + `second=${join(root, 'b')}`, + ], + { + encoding: 'utf8', + timeout: 3000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1', + LIBRECHAT_CODE_WORKER_TOKEN: 'fixture', + LIBRECHAT_CODE_WORKER_ID: 'fixture-worker', + LIBRECHAT_CODE_COMMAND_SANDBOX: 'native-srt', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: join(root, 'shared.json'), + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /single-root override/); +}); + +test('CLI preserves distinct case-sensitive roots on a non-Linux platform', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'byom-cli-case-')); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'Foo')); + await mkdir(join(root, 'foo'), { recursive: true }); + if ( + (await stat(join(root, 'Foo'))).ino === (await stat(join(root, 'foo'))).ino + ) { + t.skip('requires a case-sensitive test filesystem'); + return; + } + const argv = [ + 'fixture', + 'run', + '--worker-dir', + join(root, 'Foo'), + '--workspace', + `second=${join(root, 'foo')}`, + '--workspace-lease-slots', + '2', + ]; + const result = spawnSync( + process.execPath, + [ + '--input-type=module', + '-e', + `Object.defineProperty(process, 'platform', {value:'darwin'}); process.argv=[process.execPath,...${JSON.stringify(argv)}]; await import(${JSON.stringify(new URL('./cli.js', import.meta.url).href)});`, + ], + { + encoding: 'utf8', + timeout: 3000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1', + LIBRECHAT_CODE_WORKER_TOKEN: 'fixture', + LIBRECHAT_CODE_WORKER_ID: 'fixture-worker', + LIBRECHAT_CODE_COMMAND_SANDBOX: 'native-srt', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: '', + }, + }, + ); + assert.match( + result.stderr, + /Concurrent workspace leases require native-srt commands/, + ); + assert.doesNotMatch(result.stderr, /overlap or alias/); +}); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 67ec396c..27a8d2d9 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { realpath } from 'node:fs/promises'; -import { basename, resolve } from 'node:path'; +import { realpath, stat } from 'node:fs/promises'; +import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { startFileRelay } from './relay.js'; @@ -28,6 +28,10 @@ import { } from './runtime.js'; import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { workspaceMutationGuard } from './workspace-guards.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GITHUB_CREDENTIAL_ENV_NAME, @@ -467,21 +471,114 @@ async function run( workspaceRoot: canonicalWorkerDirectory, }) : undefined; + const workspaceLeaseSlots = positiveInteger( + 'LIBRECHAT_CODE_WORKSPACE_LEASE_SLOTS', + option(args, '--workspace-lease-slots') ?? + process.env.LIBRECHAT_CODE_WORKSPACE_LEASE_SLOTS, + 1, + ); + if (workspaceLeaseSlots > 8) + throw new Error('Workspace lease slots cannot exceed 8'); + const roots: LocalWorkspaceConfig[] = canonicalWorkerDirectory + ? [ + { + id: workspaceId, + root: canonicalWorkerDirectory, + writable: allowWorkspaceWrites, + name: + option(args, '--workspace-name') ?? + process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? + (useDefaultWorkspace + ? workspaceId + : defaultWorkspaceName(workerDirectory!, workspaceId)), + }, + ] + : []; + for (let i = 0; i < args.length; i++) { + if ( + args[i] === '--workspace' && + (!args[i + 1] || args[i + 1].startsWith('--')) + ) { + throw new Error('--workspace requires id=path'); + } + const value = + args[i] === '--workspace' + ? args[++i] + : args[i].startsWith('--workspace=') + ? args[i].slice('--workspace='.length) + : undefined; + if (value === undefined) continue; + const separator = value.indexOf('='); + if ( + separator < 1 || + separator === value.length - 1 || + !canonicalWorkerDirectory || + commandSandboxMode !== 'native-srt' + ) { + throw new Error( + 'Additional --workspace id=path roots require a primary workspace and native-srt', + ); + } + roots.push({ + id: value.slice(0, separator), + root: await realpath(value.slice(separator + 1)), + writable: allowWorkspaceWrites, + }); + } + // Aliases and nested grants are not independent execution domains. + if (roots.length > 32) + throw new Error('At most 32 workspace roots may be registered'); + const rootIdentities = await Promise.all( + roots.map((root) => stat(root.root)), + ); + const normalized = roots.map((root) => root.root); + for (let i = 0; i < roots.length; i++) + for (let j = 0; j < i; j++) { + const inside = (a: string, b: string): boolean => { + const path = relative(a, b); + return ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ); + }; + if ( + (rootIdentities[i].dev === rootIdentities[j].dev && + rootIdentities[i].ino === rootIdentities[j].ino) || + inside(normalized[i], normalized[j]) || + inside(normalized[j], normalized[i]) + ) { + throw new Error( + 'Workspace roots must not overlap or alias one another', + ); + } + } + if ( + workspaceLeaseSlots > 1 && + (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') + ) { + throw new Error('Concurrent workspace leases require native-srt commands'); + } + if ( + roots.length > 1 && + process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() + ) { + throw new Error( + 'LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE is a single-root override; unset it for multiple workspace roots', + ); + } + const rootQuarantinePaths = new Map( + roots.map((root) => [ + root.id, + workspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: root.root, + }), + ]), + ); let workspaceTools: WorkspaceToolExecutor | undefined = workerDirectory ? await LocalWorkspaceTools.create({ - workspaces: [ - { - id: workspaceId, - name: - option(args, '--workspace-name') ?? - process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? - (useDefaultWorkspace - ? workspaceId - : defaultWorkspaceName(workerDirectory, workspaceId)), - root: workerDirectory, - writable: allowWorkspaceWrites, - }, - ], + workspaces: roots, }) : undefined; if (allowWorkspaceCommands && !canonicalWorkerDirectory) { @@ -657,47 +754,58 @@ async function run( endpoint: sandboxEndpoint, statefulWorkspace, }); + const nativeOptions: NativeProcessSandboxOptions = { + workspaceRoot: canonicalWorkerDirectory!, + protectedPaths: [ + identityPath, + ...rootQuarantinePaths.values(), + github.privateKeyPath, + ].filter((path): path is string => path != null), + allowedDomains: commandAllowedDomains, + ...(github.provider + ? { + maskedEnvironment: { + variables: [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: [github.host], + }, + ], + async resolve(signal?: AbortSignal) { + return gitHubCredentialEnvironment( + await github.provider!.getCredential(signal), + ); + }, + wrapCommand(command: string, platform: NodeJS.Platform) { + return wrapGitHubCredentialCommand( + command, + github.host, + platform, + ); + }, + }, + } + : {}), + }; const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? new NativeProcessWorkspaceCommandSandbox({ - workspaceRoot: canonicalWorkerDirectory!, - protectedPaths: [ - identityPath, - mutationQuarantinePath, - github.privateKeyPath, - ].filter((path): path is string => path != null), - allowedDomains: commandAllowedDomains, - ...(github.provider - ? { - maskedEnvironment: { - variables: [ - { - name: GITHUB_CREDENTIAL_ENV_NAME, - extract: '^(.+)$', - injectHosts: [github.host], - }, - ], - async resolve(signal?: AbortSignal) { - return gitHubCredentialEnvironment( - await github.provider!.getCredential(signal), - ); - }, - wrapCommand(command: string, platform: NodeJS.Platform) { - return wrapGitHubCredentialCommand( - command, - github.host, - platform, - ); - }, - }, - } - : {}), - }) + ? roots.length > 1 || workspaceLeaseSlots > 1 + ? new NativeWorkspaceCommandPool( + new Map( + roots.map((root) => [ + root.id, + { ...nativeOptions, workspaceRoot: root.root }, + ]), + ), + workspaceLeaseSlots, + ) + : new NativeProcessWorkspaceCommandSandbox(nativeOptions) : undefined; if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ workspaceTools, - commandWorkspaces: [workspaceId], + commandWorkspaces: roots.map((root) => root.id), commandSandbox: nativeCommandSandbox ?? new RuntimeWorkspaceCommandSandbox({ @@ -726,6 +834,9 @@ async function run( ) .digest('hex'), ...(fileRelayEnabled ? { requiresReadyConfirmation: true } : {}), + ...(workspaceLeaseSlots > 1 + ? { workspaceLeaseSlots, requiresReadyConfirmation: true } + : {}), ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { @@ -752,44 +863,62 @@ async function run( runtimeSupervisor, capabilities, workspaceTools, - workspaceMutationQuarantine: mutationQuarantinePath + ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { - async assertAvailable() { - const record = await loadWorkspaceMutationQuarantine( - mutationQuarantinePath, - ); - if (record != null) { - throw new BridgeProtocolError( - `Workspace mutations are quarantined since ${record.quarantinedAt}: ${record.reason}. Inspect or restore the workspace, then run librechat-code clear-workspace-quarantine`, - undefined, - 'WORKER_QUARANTINED', - ); - } - }, - async arm(reason) { - await saveWorkspaceMutationQuarantine(mutationQuarantinePath, { - version: 1, - workerId, - workspaceId, - ownerId: incarnationId, - quarantinedAt: new Date().toISOString(), - reason, - }); - }, - async clear() { - await clearWorkspaceMutationQuarantine( - mutationQuarantinePath, - incarnationId, - ); - }, - async quarantine() { - await assertWorkspaceMutationQuarantineOwner( - mutationQuarantinePath, - incarnationId, - ); - }, + workspaceQuarantines: new Map( + roots.map((root) => [ + root.id, + workspaceMutationGuard( + rootQuarantinePaths.get(root.id)!, + workerId, + root.id, + incarnationId, + ), + ]), + ), } - : undefined, + : {}), + workspaceMutationQuarantine: + mutationQuarantinePath && + workspaceLeaseSlots === 1 && + roots.length === 1 + ? { + async assertAvailable() { + const record = await loadWorkspaceMutationQuarantine( + mutationQuarantinePath, + ); + if (record != null) { + throw new BridgeProtocolError( + `Workspace mutations are quarantined since ${record.quarantinedAt}: ${record.reason}. Inspect or restore the workspace, then run librechat-code clear-workspace-quarantine`, + undefined, + 'WORKER_QUARANTINED', + ); + } + }, + async arm(reason) { + await saveWorkspaceMutationQuarantine(mutationQuarantinePath, { + version: 1, + workerId, + workspaceId, + ownerId: incarnationId, + quarantinedAt: new Date().toISOString(), + reason, + }); + }, + async clear() { + await clearWorkspaceMutationQuarantine( + mutationQuarantinePath, + incarnationId, + ); + }, + async quarantine() { + await assertWorkspaceMutationQuarantineOwner( + mutationQuarantinePath, + incarnationId, + ); + }, + } + : undefined, onIdentityChange: pairedIdentity && identityPath ? async (identity) => { @@ -832,6 +961,16 @@ async function run( ); return; } + const resetNativeRoot = option(args, '--reset-workspace-quarantine'); + if (resetNativeRoot != null) { + await worker.refreshCredential(controller.signal); + await worker.registerForMaintenance(controller.signal); + await worker.resetNativeWorkspace(resetNativeRoot, controller.signal); + process.stdout.write( + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, + ); + return; + } await worker.run(controller.signal); } finally { try { diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts new file mode 100644 index 00000000..3a24db34 --- /dev/null +++ b/packages/code/src/native-pool.test.ts @@ -0,0 +1,154 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { WorkspaceToolError } from './workspace.js'; +import type { WorkspaceExecuteCommandRequest } from './protocol.js'; + +const roots = new Map( + ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]), +); +const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId, + command: 'fixture', +}); +test('a known-clean executor failure is retired without replaying the command', async () => { + let created = 0; + let executed = 0; + let closed = 0; + const pool = new NativeWorkspaceCommandPool(roots, 2, () => { + const first = ++created === 1; + return { + async prepare() {}, + async close() { + closed++; + }, + async execute(req) { + executed++; + if (first) + throw new WorkspaceToolError( + 'prepare failed', + 'COMMAND_UNAVAILABLE', + false, + ); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }; + }); + await assert.rejects(pool.execute(request('b')), { + code: 'COMMAND_UNAVAILABLE', + }); + assert.equal(executed, 1); + assert.equal(closed, 1); + await pool.execute(request('b')); + assert.equal(created, 2); + await pool.close(); +}); +test('native pool reuses roots and evicts only idle processes within its bound', async () => { + const created: string[] = []; + const closed: string[] = []; + const pool = new NativeWorkspaceCommandPool(roots, 2, (options) => { + created.push(options.workspaceRoot); + return { + async prepare() {}, + async close() { + closed.push(options.workspaceRoot); + }, + async execute(req) { + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }; + }); + await pool.execute(request('a')); + await pool.execute(request('b')); + await pool.execute(request('a')); + assert.equal(created.length, 2); + await pool.execute(request('c')); + assert.deepEqual(closed, ['/fixture/b']); + await pool.close(); + assert.equal(closed.length, 3); +}); + +test('native pool never evicts an executing root or misclassifies pre-dispatch exhaustion', async () => { + let finish!: () => void; + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const pending = new Promise((resolve) => { + finish = resolve; + }); + const pool = new NativeWorkspaceCommandPool(roots, 1, () => ({ + async prepare() {}, + async close() {}, + async execute(req) { + entered(); + await pending; + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + })); + const executing = pool.execute(request('a')); + await started; + await assert.rejects(pool.execute(request('b')), { + code: 'COMMAND_UNAVAILABLE', + mutationMayHaveCommitted: false, + }); + finish(); + await executing; + await pool.close(); +}); + +test('idle eviction failure stays mutation-atomic for the new root', async () => { + const pool = new NativeWorkspaceCommandPool(roots, 1, () => ({ + async prepare() {}, + async close() { + throw new Error('fixture cleanup failure'); + }, + async execute(req) { + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + })); + await pool.execute(request('a')); + await assert.rejects(pool.execute(request('b')), { + code: 'COMMAND_UNAVAILABLE', + mutationMayHaveCommitted: false, + }); + await assert.rejects(pool.close(), AggregateError); +}); diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts new file mode 100644 index 00000000..28f155e2 --- /dev/null +++ b/packages/code/src/native-pool.ts @@ -0,0 +1,152 @@ +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import { WorkspaceToolError } from './workspace.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { + WorkspaceExecuteCommandRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; + +interface Entry { + sandbox: Pick< + NativeProcessWorkspaceCommandSandbox, + 'prepare' | 'execute' | 'close' + >; + busy: boolean; +} + +/** Bounded persistent executor cache. Each child owns one root's SRT policy; + * only idle children may be evicted, and ambiguous failures are never retried. + */ +export class NativeWorkspaceCommandPool { + readonly mutationFailuresAreAtomic = true as const; + private readonly entries = new Map(); + private allocation: Promise = Promise.resolve(); + private closing = false; + constructor( + private readonly roots: ReadonlyMap, + private readonly capacity: number, + private readonly createSandbox: ( + options: NativeProcessSandboxOptions, + ) => Entry['sandbox'] = (options) => + new NativeProcessWorkspaceCommandSandbox(options), + ) { + if ( + !Number.isSafeInteger(capacity) || + capacity < 1 || + capacity > 8 || + roots.size === 0 + ) { + throw new Error('Native executor capacity must be between 1 and 8'); + } + } + + private allocate(root: string): Promise { + const pending = this.allocation.then(async () => { + const options = this.roots.get(root); + if (this.closing || !options) + throw new WorkspaceToolError( + 'Native workspace unavailable', + 'REGISTRATION_INVALID', + ); + let entry = this.entries.get(root); + if (entry?.busy) + throw new WorkspaceToolError( + 'Native workspace already executing', + 'COMMAND_UNAVAILABLE', + ); + if (!entry) { + if (this.entries.size >= this.capacity) { + const idle = [...this.entries].find( + ([, candidate]) => !candidate.busy, + ); + if (!idle) + throw new WorkspaceToolError( + 'Native executor capacity reached', + 'COMMAND_UNAVAILABLE', + ); + await idle[1].sandbox.close(); + this.entries.delete(idle[0]); + } + entry = { + sandbox: this.createSandbox(options), + busy: false, + }; + } + entry.busy = true; + // Map insertion order is the idle eviction order. + this.entries.delete(root); + this.entries.set(root, entry); + return entry; + }); + const checked = pending.catch((error) => { + // Allocation/idle eviction precedes dispatch into the requested root. + // Do not turn a pool resource failure into an uncertain mutation there. + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError( + 'Native executor allocation failed', + 'COMMAND_UNAVAILABLE', + ); + }); + this.allocation = checked.catch(() => undefined); + return checked; + } + + async prepare(): Promise { + const entry = await this.allocate(this.roots.keys().next().value!); + try { + await entry.sandbox.prepare(); + } finally { + entry.busy = false; + } + } + + async execute( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + ): Promise { + const entry = await this.allocate(request.workspaceId); + let enteredExecutor = false; + try { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Command cancelled before dispatch', + 'EXECUTION_ABORTED', + ); + enteredExecutor = true; + return await entry.sandbox.execute(request, signal); + } catch (error) { + if ( + enteredExecutor && + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted + ) { + // Never retry the command here. Retire a failed executor only after + // close succeeds, allowing a later assignment to create a fresh child. + try { + await entry.sandbox.close(); + if (this.entries.get(request.workspaceId) === entry) + this.entries.delete(request.workspaceId); + } catch { + /* Retain ownership for subsequent cleanup/shutdown. */ + } + } + throw error; + } finally { + entry.busy = false; + } + } + + async close(): Promise { + this.closing = true; + await this.allocation; + const results = await Promise.allSettled( + [...this.entries.values()].map((entry) => entry.sandbox.close()), + ); + this.entries.clear(); + const errors = results + .filter((result) => result.status === 'rejected') + .map((result) => result.reason); + if (errors.length) + throw new AggregateError(errors, 'Native executor pool shutdown failed'); + } +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 37288d63..44cfdfd0 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -398,6 +398,8 @@ const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ ]); export interface BridgeWorkerCapabilities { + /** Opt-in protocol: maximum concurrently leased independent workspace roots. */ + workspaceLeaseSlots?: number; statefulWorkspace: boolean; sandboxProfile: string; runtimes: string[]; @@ -414,6 +416,8 @@ export interface BridgeWorkerRegistration { } export interface BridgeWorkerRegistrationResponse { + /** Absent on legacy servers. Workers must not parallelize without this receipt. */ + workspaceLeaseSlots?: number; protocolVersion: BridgeProtocolVersion; workerId: string; incarnationId: string; @@ -464,6 +468,7 @@ export interface BridgeSandboxRequest { } export interface BridgeAssignment { + workspaceLeaseSlot?: number; protocolVersion: BridgeProtocolVersion; assignmentId: string; workerId: string; @@ -1125,6 +1130,10 @@ export function isValidBridgeWorkerCapabilities( if (typeof value !== 'object' || value === null) return false; const capabilities = value as Record; return ( + (capabilities.workspaceLeaseSlots === undefined || + (Number.isSafeInteger(capabilities.workspaceLeaseSlots) && + Number(capabilities.workspaceLeaseSlots) >= 1 && + Number(capabilities.workspaceLeaseSlots) <= 8)) && typeof capabilities.statefulWorkspace === 'boolean' && typeof capabilities.sandboxProfile === 'string' && capabilities.sandboxProfile.trim().length > 0 && diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts new file mode 100644 index 00000000..ba65a3cf --- /dev/null +++ b/packages/code/src/worker-slots.test.ts @@ -0,0 +1,314 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { BridgeWorker } from './worker.js'; +import type { + BridgeAssignment, + BridgeWorkspaceToolCapabilities, +} from './protocol.js'; + +const capabilities: BridgeWorkspaceToolCapabilities = { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'a' }, { id: 'b' }], +}; +for (const requestedSlots of [1, 2]) { + test(`mapped quarantine blocks serial readiness with ${requestedSlots} requested slots`, async () => { + const paths: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId: 'incarnation-guard', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: requestedSlots, + requiresReadyConfirmation: true, + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + async execute() { + throw new Error('must not execute'); + }, + }, + workspaceQuarantines: new Map([ + [ + 'a', + { + async assertAvailable() { + throw new Error('retained guard'); + }, + async arm() {}, + async clear() {}, + async quarantine() {}, + }, + ], + [ + 'b', + { + async assertAvailable() {}, + async arm() {}, + async clear() {}, + async quarantine() {}, + }, + ], + ]), + fetchImpl: async (url) => { + paths.push(new URL(String(url)).pathname); + return Response.json({ + protocolVersion: 1, + workerId: 'worker', + incarnationId: 'incarnation-guard', + registrationGeneration: 1, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + workspaceLeaseSlots: 1, + }); + }, + }); + await assert.rejects(worker.register(), { code: 'WORKER_QUARANTINED' }); + assert.ok(paths.every((path) => path.endsWith('/register'))); + if (requestedSlots === 1) assert.equal(paths.length, 0); + }); +} +test('clean rejection receipt failure uses lane-local quarantine classification', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + maintainRegistration: () => Promise; + settleWithRetry: () => Promise; + reportWorkspaceOwnership: () => Promise; + rejectUnexecutedAssignment: ( + assignment: BridgeAssignment, + message: string, + ) => Promise; + }; + internals.maintainRegistration = async () => {}; + internals.settleWithRetry = async () => {}; + internals.reportWorkspaceOwnership = async () => { + throw new TypeError('receipt outage'); + }; + await assert.rejects( + internals.rejectUnexecutedAssignment( + { workspaceLeaseSlot: 0 } as BridgeAssignment, + 'expired', + ), + { name: 'BridgeWorkspaceQuarantinedError' }, + ); +}); +test('maintenance registration never advertises readiness or starts leasing', async () => { + const paths: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId: 'incarnation-maintenance', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + }, + fetchImpl: async (url, init) => { + const path = new URL(String(url)).pathname; + paths.push(path); + assert.equal( + JSON.parse(String(init?.body)).capabilities.requiresReadyConfirmation, + true, + ); + return Response.json({ + protocolVersion: 1, + workerId: 'worker', + incarnationId: 'incarnation-maintenance', + registrationGeneration: 1, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + }); + }, + }); + await worker.registerForMaintenance(); + assert.equal(paths.length, 1); + assert.ok(paths[0].endsWith('/register')); + await assert.rejects(worker.run(), /maintenance/i); +}); +for (const receipt of [undefined, 1]) { + test(`worker keeps serial lease wire format for receipt ${receipt}`, async () => { + const controller = new AbortController(); + let leases = 0; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId: 'incarnation-test-slots', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + async execute() { + throw new Error('must not execute'); + }, + }, + workspaceQuarantines: new Map( + ['a', 'b'].map((root) => [ + root, + { + async assertAvailable() {}, + async arm() {}, + async clear() {}, + async quarantine() {}, + }, + ]), + ), + fetchImpl: async (url, init) => { + const path = new URL(String(url)).pathname; + if (path.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId: 'worker', + incarnationId: 'incarnation-test-slots', + registrationGeneration: 1, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + ...(receipt === undefined ? {} : { workspaceLeaseSlots: receipt }), + }); + if (path.endsWith('/lease')) { + leases++; + assert.equal( + JSON.parse(String(init?.body)).workspaceLeaseSlot, + undefined, + ); + controller.abort(); + } + return Response.json({ protocolVersion: 1, ready: true }); + }, + }); + await worker.run(controller.signal); + assert.equal(leases, 1); + await assert.rejects(worker.lease(undefined, 0), /negotiated capacity/); + }); +} + +for (const cancelled of [false, true]) { + test(`local cleanup wait rejects unexecuted work on ${cancelled ? 'cancellation' : 'expiry'}`, async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + // Exercise the handoff seam without involving the unrelated HTTP settlement retry loop. + const internals = worker as unknown as { + activeWorkspaceAssignments: Map< + string, + { id: string; done: Promise } + >; + rejectUnexecutedAssignment: () => Promise; + executeOwned: () => Promise; + }; + internals.activeWorkspaceAssignments.set('a', { + id: 'previous', + done: new Promise(() => {}), + }); + let rejected = false; + internals.rejectUnexecutedAssignment = async () => { + rejected = true; + }; + internals.executeOwned = async () => { + assert.fail('must not enter a root still cleaning up'); + }; + const controller = new AbortController(); + if (cancelled) controller.abort(); + await worker.executeAndSettle( + { + assignmentId: 'next', + executionKind: 'workspace_tool', + remainingMs: cancelled ? 60000 : 5, + request: { + protocolVersion: 1, + workspaceId: 'a', + operation: 'read_file', + path: 'test.txt', + }, + } as BridgeAssignment, + controller.signal, + ); + assert.equal(rejected, true); + assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + }); +} + +test('a local cleanup handoff preserves the new assignment owner and remaining budget', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map< + string, + { id: string; done: Promise } + >; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + let release!: () => void; + internals.activeWorkspaceAssignments.set('a', { + id: 'previous', + done: new Promise((resolve) => { + release = resolve; + }), + }); + let executed = false; + internals.executeOwned = async (assignment) => { + executed = true; + assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'next'); + assert.ok(assignment.remainingMs! < 1000 && assignment.remainingMs! > 0); + }; + const pending = worker.executeAndSettle({ + assignmentId: 'next', + executionKind: 'workspace_tool', + remainingMs: 1000, + request: { + protocolVersion: 1, + workspaceId: 'a', + operation: 'read_file', + path: 'test.txt', + }, + } as BridgeAssignment); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(executed, false); + internals.activeWorkspaceAssignments.delete('a'); + release(); + await pending; + assert.equal(executed, true); + assert.equal(internals.activeWorkspaceAssignments.size, 0); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 1fda70f7..545b96ca 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -35,10 +35,13 @@ export interface BridgeWorkerOptions { capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; workspaceMutationQuarantine?: WorkspaceMutationQuarantine; + /** Required per-root durable guards when opting into concurrent workspace leases. */ + workspaceQuarantines?: ReadonlyMap; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; leaseAckTransportTimeoutMs?: number; + workspaceCleanupTimeoutMs?: number; resetTransportTimeoutMs?: number; cancellationPollIntervalMs?: number; cancellationTransportTimeoutMs?: number; @@ -59,9 +62,13 @@ export interface BridgeWorkerOptions { export interface WorkspaceMutationQuarantine { assertAvailable(): Promise; - arm(reason: string): Promise; - clear(): Promise; - quarantine(reason: string, cause?: unknown): Promise; + arm(reason: string, assignmentId?: string): Promise; + clear(assignmentId?: string): Promise; + quarantine( + reason: string, + cause?: unknown, + assignmentId?: string, + ): Promise; } export interface BridgeWorkerIdentity { @@ -343,10 +350,32 @@ export class BridgeWorker { private activeCapabilities: BridgeWorkerCapabilities; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; + private maintenanceOnly = false; private mutationGuardArmed = false; + private readonly quarantinedWorkspaces = new Set(); + private readonly activeWorkspaceAssignments = new Map< + string, + { id: string; done: Promise } + >(); + private readonly armedWorkspaces = new Set(); + private negotiatedWorkspaceSlots = 1; + private concurrentRunning = false; + private registrationInFlight?: Promise; + private credentialInFlight?: Promise; private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; constructor(private readonly options: BridgeWorkerOptions) { + const requestedSlots = options.capabilities.workspaceLeaseSlots; + if ( + requestedSlots !== undefined && + (!Number.isSafeInteger(requestedSlots) || + requestedSlots < 1 || + requestedSlots > 8) + ) { + throw new BridgeProtocolError( + 'Workspace lease slots must be an integer from 1 to 8', + ); + } if (!options.token && !options.identity) { throw new BridgeProtocolError( 'Bridge worker requires a static token or paired identity', @@ -358,7 +387,9 @@ export class BridgeWorker { ); } if (options.runtimeSupervisor == null && !options.sandboxEndpoint?.trim()) { - throw new BridgeProtocolError('Bridge worker requires a runtime supervisor'); + throw new BridgeProtocolError( + 'Bridge worker requires a runtime supervisor', + ); } if ( (options.workspaceTools == null) !== @@ -381,12 +412,26 @@ export class BridgeWorker { operation === 'edit_file' || operation === 'execute_command', ) === true && - options.workspaceMutationQuarantine == null + options.workspaceMutationQuarantine == null && + options.workspaceQuarantines == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', ); } + if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { + if ( + options.capabilities.requiresReadyConfirmation !== true || + options.workspaceQuarantines == null || + options.capabilities.workspaceTools?.workspaces.some( + (root) => !options.workspaceQuarantines!.has(root.id), + ) !== false + ) { + throw new BridgeProtocolError( + 'Concurrent workspaces require per-root durable guards and readiness confirmation', + ); + } + } this.fetchImpl = options.fetchImpl ?? fetch; this.codeApiUrl = normalizedBaseUrl(options.codeApiUrl); this.runtimeSupervisor = @@ -410,13 +455,50 @@ export class BridgeWorker { return await this.registerWithPolicy(signal, false); } + async registerForMaintenance( + signal?: AbortSignal, + ): Promise { + if ( + this.lastRegisteredAtMs !== 0 || + this.registrationInFlight != null || + this.concurrentRunning + ) { + throw new BridgeProtocolError( + 'Maintenance registration requires a fresh worker', + ); + } + this.maintenanceOnly = true; + return this.register(signal); + } + private async registerWithPolicy( signal: AbortSignal | undefined, allowActiveMutation: boolean, + ): Promise { + if (this.registrationInFlight) return await this.registrationInFlight; + const pending = this.registerOwned(signal, allowActiveMutation); + this.registrationInFlight = pending; + try { + return await pending; + } finally { + this.registrationInFlight = undefined; + } + } + + private async registerOwned( + signal: AbortSignal | undefined, + allowActiveMutation: boolean, ): Promise { if (!allowActiveMutation) { try { await this.options.workspaceMutationQuarantine?.assertAvailable(); + if ( + !this.maintenanceOnly && + (this.options.capabilities.workspaceLeaseSlots ?? 1) === 1 + ) { + for (const guard of this.options.workspaceQuarantines?.values() ?? []) + await guard.assertAvailable(); + } } catch (error) { if ( error instanceof BridgeProtocolError && @@ -457,7 +539,9 @@ export class BridgeWorker { protocolVersion: BRIDGE_PROTOCOL_VERSION, workerId: this.options.workerId, incarnationId: this.incarnationId, - capabilities, + capabilities: this.maintenanceOnly + ? { ...capabilities, requiresReadyConfirmation: true } + : capabilities, }, registrationController.signal, ); @@ -503,6 +587,38 @@ export class BridgeWorker { 'Code API registered a different worker incarnation', ); } + const slots = registration.workspaceLeaseSlots ?? 1; + if ( + !Number.isSafeInteger(slots) || + slots < 1 || + slots > (this.options.capabilities.workspaceLeaseSlots ?? 1) || + slots > 8 || + (this.concurrentRunning && slots !== this.negotiatedWorkspaceSlots) + ) { + throw new BridgeProtocolError( + 'Code API workspace slot negotiation changed or exceeded local policy', + undefined, + 'WORKER_FENCED', + ); + } + this.negotiatedWorkspaceSlots = slots; + if ( + !this.maintenanceOnly && + !allowActiveMutation && + slots === 1 && + (this.options.capabilities.workspaceLeaseSlots ?? 1) > 1 + ) { + try { + for (const guard of this.options.workspaceQuarantines?.values() ?? []) + await guard.assertAvailable(); + } catch { + throw new BridgeProtocolError( + 'Serial workspace quarantine state could not be verified', + undefined, + 'WORKER_QUARANTINED', + ); + } + } const registeredAtMs = Date.parse(registration.registeredAt); if (Number.isFinite(registeredAtMs)) { this.serverClockOffsetMs = registeredAtMs - registrationStartedAtMs; @@ -510,7 +626,10 @@ export class BridgeWorker { this.registrationTtlMs = registration.leaseTtlMs; this.activeCapabilities = this.registrationCapabilities; await this.options.onRegistered?.(registration); - if (this.options.capabilities.requiresReadyConfirmation === true) { + if ( + !this.maintenanceOnly && + this.options.capabilities.requiresReadyConfirmation === true + ) { await this.confirmReady(registration, signal); } this.lastRegisteredAtMs = registrationStartedAtMs; @@ -571,7 +690,55 @@ export class BridgeWorker { ); } - async lease(signal?: AbortSignal): Promise { + async resetNativeWorkspace( + workspaceId: string, + signal?: AbortSignal, + ): Promise { + const guard = this.options.workspaceQuarantines?.get(workspaceId); + if ( + !guard || + this.activeWorkspaceAssignments.size > 0 || + !this.options.capabilities.workspaceTools?.workspaces.some( + (root) => root.id === workspaceId, + ) + ) { + throw new BridgeProtocolError( + 'Native workspace reset requires an idle registered root', + ); + } + // The operator must have inspected/restored the root and cleared its + // machine-local guard before the remote fence can be removed. + await guard.assertAvailable(); + await this.timedRequest( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId: this.incarnationId, + runtimeSessionId: `native-workspace:${workspaceId}`, + confirmDiscarded: true, + }, + this.options.resetTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, + signal, + ); + this.quarantinedWorkspaces.delete(workspaceId); + } + + async lease( + signal?: AbortSignal, + workspaceLeaseSlot?: number, + ): Promise { + if ( + workspaceLeaseSlot !== undefined && + (!Number.isSafeInteger(workspaceLeaseSlot) || + workspaceLeaseSlot < 0 || + this.negotiatedWorkspaceSlots <= 1 || + workspaceLeaseSlot >= this.negotiatedWorkspaceSlots) + ) { + throw new BridgeProtocolError( + 'Workspace lease slot exceeds negotiated capacity', + ); + } const waitMs = Math.min( MAX_LEASE_WAIT_MS, Math.max(0, this.options.leaseWaitMs ?? DEFAULT_LEASE_WAIT_MS), @@ -601,6 +768,7 @@ export class BridgeWorker { protocolVersion: BRIDGE_PROTOCOL_VERSION, waitMs, incarnationId: this.incarnationId, + ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot }), }, leaseController.signal, ); @@ -610,7 +778,8 @@ export class BridgeWorker { } if ( response.assignment != null && - response.assignment.incarnationId !== this.incarnationId + (response.assignment.incarnationId !== this.incarnationId || + response.assignment.workspaceLeaseSlot !== workspaceLeaseSlot) ) { throw new BridgeProtocolError( 'Code API leased an assignment for a different worker incarnation', @@ -698,11 +867,19 @@ export class BridgeWorker { } async run(signal?: AbortSignal): Promise { + if (this.maintenanceOnly) + throw new BridgeProtocolError( + 'Maintenance workers cannot execute assignments', + ); let reconnectAttempt = 0; while (!signal?.aborted) { try { await this.refreshCredential(signal); await this.register(signal); + if (this.negotiatedWorkspaceSlots > 1) { + await this.runConcurrent(signal); + return; + } const assignment = await this.lease(signal); reconnectAttempt = 0; if (!assignment) continue; @@ -734,13 +911,132 @@ export class BridgeWorker { } } + private async runConcurrent(signal?: AbortSignal): Promise { + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + let failure: unknown; + const fail = (error: unknown): void => { + failure ??= error; + controller.abort(error); + }; + this.concurrentRunning = true; + const heartbeat = this.maintainRegistration(controller.signal).catch(fail); + const lane = async (slot?: number): Promise => { + let retries = 0; + while (!controller.signal.aborted) { + let assignment: BridgeAssignment | undefined; + try { + assignment = await this.lease(controller.signal, slot); + retries = 0; + if (assignment == null) continue; + await this.executeAndSettle(assignment, controller.signal); + } catch (error) { + if ( + error instanceof BridgeWorkspaceQuarantinedError && + assignment?.workspaceLeaseSlot !== undefined + ) { + // The durable local guard is retained. A distinct receipt tells + // Code API to release this slot without declaring the root clean. + try { + await this.reportWorkspaceOwnership( + assignment, + 'quarantine', + controller.signal, + ); + this.options.onError?.(error); + continue; + } catch (quarantineError) { + // The durable server fence remains until explicit cleanup/reset. + // An unavailable control receipt must not cancel healthy roots. + if ( + quarantineError instanceof BridgeProtocolError && + (quarantineError.status === 401 || + quarantineError.status === 403 || + quarantineError.code === 'WORKER_FENCED') + ) { + fail(quarantineError); + return; + } + this.options.onError?.(quarantineError); + // Keep every advertised slot polled. The root remains fenced, + // but a committed receipt may already have released this slot. + continue; + } + } + if (controller.signal.aborted) return; + if ( + assignment != null || + (error instanceof BridgeProtocolError && + (error.status === 401 || + error.status === 403 || + error.code === 'WORKER_FENCED' || + error.code === 'WORKER_QUARANTINED')) + ) { + fail(error); + return; + } + this.options.onError?.(error); + await abortableDelay( + reconnectDelayMs( + retries++, + this.options.reconnectDelayMs, + this.options.reconnectMaxDelayMs, + this.options.reconnectRandom, + ), + controller.signal, + ); + } + } + }; + try { + // The legacy lane serves run-code requests only when the aggregate lock + // excludes workspace slots. It never increases simultaneous executions. + await Promise.all([ + lane(), + ...Array.from({ length: this.negotiatedWorkspaceSlots }, (_, i) => + lane(i), + ), + ]); + } finally { + controller.abort(); + await heartbeat; + this.concurrentRunning = false; + signal?.removeEventListener('abort', abort); + } + if (failure != null) throw failure; + } + async refreshCredential( signal?: AbortSignal, - validThroughMs = - Date.now() + + validThroughMs = Date.now() + this.serverClockOffsetMs + (this.options.credentialRefreshWindowMs ?? CREDENTIAL_REFRESH_WINDOW_MS), transportTimeoutMs = Number.POSITIVE_INFINITY, + ): Promise { + while (this.credentialInFlight) { + await this.credentialInFlight; + // A longer-lived caller may still need another refresh after this one. + } + const pending = this.refreshCredentialOwned( + signal, + validThroughMs, + transportTimeoutMs, + ); + this.credentialInFlight = pending; + try { + await pending; + } finally { + if (this.credentialInFlight === pending) + this.credentialInFlight = undefined; + } + } + + private async refreshCredentialOwned( + signal: AbortSignal | undefined, + validThroughMs: number, + transportTimeoutMs: number, ): Promise { const identity = this.options.identity; if (identity == null) return; @@ -833,6 +1129,101 @@ export class BridgeWorker { assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { + const root = + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ? assignment.request.workspaceId + : undefined; + const waitingAt = Date.now(); + while (root != null && this.activeWorkspaceAssignments.has(root)) { + const active = this.activeWorkspaceAssignments.get(root)!; + if (active.id === assignment.assignmentId) + throw new BridgeProtocolError( + 'Code API replayed an active workspace assignment', + undefined, + 'WORKER_FENCED', + ); + // A settlement can commit remotely before the local durable guard clears. + // Keep the next lane out of the root until that cleanup has finished. + const waitController = new AbortController(); + const onAbort = (): void => waitController.abort(); + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) waitController.abort(); + let finished: boolean; + try { + finished = await Promise.race([ + active.done.then(() => true), + abortableDelay( + Math.max( + 0, + this.assignmentRemainingMs(assignment) - (Date.now() - waitingAt), + ), + waitController.signal, + ).then(() => false), + ]); + } finally { + waitController.abort(); + signal?.removeEventListener('abort', onAbort); + } + if (!finished || signal?.aborted) { + await this.rejectUnexecutedAssignment( + assignment, + 'Workspace cleanup wait ended before execution', + ); + return; + } + } + let release!: () => void; + if (root != null) + this.activeWorkspaceAssignments.set(root, { + id: assignment.assignmentId, + done: new Promise((resolve) => { + release = resolve; + }), + }); + const adjusted = + assignment.remainingMs === undefined + ? assignment + : { + ...assignment, + remainingMs: Math.max( + 0, + assignment.remainingMs - (Date.now() - waitingAt), + ), + }; + try { + await this.executeOwned(adjusted, signal); + } catch (error) { + if (root != null && error instanceof BridgeWorkspaceQuarantinedError) { + // A failed unlink/fsync may have removed the durable marker already. + // Fence locally before releasing the handoff to the next root assignment. + this.quarantinedWorkspaces.add(root); + } + throw error; + } finally { + if (root != null) { + this.activeWorkspaceAssignments.delete(root); + release(); + } + } + } + + private workspaceGuard( + assignment: BridgeAssignment, + ): WorkspaceMutationQuarantine | undefined { + return assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ? (this.options.workspaceQuarantines?.get( + assignment.request.workspaceId, + ) ?? this.options.workspaceMutationQuarantine) + : this.options.workspaceMutationQuarantine; + } + + private async executeOwned( + assignment: BridgeAssignment, + signal?: AbortSignal, + ): Promise { + const guard = this.workspaceGuard(assignment); if (signal?.aborted === true) { throw signal.reason instanceof Error ? signal.reason @@ -897,8 +1288,10 @@ export class BridgeWorker { } const heartbeatController = new AbortController(); let heartbeatError: unknown; - const heartbeat = this.maintainRegistration( - heartbeatController.signal, + const heartbeat = ( + this.concurrentRunning + ? Promise.resolve() + : this.maintainRegistration(heartbeatController.signal) ).catch((error) => { heartbeatError = error; executionController.abort(); @@ -914,7 +1307,9 @@ export class BridgeWorker { let settlement: BridgeSettlement; let ambiguousSandboxError: unknown; let ambiguousWorkspaceMutationError: unknown; - let workspaceMutationGuardError: BridgeWorkspaceQuarantinedError | undefined; + let workspaceMutationGuardError: + | BridgeWorkspaceQuarantinedError + | undefined; let sandboxRejectedExecution = false; let sandboxStarted = false; let workspaceMutationArmed = false; @@ -941,6 +1336,18 @@ export class BridgeWorker { throw new BridgeProtocolError('Invalid workspace tool request'); } const workspaceRequest = assignment.request; + try { + if (this.quarantinedWorkspaces.has(workspaceRequest.workspaceId)) { + throw new Error('Workspace requires an explicit quarantine reset'); + } + if (this.options.workspaceQuarantines != null) + await guard?.assertAvailable(); + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Workspace is quarantined', + error, + ); + } const advertised = this.activeCapabilities.workspaceTools; if (advertised == null) { throw new BridgeProtocolError( @@ -983,7 +1390,8 @@ export class BridgeWorker { workspaceRequest.operation === 'preview_edit' || workspaceRequest.operation === 'edit_file' ) { - const mode = workspaceRequest.edits === undefined ? 'single' : 'batch'; + const mode = + workspaceRequest.edits === undefined ? 'single' : 'batch'; const modes = advertised.editFileModes; if ( (modes == null && mode !== 'single') || @@ -1019,8 +1427,10 @@ export class BridgeWorker { if (isMutation) { this.mutationGuardArmed = true; try { - await this.options.workspaceMutationQuarantine!.arm( + this.armedWorkspaces.add(workspaceRequest.workspaceId); + await guard!.arm( `Workspace mutation ${workspaceRequest.operation} is pending settlement`, + assignment.assignmentId, ); workspaceMutationArmed = true; } catch (error) { @@ -1040,10 +1450,8 @@ export class BridgeWorker { !advertised.listFileFeatures?.includes('after_path') && 'nextAfterPath' in payload ) { - const { - nextAfterPath: _nextAfterPath, - ...compatiblePayload - } = payload; + const { nextAfterPath: _nextAfterPath, ...compatiblePayload } = + payload; payload = compatiblePayload; } workspaceMutationApplied = isMutation; @@ -1176,11 +1584,10 @@ export class BridgeWorker { error instanceof WorkspaceToolError ? { errorCode: error.code } : {}), - error: - (error instanceof Error - ? error.message - : 'Sandbox execution failed' - ).slice(0, MAX_SETTLEMENT_ERROR_LENGTH), + error: (error instanceof Error + ? error.message + : 'Sandbox execution failed' + ).slice(0, MAX_SETTLEMENT_ERROR_LENGTH), }; } @@ -1190,12 +1597,14 @@ export class BridgeWorker { credentialController.abort(); await credentialMaintenance; try { - if (workspaceMutationGuardError != null) throw workspaceMutationGuardError; + if (workspaceMutationGuardError != null) + throw workspaceMutationGuardError; if (ambiguousWorkspaceMutationError != null) { throw await this.quarantineWorkspace( undefined, 'Worker stopped after a workspace mutation completed without a fulfilled settlement', ambiguousWorkspaceMutationError, + assignment, ); } if (ambiguousSandboxError != null) { @@ -1203,6 +1612,7 @@ export class BridgeWorker { assignment.runtimeSessionId, `Stateful workspace ${assignment.runtimeSessionId} was quarantined after an ambiguous sandbox execution`, ambiguousSandboxError, + assignment, ); } const knownCleanStatefulRejection = @@ -1242,7 +1652,37 @@ export class BridgeWorker { } if (workspaceMutationArmed) { try { - await this.options.workspaceMutationQuarantine!.clear(); + if (assignment.workspaceLeaseSlot === undefined) { + await guard!.clear(assignment.assignmentId); + } else { + let timer!: ReturnType; + try { + await Promise.race([ + guard!.clear(assignment.assignmentId), + new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject(new Error('Workspace guard cleanup timed out')), + Math.min( + 5000, + Math.max( + 1, + this.options.workspaceCleanupTimeoutMs ?? 5000, + ), + ), + ); + }), + ]); + } finally { + clearTimeout(timer); + } + } + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + this.armedWorkspaces.delete(assignment.request.workspaceId); + } this.mutationGuardArmed = false; } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1251,6 +1691,20 @@ export class BridgeWorker { ); } } + if (assignment.workspaceLeaseSlot !== undefined) { + try { + await this.reportWorkspaceOwnership( + assignment, + 'workspace-cleanup', + signal, + ); + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Workspace cleanup acknowledgement could not be confirmed', + error, + ); + } + } } finally { heartbeatController.abort(); try { @@ -1262,6 +1716,53 @@ export class BridgeWorker { } } + private async reportWorkspaceOwnership( + assignment: BridgeAssignment, + operation: 'quarantine' | 'workspace-cleanup', + signal?: AbortSignal, + ): Promise { + let failure: unknown; + for (let attempt = 0; attempt < 3 && !signal?.aborted; attempt++) { + try { + await this.timedRequest( + this.assignmentUrl(assignment, operation), + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId: this.incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: + operation === 'quarantine' + ? 'Workspace requires inspection before reset.' + : 'Local workspace cleanup confirmed.', + }, + this.options.leaseAckTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, + signal, + ); + return; + } catch (error) { + failure = error; + if ( + error instanceof BridgeProtocolError && + error.status != null && + error.status >= 400 && + error.status < 500 && + error.status !== 408 && + error.status !== 429 + ) + throw error; + await abortableDelay(100 * (attempt + 1), signal); + } + } + throw ( + failure ?? + signal?.reason ?? + new Error('Workspace ownership reporting aborted') + ); + } + private assignmentRemainingMs(assignment: BridgeAssignment): number { if ( Number.isSafeInteger(assignment.remainingMs) && @@ -1307,6 +1808,7 @@ export class BridgeWorker { assignment.runtimeSessionId, `Stateful workspace ${assignment.runtimeSessionId} could not release its runtime lease`, error, + assignment, ); } } @@ -1315,13 +1817,15 @@ export class BridgeWorker { runtimeSessionId: string | undefined, message: string, cause?: unknown, + assignment?: BridgeAssignment, ): Promise { if (runtimeSessionId == null) { try { - await this.options.workspaceMutationQuarantine?.quarantine( - message, - cause, - ); + await ( + assignment == null + ? this.options.workspaceMutationQuarantine + : this.workspaceGuard(assignment) + )?.quarantine(message, cause, assignment?.assignmentId); return new BridgeWorkspaceQuarantinedError(message, cause); } catch (error) { return new BridgeWorkspaceQuarantinedError( @@ -1355,15 +1859,16 @@ export class BridgeWorker { Math.floor(this.registrationTtlMs / 2), ); await this.delay( - Math.max( - 0, - this.lastRegisteredAtMs + heartbeatIntervalMs - Date.now(), - ), + Math.max(0, this.lastRegisteredAtMs + heartbeatIntervalMs - Date.now()), signal, ); if (signal.aborted) return; try { - await this.registerWithPolicy(signal, this.mutationGuardArmed); + if (this.concurrentRunning) await this.refreshCredential(signal); + await this.registerWithPolicy( + signal, + this.mutationGuardArmed || this.armedWorkspaces.size > 0, + ); } catch (error) { const terminal = error instanceof BridgeProtocolError && @@ -1403,6 +1908,16 @@ export class BridgeWorker { this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, ), ); + if (assignment.workspaceLeaseSlot !== undefined) { + try { + await this.reportWorkspaceOwnership(assignment, 'workspace-cleanup'); + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Unexecuted workspace cleanup acknowledgement could not be confirmed', + error, + ); + } + } } finally { heartbeatController.abort(); await heartbeat; @@ -1439,6 +1954,7 @@ export class BridgeWorker { ? `Stateful workspace ${assignment.runtimeSessionId} was quarantined before settlement during shutdown` : 'Worker stopped after a workspace mutation could not be settled during shutdown', signal.reason, + assignment, ); } throw signal.reason instanceof Error @@ -1483,6 +1999,7 @@ export class BridgeWorker { ? `Stateful workspace ${assignment.runtimeSessionId} was quarantined after Code API rejected its fulfilled settlement` : 'Worker stopped after Code API rejected a fulfilled workspace mutation settlement', error, + assignment, ); } throw error; @@ -1509,6 +2026,7 @@ export class BridgeWorker { ? `Stateful workspace ${assignment.runtimeSessionId} was quarantined after ambiguous settlement delivery` : 'Worker stopped after ambiguous workspace mutation settlement delivery', lastError, + assignment, ); } if (lastError instanceof Error) throw lastError; diff --git a/packages/code/src/workspace-guards.ts b/packages/code/src/workspace-guards.ts new file mode 100644 index 00000000..98b6bb02 --- /dev/null +++ b/packages/code/src/workspace-guards.ts @@ -0,0 +1,49 @@ +import { BridgeProtocolError } from './protocol.js'; +import { + assertWorkspaceMutationQuarantineOwner, + clearWorkspaceMutationQuarantine, + loadWorkspaceMutationQuarantine, + saveWorkspaceMutationQuarantine, +} from './storage.js'; +import type { WorkspaceMutationQuarantine } from './worker.js'; + +/** Each path is outside sandbox roots and each pending write has a unique owner. */ +export function workspaceMutationGuard( + path: string, + workerId: string, + workspaceId: string, + incarnationId: string, +): WorkspaceMutationQuarantine { + const owner = (assignmentId?: string): string => { + if (!assignmentId) + throw new Error('Workspace mutation requires an assignment owner'); + return `${incarnationId}:${assignmentId}`; + }; + return { + async assertAvailable() { + const record = await loadWorkspaceMutationQuarantine(path); + if (record != null) + throw new BridgeProtocolError( + `Workspace ${workspaceId} is quarantined; inspect it and clear its quarantine before restarting the worker`, + undefined, + 'WORKSPACE_QUARANTINED', + ); + }, + async arm(reason, assignmentId) { + await saveWorkspaceMutationQuarantine(path, { + version: 1, + workerId, + workspaceId, + ownerId: owner(assignmentId), + quarantinedAt: new Date().toISOString(), + reason, + }); + }, + async clear(assignmentId) { + await clearWorkspaceMutationQuarantine(path, owner(assignmentId)); + }, + async quarantine(_reason, _cause, assignmentId) { + await assertWorkspaceMutationQuarantineOwner(path, owner(assignmentId)); + }, + }; +} diff --git a/service/src/bridge/admission.ts b/service/src/bridge/admission.ts index 1e19286e..4002ed6d 100644 --- a/service/src/bridge/admission.ts +++ b/service/src/bridge/admission.ts @@ -7,15 +7,21 @@ export class BridgeAdmissionQueue { private readonly capacity = 32, ) {} - private keys(workerId: string): [string, string, string] { + private keys(workerId: string): [string, string, string, string] { const prefix = `codeapi:bridge:v1:worker:${encodeURIComponent(workerId)}:admission`; - return [prefix, `${prefix}:deadlines`, `${prefix}:sequence`]; + return [ + prefix, + `${prefix}:deadlines`, + `${prefix}:sequence`, + `${prefix}:workspaces`, + ]; } async enter( workerId: string, id: string, deadlineAtMs: number, + workspaceId?: string, ): Promise { return ( Number( @@ -25,31 +31,34 @@ export class BridgeAdmissionQueue { 'for _, id in ipairs(expired) do', " redis.call('ZREM', KEYS[1], id)", " redis.call('ZREM', KEYS[2], id)", + " redis.call('HDEL', KEYS[4], id)", 'end', "if redis.call('ZSCORE', KEYS[1], ARGV[1]) then return 1 end", "if redis.call('ZCARD', KEYS[1]) >= tonumber(ARGV[4]) then return 0 end", "local sequence = redis.call('INCR', KEYS[3])", "redis.call('ZADD', KEYS[1], sequence, ARGV[1])", "redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])", + "if ARGV[5] ~= '' then redis.call('HSET', KEYS[4], ARGV[1], ARGV[5]) end", "local latest = redis.call('ZREVRANGE', KEYS[2], 0, 0, 'WITHSCORES')", 'for _, key in ipairs(KEYS) do', " redis.call('PEXPIREAT', key, tonumber(latest[2]) + 30000)", 'end', 'return 1', ].join('\n'), - 3, + 4, ...this.keys(workerId), id, Date.now(), deadlineAtMs, this.capacity, + workspaceId ?? '', ), ) === 1 ); } async isHead(workerId: string, id: string): Promise { - const [order, deadlines] = this.keys(workerId); + const [order, deadlines, , workspaces] = this.keys(workerId); return ( Number( await this.redis.eval( @@ -58,14 +67,16 @@ export class BridgeAdmissionQueue { 'for _, id in ipairs(expired) do', " redis.call('ZREM', KEYS[1], id)", " redis.call('ZREM', KEYS[2], id)", + " redis.call('HDEL', KEYS[3], id)", 'end', "local head = redis.call('ZRANGE', KEYS[1], 0, 0)", 'if head[1] == ARGV[1] then return 1 end', 'return 0', ].join('\n'), - 2, + 3, order, deadlines, + workspaces, id, Date.now(), ), @@ -74,16 +85,18 @@ export class BridgeAdmissionQueue { } async leave(workerId: string, id: string): Promise { - const [order, deadlines] = this.keys(workerId); + const [order, deadlines, , workspaces] = this.keys(workerId); await this.redis.eval( [ "redis.call('ZREM', KEYS[1], ARGV[1])", "redis.call('ZREM', KEYS[2], ARGV[1])", + "redis.call('HDEL', KEYS[3], ARGV[1])", 'return 1', ].join('\n'), - 2, + 3, order, deadlines, + workspaces, id, ); } diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts new file mode 100644 index 00000000..fe53c44f --- /dev/null +++ b/service/src/bridge/concurrent-store.test.ts @@ -0,0 +1,533 @@ +import { afterEach, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { RedisBridgeStore } from './store'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import type { CodeBridgeAssignment } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const store = new RedisBridgeStore(redis, 60, 1000, 2); +const workerId = 'concurrent-worker'; +const incarnationId = 'concurrent-incarnation'; +afterEach(async () => { + await redis.flushall(); +}); +async function register(workspaceLeaseSlots = 2) { + const generation = await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + runtimes: [], + sandboxProfile: 'native-srt', + requiresReadyConfirmation: true, + workspaceLeaseSlots, + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file'], + workspaces: [{ id: 'a' }, { id: 'b' }], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); +} +function dispatch(workspaceId: string, signal = new AbortController().signal) { + const promise = store.dispatchWorkspaceTool({ + workerId, + signal, + deadlineAtMs: Date.now() + 3000, + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId, + path: 'file.txt', + }, + }); + void promise.catch(() => undefined); + return promise; +} +async function settle(assignment: CodeBridgeAssignment, cleanup = true) { + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'fixture clean rejection', + }); + if (cleanup) + await store.confirmWorkspaceCleanup(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'local cleanup confirmed', + }); +} +test('committed results retain the root fence until cleanup, including receipt expiry', async () => { + await register(); + const pending = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + const intent = { + protocolVersion: 1 as const, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected' as const, + error: 'fixture', + }; + await store.settle(workerId, assignment.assignmentId, intent); + await pending; + const fenceKey = ( + await redis.keys( + `codeapi:bridge:v1:worker:${workerId}:workspace:*:quarantined`, + ) + )[0]; + expect(await redis.get(fenceKey)).toBe(assignment.assignmentId); + await redis.del( + `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`, + ); + await expect( + store.confirmWorkspaceCleanup(workerId, assignment.assignmentId, intent), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + expect(await redis.get(fenceKey)).toBe(assignment.assignmentId); + expect(await redis.ttl(fenceKey)).toBe(-1); + const healthy = dispatch('b'); + const next = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ))!; + await settle(next); + await healthy; +}); +test('late quarantine after confirmed cleanup cannot fence a newer assignment', async () => { + await register(); + const first = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await settle(assignment); + await first; + const second = dispatch('a'); + const next = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await store.settle( + workerId, + assignment.assignmentId, + { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'lost cleanup response', + }, + undefined, + undefined, + true, + ); + await settle(next); + await expect(second).resolves.toMatchObject({ status: 'rejected' }); +}); +test('duplicate rejected settlement retries finalization after the dispatcher leaves', async () => { + await register(); + const controller = new AbortController(); + const pending = dispatch('a', controller.signal); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + controller.abort(); + await pending.catch(() => undefined); + const intent = { + protocolVersion: 1 as const, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected' as const, + error: 'clean rejection', + }; + const originalEval = redis.eval.bind(redis); + let failed = false; + redis.eval = ((script: string, ...args: unknown[]) => { + if (!failed && script.includes("'resultCommitted', '1'")) { + failed = true; + return Promise.reject(new Error('injected finalization outage')); + } + return (originalEval as (...args: unknown[]) => unknown)(script, ...args); + }) as typeof redis.eval; + try { + await expect( + store.settle(workerId, assignment.assignmentId, intent), + ).rejects.toThrow('injected finalization outage'); + } finally { + redis.eval = originalEval; + } + expect(failed).toBe(true); + await store.settle(workerId, assignment.assignmentId, intent); + await store.confirmWorkspaceCleanup( + workerId, + assignment.assignmentId, + intent, + ); + const next = dispatch('a'); + await settle( + (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!, + ); + await expect(next).resolves.toMatchObject({ status: 'rejected' }); +}); +test('store routes simultaneous roots through separate acknowledged slots', async () => { + await register(); + const a = dispatch('a'); + const b = dispatch('b'); + const first = await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + const second = await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ); + expect(first?.workspaceLeaseSlot).toBe(0); + expect(second?.workspaceLeaseSlot).toBe(1); + expect(first?.assignmentId).not.toBe(second?.assignmentId); + await settle(first!); + await settle(second!); + await expect(a).resolves.toMatchObject({ status: 'rejected' }); + await expect(b).resolves.toMatchObject({ status: 'rejected' }); + expect( + await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`), + ).toBeNull(); +}); + +test('a replica never dispatches above its own configured ceiling', async () => { + await register(); + const serialReplica = new RedisBridgeStore(redis, 60, 1000, 1); + await expect( + serialReplica.dispatchWorkspaceTool({ + workerId, + signal: new AbortController().signal, + deadlineAtMs: Date.now() + 1000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'a', + path: 'file.txt', + }, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + await expect( + serialReplica.lease(workerId, incarnationId, 0, undefined, undefined, 1), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect( + await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`), + ).toBeNull(); +}); + +test('capacity changes require the active slots to drain', async () => { + await register(); + const pending = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await expect(register(1)).rejects.toMatchObject({ code: 'WORKER_BUSY' }); + await settle(assignment); + await pending; + await register(1); + expect( + (await store.workerStatus(workerId)).capabilities?.workspaceLeaseSlots, + ).toBe(1); +}); + +test('same-root work waits while another root progresses', async () => { + await register(); + const a = dispatch('a'); + const first = await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + const nextA = dispatch('a'); + const b = dispatch('b'); + const second = await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ); + expect(second?.request).toMatchObject({ workspaceId: 'b' }); + await settle(first!); + await a; + const third = await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(third?.request).toMatchObject({ workspaceId: 'a' }); + await settle(second!); + await settle(third!); + await Promise.all([nextA, b]); +}); + +test('queued cancellation never leases and does not block another root', async () => { + await register(); + const a = dispatch('a'); + const first = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + const controller = new AbortController(); + const cancelled = dispatch('a', controller.signal); + const queue = `codeapi:bridge:v1:worker:${workerId}:admission`; + for (let i = 0; i < 100 && (await redis.zcard(queue)) < 2; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(await redis.zcard(queue)).toBe(2); + controller.abort(); + await expect(cancelled).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + const b = dispatch('b'); + const second = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ))!; + expect(second.request).toMatchObject({ workspaceId: 'b' }); + await settle(first); + await settle(second); + await Promise.all([a, b]); + expect( + await store.lease(workerId, incarnationId, 0, undefined, undefined, 0), + ).toBeUndefined(); +}); + +test('late quarantine releases its slot after caller cancellation and retains only its root fence', async () => { + await register(); + const controller = new AbortController(); + const a = dispatch('a', controller.signal); + const first = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await store.acknowledgeLease( + workerId, + incarnationId, + first.assignmentId, + first.generation, + first.leaseToken, + ); + controller.abort(); + await expect(a).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + await store.settle( + workerId, + first.assignmentId, + { + protocolVersion: 1, + incarnationId, + generation: first.generation, + leaseToken: first.leaseToken, + status: 'rejected', + error: 'uncertain mutation', + }, + undefined, + undefined, + true, + ); + expect( + await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`), + ).toBeNull(); + const b = dispatch('b'); + const next = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(next.request).toMatchObject({ workspaceId: 'b' }); + await settle(next); + await b; + await expect(dispatch('a')).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); +}); + +test('post-settlement fences are authenticated, idempotent, and invalidated by reset', async () => { + await register(); + const pending = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await settle(assignment, false); + await pending; // Result is committed, but local cleanup remains outstanding. + const receiptKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + expect(await redis.ttl(receiptKey)).toBeGreaterThan(0); + expect( + JSON.parse((await redis.hget(receiptKey, 'metadata'))!), + ).not.toHaveProperty('request'); + const resultKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:settlement`; + const originalResult = await redis.get(resultKey); + const intent = { + protocolVersion: 1 as const, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected' as const, + error: 'local cleanup failed after commit', + }; + await expect( + store.settle( + workerId, + assignment.assignmentId, + { ...intent, leaseToken: 'forged' }, + undefined, + undefined, + true, + ), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + await expect( + store.settle( + workerId, + assignment.assignmentId, + intent, + undefined, + 'different-principal', + true, + ), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + await store.settle( + workerId, + assignment.assignmentId, + intent, + undefined, + undefined, + true, + ); + await store.settle( + workerId, + assignment.assignmentId, + intent, + undefined, + undefined, + true, + ); + expect(await redis.get(resultKey)).toBe(originalResult); + await expect(dispatch('a')).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + await store.resetWorkspace(workerId, incarnationId, 'native-workspace:a'); + await expect( + store.settle( + workerId, + assignment.assignmentId, + intent, + undefined, + undefined, + true, + ), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + const next = dispatch('a'); + const nextAssignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + await settle(nextAssignment); + await next; +}); diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts new file mode 100644 index 00000000..5aec49cf --- /dev/null +++ b/service/src/bridge/concurrent-worker.test.ts @@ -0,0 +1,303 @@ +import { expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { RedisBridgeStore } from './store'; +import { BridgeWorker } from '../../../packages/code/src/worker'; +import { WorkspaceToolError } from '../../../packages/code/src/workspace'; +import type { WorkspaceMutationQuarantine } from '../../../packages/code/src/worker'; +import type { BridgeWorkspaceToolCapabilities } from '../../../packages/code/src/protocol'; + +for (const failure of [ + 'execution', + 'cleanup', + 'post-unlink', + 'hung-cleanup', + 'lost-response', + 'all-responses-lost', + 'delivery-outage', +]) { + const cleanupFailure = ['cleanup', 'post-unlink', 'hung-cleanup'].includes( + failure, + ); + test(`concurrent worker isolates ${failure} failure`, async () => { + const redis = new RedisMock() as unknown as Redis; + const store = new RedisBridgeStore(redis, 60, 1000, 2); + const controller = new AbortController(); + const workerId = 'worker-concurrency'; + const incarnationId = 'incarnation-concurrency'; + const guards = new Map(); + const pending = new Set(); + for (const root of ['a', 'b']) + guards.set(root, { + async assertAvailable() { + if (pending.has(root)) throw new Error('quarantined'); + }, + async arm() { + pending.add(root); + }, + async clear() { + if (failure === 'hung-cleanup' && root === 'a') { + pending.delete(root); + await new Promise(() => {}); + } + if (failure === 'post-unlink') pending.delete(root); + if (cleanupFailure && root === 'a') + throw new Error('injected guard cleanup failure'); + pending.delete(root); + }, + async quarantine() { + expect(pending.has(root)).toBe(true); + }, + }); + const capabilities: BridgeWorkspaceToolCapabilities = { + protocolVersion: 1, + operations: ['execute_command'], + workspaces: [{ id: 'a' }, { id: 'b' }], + }; + let startBoth!: () => void; + const bothStarted = new Promise((resolve) => { + startBoth = resolve; + }); + const started = new Set(); + let failedRootExecutions = 0; + const errors: unknown[] = []; + let quarantineAttempts = 0; + let registered!: () => void; + const ready = new Promise((resolve) => { + registered = resolve; + }); + const worker = new BridgeWorker({ + codeApiUrl: 'http://fixture.invalid', + token: 'fixture', + workerId, + incarnationId, + sandboxEndpoint: 'http://sandbox.invalid', + leaseWaitMs: 50, + workspaceCleanupTimeoutMs: 20, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + requiresReadyConfirmation: true, + workspaceLeaseSlots: 2, + workspaceTools: capabilities, + }, + workspaceQuarantines: guards, + onError: (error) => { + errors.push(error); + }, + workspaceTools: { + capabilities, + mutationFailuresAreAtomic: true, + async execute(request) { + if (request.workspaceId === 'a') failedRootExecutions++; + started.add(request.workspaceId); + if (started.size === 2) startBoth(); + await bothStarted; + if (request.workspaceId === 'a' && !cleanupFailure) + throw new WorkspaceToolError( + 'uncertain command', + 'COMMAND_UNAVAILABLE', + true, + ); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + stdout: 'completed', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }, + fetchImpl: (async (url, init) => { + const path = new URL(String(url)).pathname; + const body = JSON.parse(String(init?.body)); + const signal = init?.signal ?? undefined; + let result: object; + if (path.endsWith('/register')) { + const generation = await store.register(body); + result = { + protocolVersion: 1, + workerId, + incarnationId, + registrationGeneration: generation, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + workspaceLeaseSlots: 2, + supportedWorkspaceToolOperations: ['execute_command'], + }; + } else if (path.endsWith('/ready')) { + await store.confirmReady( + workerId, + incarnationId, + body.registrationGeneration, + ); + registered(); + result = { protocolVersion: 1, ready: true }; + } else if (path.endsWith('/lease')) { + result = { + protocolVersion: 1, + serverElapsedMs: 0, + assignment: await store.lease( + workerId, + incarnationId, + body.waitMs, + signal, + undefined, + body.workspaceLeaseSlot, + ), + }; + } else { + const id = path.split('/').at(-2)!; + if (path.endsWith('/ack')) { + await store.acknowledgeLease( + workerId, + incarnationId, + id, + body.generation, + body.leaseToken, + signal, + ); + result = { protocolVersion: 1, accepted: true }; + } else if (path.endsWith('/cancellation')) { + result = { + protocolVersion: 1, + cancelled: await store.cancelled( + workerId, + incarnationId, + id, + signal, + ), + }; + } else if (path.endsWith('/workspace-cleanup')) { + await store.confirmWorkspaceCleanup(workerId, id, body, signal); + result = { protocolVersion: 1, accepted: true }; + } else { + if (path.endsWith('/quarantine')) { + quarantineAttempts++; + if (failure === 'delivery-outage') + throw new TypeError('injected transport outage'); + } + await store.settle( + workerId, + id, + body, + signal, + undefined, + path.endsWith('/quarantine'), + ); + if ( + path.endsWith('/quarantine') && + ((failure === 'lost-response' && quarantineAttempts === 1) || + failure === 'all-responses-lost') + ) + throw new TypeError('injected lost response after commit'); + result = { protocolVersion: 1, accepted: true }; + } + } + return Response.json(result); + }) as typeof fetch, + }); + const running = worker.run(controller.signal); + void running.catch(() => undefined); + try { + await ready; + const results = await Promise.allSettled( + ['a', 'b'].map((workspaceId) => + store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 3000, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId, + command: 'fixture', + }, + }), + ), + ); + if (failure === 'delivery-outage') + expect(results[0]).toMatchObject({ + status: 'rejected', + reason: { code: 'ASSIGNMENT_EXPIRED' }, + }); + else if (!cleanupFailure) + expect(results[0]).toMatchObject({ + status: 'fulfilled', + value: { status: 'rejected' }, + }); + else if (results[0].status === 'fulfilled') + expect(results[0].value).toMatchObject({ + status: 'fulfilled', + result: { stdout: 'completed' }, + }); + else + expect(results[0].reason).toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + expect(results[1]).toMatchObject({ + status: 'fulfilled', + value: { status: 'fulfilled' }, + }); + for (let i = 0; i < 300 && errors.length === 0; i++) + await new Promise((resolve) => setTimeout(resolve, 5)); + if (failure === 'delivery-outage') + expect(errors.length).toBeGreaterThanOrEqual(1); + else expect(errors.length).toBe(1); + if (failure === 'lost-response') expect(quarantineAttempts).toBe(2); + if (failure === 'delivery-outage') + expect(quarantineAttempts).toBeGreaterThanOrEqual(3); + if (failure === 'all-responses-lost') expect(quarantineAttempts).toBe(3); + await expect( + store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 1000, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'a', + command: 'must not execute', + }, + }), + ).rejects.toMatchObject({ + code: + failure === 'delivery-outage' + ? 'ASSIGNMENT_EXPIRED' + : 'WORKSPACE_QUARANTINED', + }); + await expect( + store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 1000, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'b', + command: 'still healthy', + }, + }), + ).resolves.toMatchObject({ status: 'fulfilled' }); + expect([...pending]).toEqual( + ['post-unlink', 'hung-cleanup'].includes(failure) ? [] : ['a'], + ); + expect(started.size).toBe(2); + expect(failedRootExecutions).toBe(1); + } catch (error) { + throw new AggregateError( + [error, ...errors], + `Started roots: ${[...started].join(',')}`, + ); + } finally { + controller.abort(); + await running; + await redis.flushall(); + redis.disconnect(); + } + }); +} diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 08ef1b07..fc409ad2 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -4,7 +4,12 @@ import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; -export const bridgeStore = new RedisBridgeStore(connection); +export const bridgeStore = new RedisBridgeStore( + connection, + undefined, + undefined, + env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, +); export const bridgePairings = new RedisBridgePairingStore(connection); export default createBridgeRouter({ diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index f428bdbe..72e51b4d 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -396,6 +396,9 @@ router.post( registrationGeneration, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + workspaceLeaseSlots: options.store.workspaceLeaseCapacity( + registration.capabilities.workspaceLeaseSlots, + ), supportedWorkspaceToolOperations: [ 'read_file', 'search_text', @@ -529,7 +532,11 @@ router.post( body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !validIncarnationId(body.incarnationId) || !Number.isFinite(requestedWait) || - requestedWait < 0 + requestedWait < 0 || + (body.workspaceLeaseSlot !== undefined && + (!Number.isSafeInteger(body.workspaceLeaseSlot) || + Number(body.workspaceLeaseSlot) < 0 || + Number(body.workspaceLeaseSlot) >= 8)) ) { res.status(400).json({ error: 'Invalid bridge lease request' }); return; @@ -557,6 +564,7 @@ router.post( | { identityId: string } | undefined )?.identityId, + body.workspaceLeaseSlot === undefined ? undefined : Number(body.workspaceLeaseSlot), ); if (leaseController.signal.aborted) { if (assignment != null) await options.store.returnLease(assignment); @@ -630,7 +638,11 @@ router.post( ); router.post( - '/workers/:workerId/assignments/:assignmentId/settle', + [ + '/workers/:workerId/assignments/:assignmentId/settle', + '/workers/:workerId/assignments/:assignmentId/quarantine', + '/workers/:workerId/assignments/:assignmentId/workspace-cleanup', + ], workerAuth, asyncRoute(async (req, res) => { const settlement = req.body as unknown; @@ -644,7 +656,11 @@ router.post( req.once('aborted', abortSettlement); res.once('close', abortSettlement); try { - await options.store.settle( + if (req.path.endsWith('/workspace-cleanup')) { + await options.store.confirmWorkspaceCleanup(req.params.workerId, req.params.assignmentId, + settlement, settlementController.signal, + (res.locals.bridgeWorkerAuthorization as {identityId: string} | undefined)?.identityId); + } else await options.store.settle( req.params.workerId, req.params.assignmentId, settlement, @@ -654,6 +670,7 @@ router.post( | { identityId: string } | undefined )?.identityId, + req.path.endsWith('/quarantine'), ); if (!settlementController.signal.aborted) { res.json({ diff --git a/service/src/bridge/slots.test.ts b/service/src/bridge/slots.test.ts new file mode 100644 index 00000000..dfe89495 --- /dev/null +++ b/service/src/bridge/slots.test.ts @@ -0,0 +1,93 @@ +import { afterEach, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { BridgeAdmissionQueue } from './admission'; +import { BridgeWorkspaceSlots } from './slots'; + +const redis = new RedisMock() as unknown as Redis; +const admission = new BridgeAdmissionQueue(redis); +const slots = new BridgeWorkspaceSlots(redis); +const workerId = 'slot-worker'; +const incarnationId = 'slot-incarnation'; +const prefix = `codeapi:bridge:v1:worker:${workerId}`; +afterEach(async () => { + await redis.flushall(); +}); +async function enqueue(assignmentId: string, workspaceId: string) { + await redis.set(`${prefix}:incarnation`, incarnationId); + await redis.set(`${prefix}:workspace-slot-capacity`, '2'); + await admission.enter(workerId, assignmentId, Date.now() + 5000, workspaceId); + return { + workerId, + incarnationId, + assignmentId, + workspaceId, + capacity: 2, + expiresAtMs: Date.now() + 10000, + }; +} + +test('slots admit independent workspaces, skip a busy root, and bound capacity', async () => { + const a = await enqueue('a', 'root-a'); + const a2 = await enqueue('a2', 'root-a'); + const b = await enqueue('b', 'root-b'); + const c = await enqueue('c', 'root-c'); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(a)).toBe(0); + expect(await slots.reserve(a2)).toBeUndefined(); + expect(await slots.reserve(b)).toBe(1); + expect(await slots.reserve(c)).toBeUndefined(); + expect(await slots.reserve(a)).toBe(0); + await slots.release(workerId, incarnationId, 'a'); + await admission.leave(workerId, 'a'); + expect(await slots.reserve(c)).toBeUndefined(); + expect(await slots.reserve(a2)).toBe(0); +}); + +test('stale slot release cannot erase replacement reservation or aggregate lock', async () => { + const a = await enqueue('a', 'root-a'); + expect(await slots.reserve(a)).toBe(0); + await slots.release(workerId, incarnationId, 'a'); + await admission.leave(workerId, 'a'); + const b = await enqueue('b', 'root-b'); + expect(await slots.reserve(b)).toBe(0); + await slots.release(workerId, incarnationId, 'a'); + expect(await redis.hlen(`${prefix}:workspace-slots`)).toBe(4); + expect(await redis.get(`${prefix}:lock`)).toBe( + `workspace-slots:${incarnationId}`, + ); + await slots.release(workerId, incarnationId, 'b'); + expect(await redis.get(`${prefix}:lock`)).toBeNull(); +}); + +test('legacy locks and older serial admission remain barriers', async () => { + const a = await enqueue('a', 'root-a'); + await redis.set(`${prefix}:lock`, 'legacy-assignment'); + expect(await slots.reserve(a)).toBeUndefined(); + await redis.del(`${prefix}:lock`); + await admission.leave(workerId, 'a'); + await admission.enter(workerId, 'legacy', Date.now() + 5000); + await admission.enter(workerId, 'a', Date.now() + 5000, 'root-a'); + expect(await slots.reserve(a)).toBeUndefined(); + await admission.leave(workerId, 'legacy'); + expect(await slots.reserve(a)).toBe(0); +}); + +test('slots reject invalid capacity and replaced incarnation', async () => { + const a = await enqueue('a', 'root-a'); + for (const capacity of [0, 9, 1.5, NaN]) { + await expect(slots.reserve({ ...a, capacity })).rejects.toThrow('Invalid'); + } + await redis.set(`${prefix}:incarnation`, 'replacement'); + await expect(slots.reserve(a)).rejects.toThrow('replaced'); +}); + +test('releasing a long slot shortens the aggregate expiry to remaining work', async () => { + const a = await enqueue('a', 'root-a'); + const b = { ...await enqueue('b', 'root-b'), expiresAtMs: Date.now() + 3000 }; + await slots.reserve(a); + await slots.reserve(b); + expect(await redis.pttl(`${prefix}:lock`)).toBeGreaterThan(8000); + await slots.release(workerId, incarnationId, 'a'); + expect(await redis.pttl(`${prefix}:lock`)).toBeLessThanOrEqual(3000); +}); diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts new file mode 100644 index 00000000..9c9e7d3b --- /dev/null +++ b/service/src/bridge/slots.ts @@ -0,0 +1,139 @@ +import type Redis from 'ioredis'; + +/** Hard bound keeps every atomic scheduling scan constant-sized. */ +export const MAX_WORKSPACE_LEASE_SLOTS = 8; + +/** Reservations share the legacy admission queue and aggregate worker lock. + * Thus a serial dispatcher or replacement incarnation cannot race active slots. + * Workspace mutation uncertainty is fenced separately by the assignment store. + */ +export class BridgeWorkspaceSlots { + constructor(private readonly redis: Redis) {} + + private keys(workerId: string): string[] { + const prefix = `codeapi:bridge:v1:worker:${encodeURIComponent(workerId)}`; + return [ + `${prefix}:workspace-slots`, + `${prefix}:lock`, + `${prefix}:lock:incarnation`, + `${prefix}:incarnation`, + `${prefix}:admission`, + `${prefix}:admission:deadlines`, + `${prefix}:admission:workspaces`, + `${prefix}:workspace-slot-capacity`, + ]; + } + + async reserve(args: { + workerId: string; + incarnationId: string; + assignmentId: string; + workspaceId: string; + capacity: number; + expiresAtMs: number; + }): Promise { + if ( + !Number.isSafeInteger(args.capacity) || + args.capacity < 1 || + args.capacity > MAX_WORKSPACE_LEASE_SLOTS || + !args.workspaceId || + !Number.isSafeInteger(args.expiresAtMs) || + args.expiresAtMs <= Date.now() + ) { + throw new Error('Invalid workspace slot reservation'); + } + const result = Number( + await this.redis.eval( + [ + "if redis.call('GET', KEYS[4]) ~= ARGV[1] then return -2 end", + "if (redis.call('GET', KEYS[8]) or '1') ~= ARGV[4] then return -2 end", + "local lock = redis.call('GET', KEYS[2])", + "local owner = 'workspace-slots:' .. ARGV[1]", + 'if lock and lock ~= owner then return -1 end', + 'local busy = {}', + 'local free = nil', + 'local latest = tonumber(ARGV[5])', + `for slot = 0, ${MAX_WORKSPACE_LEASE_SLOTS - 1} do`, + " local entry = redis.call('HMGET', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'w:' .. slot, 'e:' .. slot)", + ' local occupied = entry[1]', + ' if occupied then', + ' if entry[2] ~= ARGV[1] or tonumber(entry[4]) <= tonumber(ARGV[6]) then', + " redis.call('HDEL', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'w:' .. slot, 'e:' .. slot)", + ' occupied = false', + ' else', + ' if entry[1] == ARGV[2] then return slot end', + ' busy[entry[3]] = true', + ' latest = math.max(latest, tonumber(entry[4]))', + ' end', + ' end', + ' if not occupied and free == nil and slot < tonumber(ARGV[4]) then free = slot end', + 'end', + 'if free == nil or busy[ARGV[3]] then return -1 end', + // Expiry removes queue metadata, never a workspace uncertainty fence. + "local expired = redis.call('ZRANGEBYSCORE', KEYS[6], '-inf', ARGV[6])", + 'for _, id in ipairs(expired) do', + " redis.call('ZREM', KEYS[5], id)", + " redis.call('ZREM', KEYS[6], id)", + " redis.call('HDEL', KEYS[7], id)", + 'end', + "local pending = redis.call('ZRANGE', KEYS[5], 0, 31)", + 'local selected = nil', + 'for _, id in ipairs(pending) do', + " local workspace = redis.call('HGET', KEYS[7], id)", + // An older serial request remains a barrier until its dispatcher finishes. + ' if not workspace then return -1 end', + ' if not busy[workspace] then selected = id; break end', + 'end', + 'if selected ~= ARGV[2] then return -1 end', + "redis.call('HSET', KEYS[1], 'a:' .. free, ARGV[2], 'i:' .. free, ARGV[1], 'w:' .. free, ARGV[3], 'e:' .. free, ARGV[5])", + "redis.call('PEXPIREAT', KEYS[1], latest)", + "redis.call('SET', KEYS[2], owner, 'PXAT', latest)", + "redis.call('SET', KEYS[3], ARGV[1], 'PXAT', latest)", + 'return free', + ].join('\n'), + 8, + ...this.keys(args.workerId), + args.incarnationId, + args.assignmentId, + args.workspaceId, + args.capacity, + args.expiresAtMs, + Date.now(), + ), + ); + if (result === -2) + throw new Error('Workspace slot incarnation was replaced'); + return result < 0 ? undefined : result; + } + + async release( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + await this.redis.eval( + [ + 'local latest = 0', + `for slot = 0, ${MAX_WORKSPACE_LEASE_SLOTS - 1} do`, + " local entry = redis.call('HMGET', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'e:' .. slot)", + ' if entry[2] == ARGV[1] and entry[1] == ARGV[2] then', + " redis.call('HDEL', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'w:' .. slot, 'e:' .. slot)", + ' elseif type(entry[1]) == "string" then latest = math.max(latest, tonumber(entry[3]))', + ' end', + 'end', + "if redis.call('HLEN', KEYS[1]) == 0 and redis.call('GET', KEYS[2]) == 'workspace-slots:' .. ARGV[1] then", + " redis.call('DEL', KEYS[1], KEYS[2], KEYS[3])", + "elseif latest > 0 and redis.call('GET', KEYS[2]) == 'workspace-slots:' .. ARGV[1] then", + ' for _, key in ipairs(KEYS) do', + " redis.call('PEXPIREAT', key, latest)", + ' end', + 'end', + 'return 1', + ].join('\n'), + 3, + ...this.keys(workerId).slice(0, 3), + incarnationId, + assignmentId, + ); + } +} diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index a00e4f2b..41e11839 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -19,6 +19,7 @@ import { } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; import { BridgeAdmissionQueue } from './admission'; +import { BridgeWorkspaceSlots } from './slots'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; @@ -64,6 +65,31 @@ export class BridgeStoreError extends Error { interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; workerIdentityId?: string; + workspaceFence?: string; +} + +type AssignmentOwnership = Pick< + StoredAssignment, + | 'assignmentId' + | 'workerId' + | 'incarnationId' + | 'workspaceFence' + | 'workspaceLeaseSlot' + | 'generation' + | 'leaseTokenHash' + | 'workerIdentityId' + | 'expiresAt' + | 'runtimeSessionId' +>; + +function workspaceFenceReceiptKey(assignmentId: string): string { + return `${assignmentKey(assignmentId)}:workspace-fence-owner`; +} + +function assignmentWorkspace( + assignment: AssignmentOwnership, +): string | undefined { + return assignment.workspaceFence ?? assignment.runtimeSessionId; } export interface RegisteredBridgeWorker extends BridgeWorkerRegistration { @@ -173,16 +199,28 @@ function workspaceQuarantineKey( return `${PREFIX}:worker:${encodeURIComponent(workerId)}:workspace:${sessionHash}:quarantined`; } -function queueKey(workerId: string, incarnationId: string): string { - return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:assignments`; +function queueKey( + workerId: string, + incarnationId: string, + slot?: number, +): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:assignments${slot === undefined ? '' : `:slot:${slot}`}`; } -function leaseClaimKey(workerId: string, incarnationId: string): string { - return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:lease-claim`; +function leaseClaimKey( + workerId: string, + incarnationId: string, + slot?: number, +): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:lease-claim${slot === undefined ? '' : `:slot:${slot}`}`; } -function leaseAckKey(workerId: string, incarnationId: string): string { - return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:lease-ack`; +function leaseAckKey( + workerId: string, + incarnationId: string, + slot?: number, +): string { + return `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:${incarnationId}:lease-ack${slot === undefined ? '' : `:slot:${slot}`}`; } function generationKey(workerId: string): string { @@ -283,7 +321,22 @@ export class RedisBridgeStore { private readonly redis: Redis, private readonly workerTtlSeconds = DEFAULT_WORKER_TTL_SECONDS, private readonly redisCommandTimeoutMs = DEFAULT_REDIS_COMMAND_TIMEOUT_MS, - ) {} + private readonly maxWorkspaceLeaseSlots = 1, + ) { + if ( + !Number.isSafeInteger(maxWorkspaceLeaseSlots) || + maxWorkspaceLeaseSlots < 1 || + maxWorkspaceLeaseSlots > 8 + ) { + throw new Error( + 'Workspace lease slot ceiling must be an integer from 1 to 8', + ); + } + } + + workspaceLeaseCapacity(requested = 1): number { + return Math.min(this.maxWorkspaceLeaseSlots, requested); + } private async dispatchCommand( command: () => Promise, @@ -379,12 +432,34 @@ export class RedisBridgeStore { async register( registration: RegisteredBridgeWorker, - authorization?: string | { - identityId?: string; - pairingGeneration?: number; - activeCredentialId?: string; - }, + authorization?: + | string + | { + identityId?: string; + pairingGeneration?: number; + activeCredentialId?: string; + }, ): Promise { + if (registration.capabilities.workspaceLeaseSlots !== undefined) { + if ( + !isValidBridgeWorkerCapabilities(registration.capabilities) || + registration.capabilities.requiresReadyConfirmation !== true + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + 'Concurrent workspaces require readiness negotiation', + ); + } + registration = { + ...registration, + capabilities: { + ...registration.capabilities, + workspaceLeaseSlots: this.workspaceLeaseCapacity( + registration.capabilities.workspaceLeaseSlots, + ), + }, + }; + } const authorizationObject = typeof authorization === 'object' ? authorization : undefined; const expectedActiveCredentialId = @@ -396,12 +471,12 @@ export class RedisBridgeStore { ' local pairingGeneration = redis.call(\'GET\', KEYS[7]) or "0"', ' if pairingGeneration ~= ARGV[5] then return -5 end', ' if ARGV[6] ~= "" then', - ' if redis.call(\'GET\', KEYS[8]) ~= ARGV[6] then return -5 end', + " if redis.call('GET', KEYS[8]) ~= ARGV[6] then return -5 end", ' elseif ARGV[7] ~= "" and redis.call(\'GET\', KEYS[9]) ~= ARGV[7] then return -5', ' end', 'end', 'if ARGV[8] ~= "" then', - ' local stableIdentity = redis.call(\'GET\', KEYS[8])', + " local stableIdentity = redis.call('GET', KEYS[8])", ' if stableIdentity and stableIdentity ~= ARGV[8] then return -4 end', ' if not stableIdentity then', ' if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[9]) ~= ARGV[7] then return -4 end', @@ -409,29 +484,31 @@ export class RedisBridgeStore { ' end', 'elseif ARGV[7] ~= "" and redis.call(\'GET\', KEYS[9]) ~= ARGV[7] then return -4', 'end', - 'if redis.call(\'EXISTS\', KEYS[3]) == 1 then return -2 end', - 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return -1 end', - 'local current = redis.call(\'GET\', KEYS[4])', - 'if not current and redis.call(\'EXISTS\', KEYS[5]) == 1 then', - ' local owner = redis.call(\'GET\', KEYS[6])', + "if redis.call('EXISTS', KEYS[3]) == 1 then return -2 end", + "if redis.call('EXISTS', KEYS[2]) == 1 then return -1 end", + "local current = redis.call('GET', KEYS[4])", + "if current == ARGV[1] and redis.call('EXISTS', KEYS[5]) == 1 and (redis.call('GET', KEYS[13]) or \"1\") ~= ARGV[10] then return -3 end", + "if not current and redis.call('EXISTS', KEYS[5]) == 1 then", + " local owner = redis.call('GET', KEYS[6])", ' if owner ~= ARGV[1] then return -3 end', 'end', 'if current then', ' if current ~= ARGV[1] then', - ' if redis.call(\'EXISTS\', KEYS[5]) == 1 then return -3 end', - ' redis.call(\'SET\', ARGV[4] .. current .. \':fenced\', \"1\")', + " if redis.call('EXISTS', KEYS[5]) == 1 then return -3 end", + " redis.call('SET', ARGV[4] .. current .. ':fenced', \"1\")", ' end', 'end', 'local registrationGeneration = tonumber(redis.call(\'GET\', KEYS[10]) or \"0\")', - 'local registrationGenerationIncarnation = redis.call(\'GET\', KEYS[11])', + "local registrationGenerationIncarnation = redis.call('GET', KEYS[11])", 'local registrationGenerationChanged = false', 'if registrationGeneration < 1 or registrationGenerationIncarnation ~= ARGV[1] then', - ' registrationGeneration = redis.call(\'INCR\', KEYS[10])', - ' redis.call(\'SET\', KEYS[11], ARGV[1])', + " registrationGeneration = redis.call('INCR', KEYS[10])", + " redis.call('SET', KEYS[11], ARGV[1])", ' registrationGenerationChanged = true', 'end', 'redis.call(\'SET\', KEYS[1], ARGV[2], \"EX\", ARGV[3])', 'redis.call(\'SET\', KEYS[4], ARGV[1], \"EX\", ARGV[3])', + 'redis.call(\'SET\', KEYS[13], ARGV[10], \"EX\", ARGV[3])', 'if ARGV[9] == "1" and registrationGenerationChanged then redis.call(\'DEL\', KEYS[12]) end', 'return registrationGeneration', ].join('\n'); @@ -439,9 +516,12 @@ export class RedisBridgeStore { await boundedCommand( this.redis.eval( script, - 12, + 13, workerKey(registration.workerId), - incarnationFenceKey(registration.workerId, registration.incarnationId), + incarnationFenceKey( + registration.workerId, + registration.incarnationId, + ), quarantineKey(registration.workerId, registration.incarnationId), workerIncarnationKey(registration.workerId), lockKey(registration.workerId), @@ -452,6 +532,7 @@ export class RedisBridgeStore { workerRegistrationGenerationKey(registration.workerId), workerRegistrationGenerationIncarnationKey(registration.workerId), workerReadyKey(registration.workerId), + `${PREFIX}:worker:${encodeURIComponent(registration.workerId)}:workspace-slot-capacity`, registration.incarnationId, JSON.stringify(registration), String(this.workerTtlSeconds), @@ -462,7 +543,10 @@ export class RedisBridgeStore { authorizationObject?.identityId ?? '', expectedActiveCredentialId ?? '', registration.identityId ?? '', - registration.capabilities.requiresReadyConfirmation === true ? '1' : '0', + registration.capabilities.requiresReadyConfirmation === true + ? '1' + : '0', + String(registration.capabilities.workspaceLeaseSlots ?? 1), ), this.redisCommandTimeoutMs, 'Bridge worker registration', @@ -499,7 +583,9 @@ export class RedisBridgeStore { ); } if (!Number.isSafeInteger(result) || result < 1) { - throw new Error('Bridge worker registration returned an invalid generation'); + throw new Error( + 'Bridge worker registration returned an invalid generation', + ); } return result; } @@ -622,12 +708,17 @@ export class RedisBridgeStore { registration: RegisteredBridgeWorker, ) => Promise; }): Promise { - if (args.executionTimeoutMs !== undefined && ( - args.workspaceRequest == null || - !Number.isSafeInteger(args.executionTimeoutMs) || - args.executionTimeoutMs < 1 || args.executionTimeoutMs > 305_000 - )) { - throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid workspace execution budget'); + if ( + args.executionTimeoutMs !== undefined && + (args.workspaceRequest == null || + !Number.isSafeInteger(args.executionTimeoutMs) || + args.executionTimeoutMs < 1 || + args.executionTimeoutMs > 305_000) + ) { + throw new BridgeStoreError( + 'ASSIGNMENT_INVALID', + 'Invalid workspace execution budget', + ); } this.assertDispatchActive(args.signal, args.deadlineAtMs); const dispatchable = await this.dispatchCommand( @@ -642,6 +733,15 @@ export class RedisBridgeStore { ); } let { registration, readyToken } = dispatchable; + if ( + (registration.capabilities.workspaceLeaseSlots ?? 1) > + this.maxWorkspaceLeaseSlots + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + 'Worker slot negotiation exceeds this Code API replica ceiling; use consistent replica configuration', + ); + } if ( (args.requireTenantBinding === true && registration.binding == null) || (registration.binding != null && @@ -692,44 +792,94 @@ export class RedisBridgeStore { const assignmentId = randomBytes(18).toString('base64url'); const leaseToken = randomBytes(32).toString('base64url'); // The lock is acquired before admission finishes; it must outlive the later execution deadline. - const ttlSeconds = assignmentTtlSeconds(args.deadlineAtMs + (args.executionTimeoutMs ?? 0)); + const ttlSeconds = assignmentTtlSeconds( + args.deadlineAtMs + (args.executionTimeoutMs ?? 0), + ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; + let workspaceLeaseSlot: number | undefined; + const workspaceSlots = + args.workspaceRequest != null && + (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 + ? new BridgeWorkspaceSlots(this.redis) + : undefined; let resultCommitted = false; - const admission = args.workspaceRequest == null - ? undefined - : new BridgeAdmissionQueue(this.redis); + const admission = + args.workspaceRequest == null + ? undefined + : new BridgeAdmissionQueue(this.redis); try { - if (admission != null && !(await this.dispatchCommand( - () => admission.enter(args.workerId, assignmentId, args.deadlineAtMs), - args, - 'Bridge admission enqueue', - ))) { - throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); - } - let locked = false; - do { - if (admission != null && !(await this.dispatchCommand( - () => admission.isHead(args.workerId, assignmentId), - args, - 'Bridge admission position', - ))) { - await delay(Math.min(POLL_INTERVAL_MS, args.deadlineAtMs - Date.now()), args.signal); - continue; - } - locked = await this.dispatchCommand( + if ( + admission != null && + !(await this.dispatchCommand( () => - this.acquireLock( + admission.enter( args.workerId, assignmentId, - lockIncarnationId, - ttlSeconds, + args.deadlineAtMs, + workspaceSlots == null + ? undefined + : args.workspaceRequest?.workspaceId, ), args, - 'Bridge assignment lock acquisition', + 'Bridge admission enqueue', + )) + ) { + throw new BridgeStoreError( + 'WORKER_QUEUE_FULL', + 'Bridge worker pending request limit reached', ); + } + let locked = false; + do { + if ( + admission != null && + workspaceSlots == null && + !(await this.dispatchCommand( + () => admission.isHead(args.workerId, assignmentId), + args, + 'Bridge admission position', + )) + ) { + await delay( + Math.min(POLL_INTERVAL_MS, args.deadlineAtMs - Date.now()), + args.signal, + ); + continue; + } + if (workspaceSlots != null) { + workspaceLeaseSlot = await this.dispatchCommand( + () => + workspaceSlots.reserve({ + workerId: args.workerId, + incarnationId: lockIncarnationId, + assignmentId, + workspaceId: args.workspaceRequest!.workspaceId, + capacity: registration.capabilities.workspaceLeaseSlots!, + expiresAtMs: Date.now() + ttlSeconds * 1000, + }), + args, + 'Bridge workspace slot acquisition', + ); + locked = workspaceLeaseSlot !== undefined; + } else { + locked = await this.dispatchCommand( + () => + this.acquireLock( + args.workerId, + assignmentId, + lockIncarnationId, + ttlSeconds, + ), + args, + 'Bridge assignment lock acquisition', + ); + } if (!locked && admission != null) { - await delay(Math.min(POLL_INTERVAL_MS, args.deadlineAtMs - Date.now()), args.signal); + await delay( + Math.min(POLL_INTERVAL_MS, args.deadlineAtMs - Date.now()), + args.signal, + ); } } while (!locked && admission != null); if (!locked) { @@ -750,12 +900,21 @@ export class RedisBridgeStore { current == null || current.registration.incarnationId !== registration.incarnationId || current.registration.identityId !== registration.identityId || - current.registration.binding?.tenantId !== registration.binding?.tenantId + current.registration.binding?.tenantId !== + registration.binding?.tenantId ) { - throw new BridgeStoreError('WORKER_OFFLINE', 'Bridge worker changed while the request was waiting'); + throw new BridgeStoreError( + 'WORKER_OFFLINE', + 'Bridge worker changed while the request was waiting', + ); } - if (!supportsWorkspaceTool(current.registration, args.workspaceRequest!)) { - throw new BridgeStoreError('WORKER_MISMATCH', 'Bridge worker capabilities changed while the request was waiting'); + if ( + !supportsWorkspaceTool(current.registration, args.workspaceRequest!) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + 'Bridge worker capabilities changed while the request was waiting', + ); } } this.assertDispatchActive(args.signal, args.deadlineAtMs); @@ -775,6 +934,12 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), + ...(workspaceLeaseSlot === undefined + ? {} + : { + workspaceLeaseSlot, + workspaceFence: `native-workspace:${args.workspaceRequest!.workspaceId}`, + }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } : {}), @@ -835,7 +1000,10 @@ export class RedisBridgeStore { } if ( args.workspaceRequest != null && - !supportsWorkspaceTool(replacement.registration, args.workspaceRequest) + !supportsWorkspaceTool( + replacement.registration, + args.workspaceRequest, + ) ) { throw new BridgeStoreError( 'WORKER_MISMATCH', @@ -894,6 +1062,17 @@ export class RedisBridgeStore { } else { await this.cleanupDispatch(args.workerId, assignmentId, assignment); } + if (workspaceSlots != null && assignment == null) { + await boundedCommand( + workspaceSlots.release( + args.workerId, + lockIncarnationId, + assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge unassigned slot cleanup', + ); + } } } @@ -903,18 +1082,32 @@ export class RedisBridgeStore { waitMs: number, signal?: AbortSignal, identityId?: string, + slot?: number, ): Promise { + if (slot !== undefined) { + const registration = await this.registration(workerId); + if ( + !Number.isSafeInteger(slot) || + slot < 0 || + slot >= this.maxWorkspaceLeaseSlots || + slot >= (registration?.capabilities.workspaceLeaseSlots ?? 1) || + (registration?.capabilities.workspaceLeaseSlots ?? 1) <= 1 || + registration?.incarnationId !== incarnationId + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + 'Workspace lease slot was not negotiated', + ); + } + } const deadline = Date.now() + waitMs; let firstPoll = true; - while ( - !signalAborted(signal) && - (firstPoll || Date.now() < deadline) - ) { + while (!signalAborted(signal) && (firstPoll || Date.now() < deadline)) { firstPoll = false; let assignmentId: string | null; try { assignmentId = await this.leaseCommand( - this.claimOrPopLease(workerId, incarnationId, identityId), + this.claimOrPopLease(workerId, incarnationId, identityId, slot), signal, 'Bridge lease claim', ); @@ -938,10 +1131,11 @@ export class RedisBridgeStore { if ( assignment == null || assignment.workerId !== workerId || - assignment.incarnationId !== incarnationId + assignment.incarnationId !== incarnationId || + assignment.workspaceLeaseSlot !== slot ) { await this.leaseCommand( - this.discardLeaseClaim(workerId, incarnationId, assignmentId), + this.discardLeaseClaim(workerId, incarnationId, assignmentId, slot), signal, 'Bridge lease claim discard', ); @@ -964,7 +1158,7 @@ export class RedisBridgeStore { } if (assignment.workerIdentityId !== identityId) { await this.leaseCommand( - this.discardLeaseClaim(workerId, incarnationId, assignmentId), + this.discardLeaseClaim(workerId, incarnationId, assignmentId, slot), signal, 'Bridge unauthorized lease discard', ); @@ -973,7 +1167,7 @@ export class RedisBridgeStore { if (Date.parse(assignment.expiresAt) <= Date.now()) { const acknowledged = (await this.leaseCommand( - this.redis.get(leaseAckKey(workerId, incarnationId)), + this.redis.get(leaseAckKey(workerId, incarnationId, slot)), signal, 'Bridge lease acknowledgement read', )) === assignmentId; @@ -985,7 +1179,7 @@ export class RedisBridgeStore { ); } await this.leaseCommand( - this.discardLeaseClaim(workerId, incarnationId, assignmentId), + this.discardLeaseClaim(workerId, incarnationId, assignmentId, slot), signal, 'Bridge expired lease discard', ); @@ -998,6 +1192,7 @@ export class RedisBridgeStore { const { leaseTokenHash: _leaseTokenHash, workerIdentityId: _workerIdentityId, + workspaceFence: _workspaceFence, ...wireAssignment } = assignment; return { @@ -1012,6 +1207,7 @@ export class RedisBridgeStore { workerId, incarnationId, assignmentId, + slot, ); if (signalAborted(signal)) return undefined; throw error; @@ -1061,8 +1257,8 @@ export class RedisBridgeStore { 'return 1', ].join('\n'), 2, - leaseClaimKey(workerId, incarnationId), - leaseAckKey(workerId, incarnationId), + leaseClaimKey(workerId, incarnationId, assignment.workspaceLeaseSlot), + leaseAckKey(workerId, incarnationId, assignment.workspaceLeaseSlot), assignmentId, String(ttlSeconds), ), @@ -1082,6 +1278,7 @@ export class RedisBridgeStore { workerId: string, incarnationId: string, identityId?: string, + slot?: number, ): Promise { const result = await this.redis.eval( [ @@ -1099,8 +1296,8 @@ export class RedisBridgeStore { 'return assignment', ].join('\n'), 3, - queueKey(workerId, incarnationId), - leaseClaimKey(workerId, incarnationId), + queueKey(workerId, incarnationId, slot), + leaseClaimKey(workerId, incarnationId, slot), workerStableIdentityKey(workerId), identityId ?? '', ); @@ -1111,6 +1308,7 @@ export class RedisBridgeStore { workerId: string, incarnationId: string, assignmentId: string, + slot?: number, ): Promise { await this.redis.eval( [ @@ -1120,8 +1318,8 @@ export class RedisBridgeStore { 'return 0', ].join('\n'), 2, - leaseClaimKey(workerId, incarnationId), - leaseAckKey(workerId, incarnationId), + leaseClaimKey(workerId, incarnationId, slot), + leaseAckKey(workerId, incarnationId, slot), assignmentId, ); } @@ -1131,6 +1329,7 @@ export class RedisBridgeStore { assignment.workerId, assignment.incarnationId, assignment.assignmentId, + assignment.workspaceLeaseSlot, ); } @@ -1138,6 +1337,7 @@ export class RedisBridgeStore { workerId: string, incarnationId: string, assignmentId: string, + slot?: number, ): Promise { await boundedCommand( this.redis.eval( @@ -1153,9 +1353,9 @@ export class RedisBridgeStore { ].join('\n'), 4, assignmentKey(assignmentId), - queueKey(workerId, incarnationId), - leaseClaimKey(workerId, incarnationId), - leaseAckKey(workerId, incarnationId), + queueKey(workerId, incarnationId, slot), + leaseClaimKey(workerId, incarnationId, slot), + leaseAckKey(workerId, incarnationId, slot), assignmentId, ), this.redisCommandTimeoutMs, @@ -1167,11 +1367,12 @@ export class RedisBridgeStore { workerId: string, incarnationId: string, assignmentId: string, + slot?: number, ): Promise { let lastError: unknown; for (let attempt = 0; attempt < 3; attempt += 1) { try { - await this.returnLeaseById(workerId, incarnationId, assignmentId); + await this.returnLeaseById(workerId, incarnationId, assignmentId, slot); return; } catch (error) { lastError = error; @@ -1184,7 +1385,7 @@ export class RedisBridgeStore { private async clearUndeliveredWorkspaceFence( assignment: StoredAssignment, ): Promise { - if (assignment.runtimeSessionId === undefined) return; + if (assignmentWorkspace(assignment) === undefined) return; await this.redis.eval( [ "if redis.call('GET', KEYS[1]) == ARGV[1] then", @@ -1195,7 +1396,7 @@ export class RedisBridgeStore { 1, workspaceQuarantineKey( assignment.workerId, - assignment.runtimeSessionId, + assignmentWorkspace(assignment)!, ), assignment.assignmentId, ); @@ -1207,15 +1408,28 @@ export class RedisBridgeStore { settlement: AnyCodeBridgeSettlement, signal?: AbortSignal, identityId?: string, + quarantineWorkspace = false, ): Promise { + if (quarantineWorkspace) { + await this.quarantineSettledWorkspace( + workerId, + assignmentId, + settlement, + signal, + identityId, + ); + return; + } const serializedSettlement = JSON.stringify(settlement); const existingSettlement = await this.leaseCommand( this.redis.get(settlementKey(assignmentId)), signal, 'Bridge settlement existing read', ); - if (existingSettlement === serializedSettlement) return; - if (existingSettlement != null) { + if ( + existingSettlement != null && + existingSettlement !== serializedSettlement + ) { throw new BridgeStoreError( 'ASSIGNMENT_FENCED', 'Bridge assignment was already settled with a different result', @@ -1226,7 +1440,14 @@ export class RedisBridgeStore { signal, 'Bridge settlement assignment read', ); + if ( + existingSettlement === serializedSettlement && + (assignment?.workspaceLeaseSlot === undefined || + settlement.status !== 'rejected') + ) + return; if (assignment == null) { + if (existingSettlement === serializedSettlement) return; throw new BridgeStoreError( 'ASSIGNMENT_NOT_FOUND', 'Bridge assignment was not found', @@ -1268,38 +1489,48 @@ export class RedisBridgeStore { const settlementKeys = [ assignmentKey(assignmentId), settlementKey(assignmentId), - leaseClaimKey(workerId, assignment.incarnationId), - leaseAckKey(workerId, assignment.incarnationId), + leaseClaimKey( + workerId, + assignment.incarnationId, + assignment.workspaceLeaseSlot, + ), + leaseAckKey( + workerId, + assignment.incarnationId, + assignment.workspaceLeaseSlot, + ), assignmentDeadlineKey(assignmentId), ]; - if (assignment.runtimeSessionId !== undefined) { + if (assignmentWorkspace(assignment) !== undefined) { settlementKeys.push( - workspaceQuarantineKey(workerId, assignment.runtimeSessionId), + workspaceQuarantineKey(workerId, assignmentWorkspace(assignment)!), ); } - const hasWorkspace = assignment.runtimeSessionId !== undefined; + const hasWorkspace = assignmentWorkspace(assignment) !== undefined; settlementKeys.push( `${PREFIX}:stable-identity:${workerId}`, workerIncarnationKey(workerId), ); const script = [ - 'local existing = redis.call(\'GET\', KEYS[2])', + "local existing = redis.call('GET', KEYS[2])", 'if existing then', ' if existing == ARGV[1] then return 2 end', ' return -1', 'end', - 'if redis.call(\'EXISTS\', KEYS[1]) == 0 then return 0 end', + "if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end", 'if ARGV[6] == "1" and redis.call(\'GET\', KEYS[6]) ~= ARGV[3] then return -2 end', 'if ARGV[4] ~= "rejected" and redis.call(\'EXISTS\', KEYS[5]) == 0 then return -3 end', 'local stableIdentityKey = KEYS[#KEYS - 1]', 'if ARGV[5] ~= "" then', - ' if redis.call(\'GET\', stableIdentityKey) ~= ARGV[5] then return -4 end', - 'elseif redis.call(\'EXISTS\', stableIdentityKey) == 1 then return -4', + " if redis.call('GET', stableIdentityKey) ~= ARGV[5] then return -4 end", + "elseif redis.call('EXISTS', stableIdentityKey) == 1 then return -4", 'end', - 'if redis.call(\'GET\', KEYS[#KEYS]) ~= ARGV[7] then return -4 end', + "if redis.call('GET', KEYS[#KEYS]) ~= ARGV[7] then return -4 end", 'redis.call(\'SET\', KEYS[2], ARGV[1], \"EX\", ARGV[2])', - 'if redis.call(\'GET\', KEYS[3]) == ARGV[3] then redis.call(\'DEL\', KEYS[3], KEYS[4]) end', - 'if ARGV[6] == "1" and ARGV[4] == "rejected" then redis.call(\'DEL\', KEYS[6]) end', + "if redis.call('GET', KEYS[3]) == ARGV[3] then redis.call('DEL', KEYS[3], KEYS[4]) end", + 'if ARGV[6] == "1" and ARGV[4] == "rejected" and ARGV[8] ~= "1" then', + " redis.call('DEL', KEYS[6])", + 'end', 'return 1', ].join('\n'); const accepted = Number( @@ -1315,6 +1546,7 @@ export class RedisBridgeStore { identityId ?? '', hasWorkspace ? '1' : '0', settlement.incarnationId, + assignment.workspaceLeaseSlot === undefined ? '0' : '1', ), signal, 'Bridge settlement commit', @@ -1350,6 +1582,15 @@ export class RedisBridgeStore { 'Bridge assignment closed before settlement was committed', ); } + if ( + assignment.workspaceLeaseSlot !== undefined && + settlement.status === 'rejected' + ) { + // The dispatcher may already have timed out and finished its cleanup. + // Release only this settled reservation, retaining a quarantine marker. + await this.commitPendingWorkspace(assignment, settlement); + await this.cleanupWithRetry(workerId, assignmentId, assignment); + } } async cancelled( @@ -1419,6 +1660,126 @@ export class RedisBridgeStore { ); } + async confirmWorkspaceCleanup( + workerId: string, + assignmentId: string, + intent: AnyCodeBridgeSettlement, + signal?: AbortSignal, + identityId?: string, + ): Promise { + await this.quarantineSettledWorkspace( + workerId, + assignmentId, + intent, + signal, + identityId, + false, + ); + } + + private async quarantineSettledWorkspace( + workerId: string, + assignmentId: string, + settlement: AnyCodeBridgeSettlement, + signal?: AbortSignal, + identityId?: string, + quarantine = true, + ): Promise { + // Keep a small, expiring ownership receipt separate from assignment cleanup. + // A local guard can fail to clear after the result has already committed. + const receiptKey = workspaceFenceReceiptKey(assignmentId); + const raw = await this.leaseCommand( + this.redis.hgetall(receiptKey), + signal, + 'Workspace fence ownership read', + ); + const receipt = + raw.metadata == null + ? undefined + : (JSON.parse(raw.metadata) as AssignmentOwnership); + if ( + receipt == null || + receipt.workerId !== workerId || + receipt.assignmentId !== assignmentId || + receipt.workspaceFence == null || + receipt.workspaceLeaseSlot === undefined || + settlement.status !== 'rejected' || + receipt.incarnationId !== settlement.incarnationId || + receipt.generation !== settlement.generation || + receipt.leaseTokenHash !== tokenHash(settlement.leaseToken) || + receipt.workerIdentityId !== identityId + ) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Workspace quarantine ownership is stale', + ); + } + const fence = workspaceQuarantineKey(workerId, receipt.workspaceFence); + const accepted = Number( + await this.leaseCommand( + this.redis.eval( + [ + "if redis.call('HGET', KEYS[1], 'metadata') ~= ARGV[1] then return 0 end", + "if redis.call('HGET', KEYS[1], 'epoch') ~= ARGV[4] then return 0 end", + "if redis.call('GET', KEYS[2]) ~= ARGV[2] then return 0 end", + "if (redis.call('GET', KEYS[3]) or '') ~= ARGV[3] then return 0 end", + "if (redis.call('GET', KEYS[4]) or '0') ~= ARGV[4] then return 0 end", + 'if ARGV[8] == "1" then', + // A completed cleanup receipt is terminal: a lost response must + // not let a late quarantine overwrite a newer root owner. + " if redis.call('HGET', KEYS[1], 'localCleanup') == '1' and redis.call('HGET', KEYS[1], 'resultCommitted') == '1' then return 1 end", + " redis.call('SET', KEYS[5], 'quarantined:' .. ARGV[5])", + // Never replace a committed result. Before settlement, terminate the waiter. + " redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[7], 'NX')", + 'else', + " local fence = redis.call('GET', KEYS[5])", + " if fence and string.sub(fence, 1, 12) == 'quarantined:' then return 0 end", + " redis.call('HSET', KEYS[1], 'localCleanup', '1')", + " if redis.call('HGET', KEYS[1], 'resultCommitted') == '1' and fence == ARGV[5] then redis.call('DEL', KEYS[5]) end", + 'end', + "if redis.call('GET', KEYS[7]) == ARGV[5] then redis.call('DEL', KEYS[7]) end", + "if redis.call('GET', KEYS[8]) == ARGV[5] then redis.call('DEL', KEYS[8]) end", + 'return 1', + ].join('\n'), + 8, + receiptKey, + workerIncarnationKey(workerId), + workerStableIdentityKey(workerId), + `${fence}:epoch`, + fence, + settlementKey(assignmentId), + leaseClaimKey( + workerId, + receipt.incarnationId, + receipt.workspaceLeaseSlot, + ), + leaseAckKey( + workerId, + receipt.incarnationId, + receipt.workspaceLeaseSlot, + ), + raw.metadata, + receipt.incarnationId, + identityId ?? '', + raw.epoch, + assignmentId, + JSON.stringify(settlement), + assignmentTtlSeconds(Date.parse(receipt.expiresAt)), + quarantine ? '1' : '0', + ), + signal, + 'Workspace quarantine fence commit', + ), + ); + if (accepted !== 1) { + throw new BridgeStoreError( + 'ASSIGNMENT_FENCED', + 'Workspace quarantine ownership changed or was reset', + ); + } + await this.cleanupWithRetry(workerId, assignmentId, receipt); + } + async resetWorkspace( workerId: string, incarnationId: string, @@ -1432,12 +1793,14 @@ export class RedisBridgeStore { "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return -1 end", "if redis.call('EXISTS', KEYS[2]) == 1 then return -2 end", "redis.call('DEL', KEYS[3])", + "if redis.call('EXISTS', KEYS[4]) == 1 then redis.call('INCR', KEYS[4]) end", 'return 1', ].join('\n'), - 3, + 4, workerIncarnationKey(workerId), lockKey(workerId), workspaceQuarantineKey(workerId, runtimeSessionId), + `${workspaceQuarantineKey(workerId, runtimeSessionId)}:epoch`, incarnationId, ), signal, @@ -1543,11 +1906,11 @@ export class RedisBridgeStore { settlementKey(assignment.assignmentId), assignmentDeadlineKey(assignment.assignmentId), ]; - if (assignment.runtimeSessionId !== undefined) { + if (assignmentWorkspace(assignment) !== undefined) { closeKeys.push( workspaceQuarantineKey( assignment.workerId, - assignment.runtimeSessionId, + assignmentWorkspace(assignment)!, ), ); } @@ -1555,11 +1918,11 @@ export class RedisBridgeStore { // Keep acknowledged assignment metadata for late clean rejection recovery, // but atomically revoke fulfillment when no settlement has won yet. const closeScript = [ - 'local settlement = redis.call(\'GET\', KEYS[2])', + "local settlement = redis.call('GET', KEYS[2])", 'if settlement then return settlement end', - 'redis.call(\'DEL\', KEYS[3])', - 'if #KEYS == 4 and redis.call(\'GET\', KEYS[4]) == ARGV[1] then return nil end', - 'redis.call(\'DEL\', KEYS[1])', + "redis.call('DEL', KEYS[3])", + "if #KEYS == 4 and redis.call('GET', KEYS[4]) == ARGV[1] then return nil end", + "redis.call('DEL', KEYS[1])", 'return nil', ].join('\n'); const finalSettlement = await boundedCommand( @@ -1586,19 +1949,14 @@ export class RedisBridgeStore { private async cancel( assignmentId: string, - assignment?: StoredAssignment, + assignment?: AssignmentOwnership, ): Promise { const ttlSeconds = assignment == null ? 30 : assignmentTtlSeconds(Date.parse(assignment.expiresAt)); await boundedCommand( - this.redis.set( - cancellationKey(assignmentId), - '1', - 'EX', - ttlSeconds, - ), + this.redis.set(cancellationKey(assignmentId), '1', 'EX', ttlSeconds), this.redisCommandTimeoutMs, 'Bridge assignment cancellation', ); @@ -1610,33 +1968,63 @@ export class RedisBridgeStore { readyToken?: string, ): Promise { const script = [ - 'if redis.call(\'GET\', KEYS[1]) ~= ARGV[1] then return 0 end', + "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', - 'if #KEYS == 7 and redis.call(\'EXISTS\', KEYS[7]) == 1 then return -1 end', + "if #KEYS >= 7 and redis.call('EXISTS', KEYS[7]) == 1 then return -1 end", 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', - 'redis.call(\'RPUSH\', KEYS[3], ARGV[4])', - 'redis.call(\'EXPIRE\', KEYS[3], ARGV[3])', - 'redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])', + "redis.call('RPUSH', KEYS[3], ARGV[4])", + "redis.call('EXPIRE', KEYS[3], ARGV[3])", + ...(assignment.workspaceLeaseSlot === undefined + ? ['redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])'] + : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', - 'if #KEYS == 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', + "if #KEYS >= 7 then redis.call('SET', KEYS[7], ARGV[4]) end", + 'if #KEYS == 9 then', + " local epoch = redis.call('GET', KEYS[9])", + " if type(epoch) ~= 'string' then epoch = '0'; redis.call('SET', KEYS[9], epoch, 'EX', ARGV[3]) end", + " if redis.call('PTTL', KEYS[9]) < tonumber(ARGV[3]) * 1000 then redis.call('EXPIRE', KEYS[9], ARGV[3]) end", + " redis.call('HSET', KEYS[8], 'metadata', ARGV[8], 'epoch', epoch)", + " redis.call('EXPIRE', KEYS[8], ARGV[3])", + 'end', 'return 1', ].join('\n'); const keys = [ workerIncarnationKey(assignment.workerId), assignmentKey(assignment.assignmentId), - queueKey(assignment.workerId, assignment.incarnationId), + queueKey( + assignment.workerId, + assignment.incarnationId, + assignment.workspaceLeaseSlot, + ), lockIncarnationKey(assignment.workerId), assignmentDeadlineKey(assignment.assignmentId), workerReadyKey(assignment.workerId), ]; - if (assignment.runtimeSessionId !== undefined) { + if (assignmentWorkspace(assignment) !== undefined) { keys.push( workspaceQuarantineKey( assignment.workerId, - assignment.runtimeSessionId, + assignmentWorkspace(assignment)!, ), ); } + const receipt: AssignmentOwnership = { + assignmentId: assignment.assignmentId, + workerId: assignment.workerId, + incarnationId: assignment.incarnationId, + workspaceFence: assignment.workspaceFence, + workspaceLeaseSlot: assignment.workspaceLeaseSlot, + generation: assignment.generation, + leaseTokenHash: assignment.leaseTokenHash, + workerIdentityId: assignment.workerIdentityId, + expiresAt: assignment.expiresAt, + }; + if (assignment.workspaceLeaseSlot !== undefined) { + keys.push( + workspaceFenceReceiptKey(assignment.assignmentId), + `${workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!)}:epoch`, + ); + } const result = await this.redis.eval( script, keys.length, @@ -1648,6 +2036,7 @@ export class RedisBridgeStore { String(ttlSeconds * 1000), String(Date.parse(assignment.expiresAt)), readyToken ?? '', + JSON.stringify(receipt), ); if (Number(result) === -1) { throw new BridgeStoreError( @@ -1685,7 +2074,7 @@ export class RedisBridgeStore { private async cleanupDispatch( workerId: string, assignmentId: string, - assignment: StoredAssignment | undefined, + assignment: AssignmentOwnership | undefined, ): Promise { await Promise.all([ this.cancel(assignmentId, assignment), @@ -1703,16 +2092,45 @@ export class RedisBridgeStore { assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, ): Promise { + if (assignment.workspaceLeaseSlot !== undefined) { + const committed = Number( + await boundedCommand( + this.redis.eval( + [ + "if redis.call('EXISTS', KEYS[2]) == 0 then return 0 end", + "redis.call('HSET', KEYS[2], 'resultCommitted', '1')", + "if redis.call('HGET', KEYS[2], 'localCleanup') == '1' and redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]) end", + 'return 1', + ].join('\n'), + 2, + workspaceQuarantineKey( + assignment.workerId, + assignment.workspaceFence!, + ), + workspaceFenceReceiptKey(assignment.assignmentId), + assignment.assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge native workspace result commit', + ), + ); + if (committed !== 1) + throw new BridgeStoreError( + 'WORKSPACE_QUARANTINED', + 'Native workspace cleanup ownership expired', + ); + return; + } if ( - assignment.runtimeSessionId === undefined || + assignmentWorkspace(assignment) === undefined || settlement.status !== 'fulfilled' ) { return; } - const runtimeSessionId = assignment.runtimeSessionId; + const runtimeSessionId = assignmentWorkspace(assignment)!; const script = [ - 'if redis.call(\'GET\', KEYS[1]) == ARGV[1] then', - ' return redis.call(\'DEL\', KEYS[1])', + "if redis.call('GET', KEYS[1]) == ARGV[1] then", + " return redis.call('DEL', KEYS[1])", 'end', 'return 0', ].join('\n'); @@ -1721,10 +2139,7 @@ export class RedisBridgeStore { this.redis.eval( script, 1, - workspaceQuarantineKey( - assignment.workerId, - runtimeSessionId, - ), + workspaceQuarantineKey(assignment.workerId, runtimeSessionId), assignment.assignmentId, ), // Once settlement wins, caller cancellation must not prevent its @@ -1744,7 +2159,7 @@ export class RedisBridgeStore { private async cleanupWithRetry( workerId: string, assignmentId: string, - assignment: StoredAssignment | undefined, + assignment: AssignmentOwnership | undefined, ): Promise { let lastError: unknown; for (let attempt = 0; attempt < 3; attempt += 1) { @@ -1759,17 +2174,29 @@ export class RedisBridgeStore { throw lastError; } - private async cleanup(assignment: StoredAssignment): Promise { + private async cleanup(assignment: AssignmentOwnership): Promise { const keys = [ assignmentKey(assignment.assignmentId), - queueKey(assignment.workerId, assignment.incarnationId), - leaseClaimKey(assignment.workerId, assignment.incarnationId), - leaseAckKey(assignment.workerId, assignment.incarnationId), - assignment.runtimeSessionId === undefined + queueKey( + assignment.workerId, + assignment.incarnationId, + assignment.workspaceLeaseSlot, + ), + leaseClaimKey( + assignment.workerId, + assignment.incarnationId, + assignment.workspaceLeaseSlot, + ), + leaseAckKey( + assignment.workerId, + assignment.incarnationId, + assignment.workspaceLeaseSlot, + ), + assignmentWorkspace(assignment) === undefined ? `${assignmentKey(assignment.assignmentId)}:no-workspace` : workspaceQuarantineKey( assignment.workerId, - assignment.runtimeSessionId, + assignmentWorkspace(assignment)!, ), ]; const cleanupScript = [ @@ -1782,6 +2209,7 @@ export class RedisBridgeStore { 'if claimed and not acknowledged then', " redis.call('DEL', KEYS[3], KEYS[4])", 'end', + 'if ARGV[3] == "1" and redis.call(\'GET\', KEYS[5]) == ARGV[1] then return -1 end', 'if queued == 0 and acknowledged and ARGV[2] == "1" and redis.call(\'GET\', KEYS[5]) == ARGV[1] then', ' return -1', 'end', @@ -1798,7 +2226,8 @@ export class RedisBridgeStore { keys.length, ...keys, assignment.assignmentId, - assignment.runtimeSessionId === undefined ? '0' : '1', + assignmentWorkspace(assignment) === undefined ? '0' : '1', + assignment.workspaceLeaseSlot === undefined ? '0' : '1', ), this.redisCommandTimeoutMs, 'Bridge assignment cleanup', @@ -1806,7 +2235,13 @@ export class RedisBridgeStore { ); if (cleanupResult !== -1) { await boundedCommand( - this.releaseLock(assignment.workerId, assignment.assignmentId), + assignment.workspaceLeaseSlot === undefined + ? this.releaseLock(assignment.workerId, assignment.assignmentId) + : new BridgeWorkspaceSlots(this.redis).release( + assignment.workerId, + assignment.incarnationId, + assignment.assignmentId, + ), this.redisCommandTimeoutMs, 'Bridge assignment lock release', ); diff --git a/service/src/config.ts b/service/src/config.ts index 53848103..94daf5a3 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -397,6 +397,10 @@ export const env = { SANDBOX_BACKEND: sandboxBackend, /** Permit trusted callers to route each execution to a paired worker ID. */ BRIDGE_DYNAMIC_WORKERS: process.env.CODEAPI_BRIDGE_DYNAMIC_WORKERS === 'true', + /** Opt-in independent native workspace concurrency; serial by default. */ + BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: Number( + process.env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? 1, + ), /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index d6c9797a..de14b553 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -14,6 +14,7 @@ function render(overrides) { CODEAPI_BRIDGE_DYNAMIC_WORKERS: '', CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', + CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', ...overrides, }, })); @@ -26,6 +27,7 @@ for (const overrides of [ CODEAPI_BRIDGE_TOKEN: token, CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', + CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', }, ]) { const config = render(overrides); @@ -34,6 +36,7 @@ for (const overrides of [ assert.equal(env.CODEAPI_HARDENED_SANDBOX_MODE, 'true'); assert.equal(env.CODEAPI_BRIDGE_AUTH_MODE, 'paired'); assert.equal(env.CODEAPI_BRIDGE_TOKEN, token); + assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); } From 6abed11203c790fd5e46a503891c70bedb26c77c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 9 Sep 2026 09:21:54 -0400 Subject: [PATCH 075/116] =?UTF-8?q?=F0=9F=AB=97=20fix:=20Drain=20Cancelled?= =?UTF-8?q?=20BYOM=20Settlements=20(#172)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: drain BYOM cancellation settlements * fix: validate cancellation cleanup proof * test: reject forged cancellation cleanup proof --- packages/code/src/native-process-child.ts | 4 + packages/code/src/native-process.test.ts | 36 +++++++- packages/code/src/native-process.ts | 5 + packages/code/src/native-sandbox.test.ts | 7 +- packages/code/src/native-sandbox.ts | 4 + packages/code/src/worker.ts | 2 +- packages/code/src/workspace-worker.test.ts | 70 ++++++++++++++ packages/code/src/workspace.ts | 2 + service/src/bridge/store.ts | 50 ++++++++++ service/src/bridge/workspace-store.test.ts | 102 +++++++++++++++++++++ 10 files changed, 278 insertions(+), 4 deletions(-) diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 164147e3..b5221731 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -101,6 +101,10 @@ process.on('message', async (raw: unknown) => { error instanceof WorkspaceToolError ? error.mutationMayHaveCommitted : true, + requiresQuarantine: + error instanceof WorkspaceToolError + ? error.requiresQuarantine + : true, }); } finally { active = undefined; diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index ffb068e4..d77ec559 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -179,13 +179,47 @@ test('executor cancellation targets the active request and preserves mutation ce ok: false, code: 'EXECUTION_ABORTED', mutation: true, + requiresQuarantine: false, }); await assert.rejects( execution, (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EXECUTION_ABORTED' && - error.mutationMayHaveCommitted, + error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + await sandbox.close(); +}); + +test('executor ignores a cleanup exemption on non-cancellation failures', async () => { + let dispatched!: () => void; + const dispatch = new Promise((resolve) => { + dispatched = resolve; + }); + const fake = fixture(() => dispatched()); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + const execution = sandbox.execute(request); + await dispatch; + const command = fake.messages.find((message) => message.type === 'execute')!; + fake.child.emit('message', { + id: command.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + mutation: true, + requiresQuarantine: false, + }); + + await assert.rejects( + execution, + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + error.mutationMayHaveCommitted && + error.requiresQuarantine, ); await sandbox.close(); }); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index fc44e3d6..fe4e1aa8 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -134,6 +134,7 @@ export class NativeProcessWorkspaceCommandSandbox ok?: unknown; result?: unknown; mutation?: unknown; + requiresQuarantine?: unknown; code?: unknown; errorMessage?: unknown; fatal?: unknown; @@ -156,6 +157,9 @@ export class NativeProcessWorkspaceCommandSandbox message.code === 'REGISTRATION_INVALID' ? message.code : 'COMMAND_UNAVAILABLE'; + const processTerminationConfirmed = + code === 'EXECUTION_ABORTED' && + message.requiresQuarantine === false; pending.reject( new WorkspaceToolError( typeof message.errorMessage === 'string' && @@ -164,6 +168,7 @@ export class NativeProcessWorkspaceCommandSandbox : 'Native executor request failed', code, pending.mutation && message.mutation !== false, + pending.mutation && !processTerminationConfirmed, ), ); } diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index a74f3727..e2dc4bda 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1030,7 +1030,8 @@ test('reports cancellation after command start as a potentially committed mutati (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EXECUTION_ABORTED' && - error.mutationMayHaveCommitted === true, + error.mutationMayHaveCommitted === true && + error.requiresQuarantine === false, ); }); @@ -1157,7 +1158,9 @@ test('cleans allocated command state exactly once on every execution exit', asyn error.code === (outcome.startsWith('abort') ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE') && - error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn'), + error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn') && + error.requiresQuarantine === + (outcome === 'abort-after-spawn' && process.platform === 'win32'), ); } assert.equal( diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 49d269c0..139a10d6 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -681,6 +681,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'Workspace command execution aborted', 'EXECUTION_ABORTED', true, + // POSIX commands run in a detached process group, so its + // observed close follows a group-wide SIGKILL. The Windows + // fallback cannot yet prove descendant termination. + this.platform === 'win32', ), ); return; diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 545b96ca..fde84768 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -1562,7 +1562,7 @@ export class BridgeWorker { !( error instanceof WorkspaceToolError && this.options.workspaceTools?.mutationFailuresAreAtomic === true && - !error.mutationMayHaveCommitted + !error.requiresQuarantine )) ) { ambiguousWorkspaceMutationError = error; diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 43c820fe..14bf2d61 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1410,6 +1410,76 @@ test('worker clears quarantine after a composed command is cleanly rejected', as assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); }); +test('worker clears quarantine after a command cancellation confirms process termination', async () => { + const lifecycle: string[] = []; + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-command-cancelled-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30', + }, + }); + assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); +}); + test('worker retains quarantine when an atomic executor cannot confirm durability', async () => { const lifecycle: string[] = []; const workspaceCapabilities = { diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index f7770e76..5bfc07fe 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -176,6 +176,8 @@ export class WorkspaceToolError extends Error { message: string, public readonly code: WorkspaceToolErrorCode, public readonly mutationMayHaveCommitted = false, + /** Retain the durable mutation guard when process or write settlement is uncertain. */ + public readonly requiresQuarantine = mutationMayHaveCommitted, ) { super(message); this.name = 'WorkspaceToolError'; diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 41e11839..32205571 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -23,6 +23,7 @@ import { BridgeWorkspaceSlots } from './slots'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; +const CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; const DEFAULT_WORKER_TTL_SECONDS = 60; const DEFAULT_REDIS_COMMAND_TIMEOUT_MS = 1_000; @@ -1901,6 +1902,55 @@ export class RedisBridgeStore { // settlement before returning an error, even when the caller is gone. pollError = error; } + const workspaceRequest = + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ? assignment.request + : undefined; + const cancelledMutation = + signal.aborted && + workspaceRequest != null && + (workspaceRequest.operation === 'write_file' || + workspaceRequest.operation === 'edit_file' || + workspaceRequest.operation === 'execute_command'); + if (cancelledMutation) { + try { + // Keep the acknowledged assignment available long enough for the + // worker to terminate its process tree and commit a clean rejection. + // Closing it first makes that rejection impossible to acknowledge and + // leaves the worker's durable mutation guard armed. + await this.cancel(assignment.assignmentId, assignment); + // Rejected settlements remain valid after the execution deadline. + // Give Stop its own grace so a near-timeout cancellation is not + // misclassified as an ambiguous timeout. + const cancellationDeadlineAtMs = + Date.now() + CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; + let cancellationPollMs = POLL_INTERVAL_MS; + while (Date.now() < cancellationDeadlineAtMs) { + const raw = await boundedCommand( + this.redis.get(settlementKey(assignment.assignmentId)), + Math.max( + 1, + Math.min( + this.redisCommandTimeoutMs, + cancellationDeadlineAtMs - Date.now(), + ), + ), + 'Bridge cancelled workspace settlement poll', + ); + if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement; + await delay( + Math.min( + cancellationPollMs, + Math.max(0, cancellationDeadlineAtMs - Date.now()), + ), + ); + cancellationPollMs = Math.min(cancellationPollMs * 2, 500); + } + } catch (error) { + pollError ??= error; + } + } const closeKeys = [ assignmentKey(assignment.assignmentId), settlementKey(assignment.assignmentId), diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index a23274d8..5b0186d0 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -77,6 +77,108 @@ test('dispatches a workspace tool only to a worker advertising its workspace and }); }); +test('drains an acknowledged workspace mutation cancellation before releasing it', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const controller = new AbortController(); + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30; touch delayed.txt', + }, + deadlineAtMs: Date.now() + 5_000, + executionTimeoutMs: 500, + signal: controller.signal, + }); + const assignment = (await store.lease( + 'workspace-worker', + incarnationId, + 1_000, + ))!; + await store.acknowledgeLease( + 'workspace-worker', + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + + await new Promise((resolve) => setTimeout(resolve, 250)); + controller.abort(); + for ( + let attempt = 0; + attempt < 100 && + !(await store.cancelled( + 'workspace-worker', + incarnationId, + assignment.assignmentId, + )); + attempt += 1 + ) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await new Promise((resolve) => setTimeout(resolve, 350)); + expect(Date.now()).toBeGreaterThan(Date.parse(assignment.expiresAt)); + await store.settle('workspace-worker', assignment.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + incarnationId, + status: 'rejected', + errorCode: 'EXECUTION_ABORTED', + error: 'Workspace command execution aborted', + }); + + await expect(completion).resolves.toMatchObject({ + status: 'rejected', + errorCode: 'EXECUTION_ABORTED', + }); + + const reuse = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId: 'primary', + command: 'printf reused', + }, + deadlineAtMs: Date.now() + 5_000, + executionTimeoutMs: 5_000, + signal: new AbortController().signal, + }); + const next = (await store.lease( + 'workspace-worker', + incarnationId, + 1_000, + ))!; + expect(next.request).toMatchObject({ command: 'printf reused' }); + await store.settle('workspace-worker', next.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: next.generation, + leaseToken: next.leaseToken, + incarnationId, + status: 'rejected', + error: 'fixture completion', + }); + await expect(reuse).resolves.toMatchObject({ status: 'rejected' }); +}); + test('rejects a workspace tool that the selected worker did not advertise', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, From 9cbff2621af3304587c9af286483b4a576edfc83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:05:01 -0400 Subject: [PATCH 076/116] fix(codeapi): require bridge credentials only when configured (#171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(codeapi): require bridge credentials only when configured Source: ClickHouse/ai@da81d707c6538b509be70e2401b649693a3f3dce * chore(codeapi): import 🫗 fix: Drain Cancelled BYOM Settlements Source: ClickHouse/ai@2391d77aab6ff81689cf8a834d74992399f41173 Co-authored-by: danny-avila <110412045+danny-avila@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: danny-avila <110412045+danny-avila@users.noreply.github.com> --- README.md | 6 ++++-- service/src/bridge/enabled.ts | 9 +++++++++ service/src/bridge/index.ts | 2 ++ service/src/bridge/router.test.ts | 17 +++++++++++++++++ service/src/bridge/router.ts | 2 ++ service/src/secure-startup.test.ts | 18 ++++++++++++++++++ service/src/secure-startup.ts | 4 +++- 7 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 service/src/bridge/enabled.ts diff --git a/README.md b/README.md index 7bff76fb..bad5b66e 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,10 @@ cut. Copy `.env.example` to `.env` and set `CODEAPI_BRIDGE_TOKEN` to a private value of at least 32 bytes (generate one with `openssl rand -hex 32`). The API exposes -bridge routes even with the default HTTP sandbox backend, so hardened mode -requires this enrollment credential. Compose defaults to +bridge routes when configured through the remote-bridge backend, paired auth, +dynamic workers, or a bridge token. Hardened deployments with none of these +configured leave bridge routes disabled and do not require a bridge token. +Enabled bridges still require this enrollment credential. Compose defaults to `CODEAPI_BRIDGE_AUTH_MODE=paired` and `CODEAPI_BRIDGE_DYNAMIC_WORKERS=true`. To restrict pairing to a fixed worker, set `CODEAPI_BRIDGE_DYNAMIC_WORKERS=false` and `CODEAPI_BRIDGE_WORKER_ID` to its ID. Keep the token outside workspaces and diff --git a/service/src/bridge/enabled.ts b/service/src/bridge/enabled.ts new file mode 100644 index 00000000..540a844a --- /dev/null +++ b/service/src/bridge/enabled.ts @@ -0,0 +1,9 @@ +import { env } from '../config'; + +/** API-only deployments can serve bridges without selecting that worker backend. */ +export function isBridgeEnabled(): boolean { + return env.SANDBOX_BACKEND === 'remote-bridge' + || env.BRIDGE_AUTH_MODE === 'paired' + || env.BRIDGE_DYNAMIC_WORKERS + || env.BRIDGE_TOKEN.length > 0; +} diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index fc409ad2..a1ec5d78 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -3,6 +3,7 @@ import { env } from '../config'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; +import { isBridgeEnabled } from './enabled'; export const bridgeStore = new RedisBridgeStore( connection, @@ -13,6 +14,7 @@ export const bridgeStore = new RedisBridgeStore( export const bridgePairings = new RedisBridgePairingStore(connection); export default createBridgeRouter({ + enabled: isBridgeEnabled(), store: bridgeStore, pairings: bridgePairings, authMode: env.BRIDGE_AUTH_MODE, diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 764613a5..af0e765c 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -25,6 +25,23 @@ afterEach(async () => { }); describe('paired bridge HTTP API', () => { + test('disabled bridges expose no HTTP routes', async () => { + const app = express(); + app.use('/v1/bridge', createBridgeRouter({ + enabled: false, + store: new RedisBridgeStore(redis), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: '', + })); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/test/status`); + expect(response.status).toBe(404); + }); + test('reports authenticated worker readiness without exposing identity or binding data', async () => { const store = new RedisBridgeStore(redis); const app = express(); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 72e51b4d..25b0bdaf 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -29,6 +29,7 @@ const PRINCIPAL_TYPES = new Set([ export type BridgeAuthMode = 'static' | 'paired'; export interface BridgeRouterOptions { + enabled?: boolean; store: RedisBridgeStore; pairings: RedisBridgePairingStore; authMode: BridgeAuthMode; @@ -130,6 +131,7 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement { export function createBridgeRouter(options: BridgeRouterOptions): Router { const router = Router(); + if (options.enabled === false) return router; const configuredWorker = (workerId: string): boolean => options.allowDynamicWorkers === true || diff --git a/service/src/secure-startup.test.ts b/service/src/secure-startup.test.ts index 6820010a..79d602df 100644 --- a/service/src/secure-startup.test.ts +++ b/service/src/secure-startup.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { env } from './config'; +import { isBridgeEnabled } from './bridge/enabled'; import { validateApiBridgePolicy, validateApiHardenedConfig, @@ -394,6 +395,23 @@ describe('sandbox backend policy', () => { expect(() => validateSandboxBackendPolicy()).not.toThrow(); }); + test('hardened HTTP and Lambda APIs start without an unused bridge credential', () => { + env.HARDENED_SANDBOX_MODE = true; + env.BRIDGE_AUTH_MODE = 'static'; + env.BRIDGE_DYNAMIC_WORKERS = false; + env.BRIDGE_TOKEN = ''; + env.BRIDGE_WORKER_ID = ''; + for (const backend of ['http', 'lambda-microvm'] as const) { + env.SANDBOX_BACKEND = backend; + expect(isBridgeEnabled()).toBe(false); + expect(() => validateApiBridgePolicy()).not.toThrow(); + } + env.BRIDGE_AUTH_MODE = 'paired'; + env.BRIDGE_DYNAMIC_WORKERS = true; + expect(isBridgeEnabled()).toBe(true); + expect(() => validateApiBridgePolicy()).toThrow('CODEAPI_BRIDGE_TOKEN'); + }); + test('API-only hardened bridge validation rejects static worker auth', () => { env.SANDBOX_BACKEND = 'http'; env.HARDENED_SANDBOX_MODE = true; diff --git a/service/src/secure-startup.ts b/service/src/secure-startup.ts index 79f69c42..26d378e3 100644 --- a/service/src/secure-startup.ts +++ b/service/src/secure-startup.ts @@ -5,6 +5,7 @@ import { } from './config'; import { INTERNAL_SERVICE_TOKEN_ENV } from './internal-service-auth'; import { isValidBridgeWorkerId } from '../../packages/code/src/protocol'; +import { isBridgeEnabled } from './bridge/enabled'; export class SecureStartupConfigError extends Error { constructor(message: string) { @@ -56,6 +57,7 @@ export function validateApiHardenedConfig(): void { /** Validate bridge credentials in every process that exposes bridge routes. */ export function validateApiBridgePolicy(): void { + if (!isBridgeEnabled()) return; if (env.BRIDGE_TOKEN !== env.BRIDGE_TOKEN.trim()) { throw new SecureStartupConfigError( 'CODEAPI_BRIDGE_TOKEN must not contain surrounding whitespace', @@ -87,7 +89,7 @@ export function validateApiBridgePolicy(): void { requireStrongSecret('CODEAPI_BRIDGE_TOKEN', env.BRIDGE_TOKEN); if (env.BRIDGE_AUTH_MODE !== 'paired') { throw new SecureStartupConfigError( - 'Hardened API deployments require CODEAPI_BRIDGE_AUTH_MODE=paired because bridge routes are always exposed', + 'Hardened API deployments with bridge routes enabled require CODEAPI_BRIDGE_AUTH_MODE=paired', ); } } From 119875979e26ea0f3be028312e2fe4c0fbe26528 Mon Sep 17 00:00:00 2001 From: Mihidum <55163074+mihidumh@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:26:17 +1000 Subject: [PATCH 077/116] fix(api): stream uploads to the file-server with fetch; bump Bun to 1.4.2 (fixes silent upload truncation) (#174) * fix(api): stream uploads to the file-server with fetch, not axios over node:http On Bun, node:http's ClientRequest can drop the tail of a chunked request body: write() accepts every byte and end() is called after the last write, yet the peer receives 32 KiB-800 KiB less. The file-server then stores a short object and reports success, so a 20 MiB upload comes back corrupt with no error anywhere (Bun 1.3.10-1.3.14; 1 MiB is unaffected). Send the busboy file part with the global fetch and a web ReadableStream instead. Bun's native fetch and Node's undici stream the body intact. Co-Authored-By: Claude Fable 5.1 * build: bump Bun base images 1.3.14 -> 1.4.2 Bun 1.3.x's node:http client drops the tail of chunked request bodies under load (see the previous commit). Bun 1.4.2 streams them intact in the same test, so the base image bump is the second line of defence for every remaining node:http client in the services. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- service/Dockerfile | 10 ++-- service/Dockerfile.api | 4 +- service/Dockerfile.bun | 2 +- service/Dockerfile.egress-gateway | 4 +- service/Dockerfile.local | 4 +- service/Dockerfile.service | 2 +- service/Dockerfile.tool-call-server | 4 +- service/Dockerfile.worker | 4 +- service/src/service/router.ts | 73 ++++++++++++++++++++--------- 9 files changed, 68 insertions(+), 39 deletions(-) diff --git a/service/Dockerfile b/service/Dockerfile index 00680111..d7bf9036 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -1,5 +1,5 @@ # File Server Dockerfile -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /app # Install dependencies @@ -27,7 +27,7 @@ RUN bun build ./src/worker-server.ts --minify --outdir .build-worker --target bu RUN bun build ./src/egress-gateway.ts --minify --outdir .build-egress-gateway --target bun --external '@opentelemetry/*' # File server production -FROM oven/bun:1.3.14 AS production +FROM oven/bun:1.4.2 AS production ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules @@ -35,7 +35,7 @@ COPY --from=builder /app/.build ./.build CMD ["bun", "run", ".build/file-server.js"] # API server (HTTP on port 3112) -FROM oven/bun:1.3.14 AS api +FROM oven/bun:1.4.2 AS api ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules @@ -45,7 +45,7 @@ COPY --from=builder /app/src/*.py ./src/ CMD ["bun", "run", ".build-api/api-server.js"] # Worker server (job processor, health on port 3113) -FROM oven/bun:1.3.14 AS worker +FROM oven/bun:1.4.2 AS worker ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules @@ -54,7 +54,7 @@ COPY --from=builder /app/src/*.py ./src/ CMD ["bun", "run", ".build-worker/worker-server.js"] # Egress gateway (sandbox outbound delegation) -FROM oven/bun:1.3.14 AS egress-gateway +FROM oven/bun:1.4.2 AS egress-gateway ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules diff --git a/service/Dockerfile.api b/service/Dockerfile.api index 2921401e..1da68c46 100644 --- a/service/Dockerfile.api +++ b/service/Dockerfile.api @@ -1,7 +1,7 @@ # API-Only Server Dockerfile # This builds the HTTP API server without workers # Scale this based on HTTP traffic -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /app # Install dependencies @@ -27,7 +27,7 @@ RUN bun build ./src/api-server.ts --minify --outdir .build --target bun --extern RUN bun build ./scripts/rehydrate-session-cache.ts --minify --outdir .build-migrations --target bun --external '@opentelemetry/*' # Production stage -FROM oven/bun:1.3.14 AS production +FROM oven/bun:1.4.2 AS production ENV NODE_ENV=production WORKDIR /app # Install curl for healthcheck (not included in bun base image) diff --git a/service/Dockerfile.bun b/service/Dockerfile.bun index b416f14e..d545f32c 100644 --- a/service/Dockerfile.bun +++ b/service/Dockerfile.bun @@ -1,5 +1,5 @@ # Base stage -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /usr/src/app # Install dependencies diff --git a/service/Dockerfile.egress-gateway b/service/Dockerfile.egress-gateway index ca9e9a9d..bd1d2b64 100644 --- a/service/Dockerfile.egress-gateway +++ b/service/Dockerfile.egress-gateway @@ -1,5 +1,5 @@ # Egress Gateway Dockerfile -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /app FROM base AS install @@ -11,7 +11,7 @@ RUN mkdir -p /temp/prod COPY service/package.json service/bun.lock /temp/prod/ RUN cd /temp/prod && bun install --frozen-lockfile --production -FROM oven/bun:1.3.14 AS production +FROM oven/bun:1.4.2 AS production ENV NODE_ENV=production WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* diff --git a/service/Dockerfile.local b/service/Dockerfile.local index cbb7af13..4ed93ee4 100644 --- a/service/Dockerfile.local +++ b/service/Dockerfile.local @@ -1,5 +1,5 @@ # Local development Dockerfile - no authentication required -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /app # Install dependencies @@ -22,7 +22,7 @@ COPY service/tsconfig.json ./ RUN bun build ./src/local-api.ts --minify --outdir .build --target bun --external '@opentelemetry/*' # Production stage -FROM oven/bun:1.3.14 AS production +FROM oven/bun:1.4.2 AS production ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules diff --git a/service/Dockerfile.service b/service/Dockerfile.service index 3a89dc15..d5d4e6d9 100644 --- a/service/Dockerfile.service +++ b/service/Dockerfile.service @@ -1,5 +1,5 @@ # Service API Dockerfile -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /usr/src/app # Install dependencies diff --git a/service/Dockerfile.tool-call-server b/service/Dockerfile.tool-call-server index 355e8ec1..76afb837 100644 --- a/service/Dockerfile.tool-call-server +++ b/service/Dockerfile.tool-call-server @@ -1,5 +1,5 @@ # Tool Call Server Dockerfile -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /app # Install dependencies @@ -13,7 +13,7 @@ COPY service/package.json service/bun.lock /temp/prod/ RUN cd /temp/prod && bun install --frozen-lockfile --production # Production stage -FROM oven/bun:1.3.14 AS production +FROM oven/bun:1.4.2 AS production ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules diff --git a/service/Dockerfile.worker b/service/Dockerfile.worker index e99c16c4..bb7b6945 100644 --- a/service/Dockerfile.worker +++ b/service/Dockerfile.worker @@ -2,7 +2,7 @@ # This builds the job processing worker without HTTP server # Deploy alongside a sandbox sidecar for execution # Scale this based on queue depth -FROM oven/bun:1.3.14 AS base +FROM oven/bun:1.4.2 AS base WORKDIR /app # Install dependencies @@ -25,7 +25,7 @@ COPY service/tsconfig.json ./ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' # Production stage -FROM oven/bun:1.3.14 AS production +FROM oven/bun:1.4.2 AS production ENV NODE_ENV=production WORKDIR /app # Install curl for healthcheck diff --git a/service/src/service/router.ts b/service/src/service/router.ts index f89bbb17..2c42f60c 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -3,7 +3,7 @@ import busboy from 'busboy'; import { nanoid } from 'nanoid'; import { Router } from 'express'; import type { Response } from 'express'; -import type { Readable } from 'stream'; +import { Readable } from 'stream'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { sessionAuth } from '../middleware/auth'; @@ -42,6 +42,43 @@ const JOB_COMPLETION_WAIT_TIMEOUT_MS = jobCompletionWaitTimeoutMs( ); const UPLOAD_TIMEOUT_MS = 30_000; + +/** + * Streams one busboy file part to the file-server. + * + * Uses the global `fetch` rather than axios on purpose. axios routes a + * stream body through `node:http`'s `ClientRequest`, and on Bun (the + * runtime in `Dockerfile.api`) that client can drop the tail of a + * chunked request body: every byte is accepted by `write()`, `end()` + * is called after the last write, yet the peer receives 32 KiB-800 KiB + * less and the file-server stores a short object while reporting + * success (reproduced on Bun 1.3.10-1.3.14 with a 20 MiB upload; a + * 1 MiB upload is unaffected). Bun's native `fetch` and Node's undici + * stream the same body intact. busboy's `limits.fileSize` already caps + * the part, so no separate body-length guard is needed here. + */ +async function putFileToFileServer( + url: string, + file: Readable, + headers: Record, + signal: AbortSignal, +): Promise { + const response = await fetch(url, { + method: 'PUT', + headers, + body: Readable.toWeb(file) as unknown as ReadableStream, + signal, + /* Required by the WHATWG fetch spec for streamed request bodies. */ + duplex: 'half', + } as RequestInit); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error( + `file-server responded ${response.status}${detail ? `: ${detail.slice(0, 200)}` : ''}`, + ); + } + return (await response.json()) as t.UploadResult; +} /* Batch cap sized for skill-priming uploads: a single skill (e.g. pptx) * can carry 60+ resource files including .xsd schemas, helper scripts, * docs, and Python __init__.py markers. The previous cap of 20 silently @@ -500,20 +537,16 @@ router.post('/upload', uploadLimiter, async (req: t.AuthenticatedRequest, res: R recordSessionOwnership(connection, session_id, sessionKey) .then(() => { logger.info(`[${INSTANCE_ID}] Upload: Session ID: ${session_id} | User ID: ${userId} | Session key: ${sessionKey}`); - return axios.put( - `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, - file, - { - headers: internalServiceHeaders(putHeaders), - maxBodyLength: planFileSize, - maxContentLength: planFileSize, - signal: abortController.signal, - }, - ); + return putFileToFileServer( + `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, + file, + internalServiceHeaders(putHeaders), + abortController.signal, + ); }) - .then(response => { + .then(result => { clearTimeout(uploadTimeout); - resolve(response.data); + resolve(result); }) .catch(error => { clearTimeout(uploadTimeout); @@ -742,18 +775,14 @@ router.post('/upload/batch', uploadLimiter, async (req: t.AuthenticatedRequest, logger.error(`[${INSTANCE_ID}] Batch upload file failed: ${filename} | Session: ${session_id}`, { error: message }); resolve({ status: 'error', filename, error: message }); }; - const forwardFile = (): Promise => axios.put( + const forwardFile = (): Promise => putFileToFileServer( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, file, - { - headers: internalServiceHeaders(putHeaders), - maxBodyLength: planFileSize, - maxContentLength: planFileSize, - signal: abortController.signal, - }, - ).then(response => { + internalServiceHeaders(putHeaders), + abortController.signal, + ).then(result => { clearTimeout(uploadTimeout); - resolve({ status: 'success', filename: response.data.filename, fileId: response.data.fileId }); + resolve({ status: 'success', filename: result.filename, fileId: result.fileId }); }, resolveUploadFailure); void ensureSessionRegistered(sessionKey) From 3142a2f6541641488d572da453efba728a6c0539 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 11:10:37 -0400 Subject: [PATCH 078/116] fix: fail denied input downloads once per batch (#177) --- api/src/download.test.ts | 70 +++++++++++++++++++++++++++++++++++++++- api/src/job.ts | 40 +++++++++++++++++++---- 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 5372a40c..2c7248e8 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from 'bun:test'; +import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, spyOn } from 'bun:test'; import * as fsp from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; @@ -439,6 +439,74 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(contents).toBe('hi'); }); + it.each([401, 403])('does not retry an HTTP %i authorization denial', async status => { + const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; + let requests = 0; + routes.set('/sessions/previous/objects/denied', { + status, + onRequest: () => { requests++; }, + }); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + await expect(job.downloadAndWriteFile(file, 5, 1)).rejects.toThrow(`HTTP error: ${status}`); + expect(requests).toBe(1); + expect(await fsp.readdir(tmpDir)).toEqual([]); + }); + + it.each([404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + const file: TFile = { id: 'transient', storage_session_id: 'previous', name: 'ready.txt' }; + let requests = 0; + const route: Route = { + status, + body: 'ready', + onRequest: () => { if (++requests === 2) route.status = 200; }, + }; + routes.set('/sessions/previous/objects/transient', route); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + await expect(job.downloadAndWriteFile(file, 5, 1)).resolves.toBe('ready.txt'); + expect(requests).toBe(2); + expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); + }); + + it('accounts for a denied 240-file batch once and stops queued downloads', async () => { + const files: TFile[] = Array.from({ length: 240 }, (_, index) => ({ + id: `file-${index}`, storage_session_id: 'previous', name: `file-${index}.txt`, + })); + let requests = 0; + routes.set('/sessions/previous/objects', { status: 200, body: '[]' }); + for (const file of files) { + routes.set(`/sessions/previous/objects/${file.id}`, { + status: 403, + delayMs: file.id === 'file-0' ? 0 : 30, + onRequest: () => { requests++; }, + }); + } + let dirty = false; + const job = makeJob(files, sessionWorkspaceAt(tmpDir, 'batch-test', () => { dirty = true; })); + const log = (job as unknown as { log: import('pino').Logger }).log; + const errorLog = spyOn(log, 'error'); + const originalConcurrency = config.prime_concurrency; + config.prime_concurrency = 8; + try { + await expect(job.prime()).rejects.toBeInstanceOf(SessionWorkspaceDirtyError); + expect(dirty).toBe(true); + expect(requests).toBeGreaterThan(0); + expect(requests).toBeLessThanOrEqual(8); + expect(errorLog).toHaveBeenCalledTimes(1); + expect(errorLog).toHaveBeenCalledWith(expect.objectContaining({ + inputCount: 240, completed: 0, failed: 1, cancelled: 7, notStarted: 232, + }), 'Input preparation batch failed'); + expect(await fsp.readdir(tmpDir)).toEqual([]); + } finally { + errorLog.mockRestore(); + config.prime_concurrency = originalConcurrency; + await job.cleanup(); + } + }); + it('fails when the server keeps 404-ing past the retry cap (no phantom write)', async () => { const file: TFile = { id: 'missing-id', diff --git a/api/src/job.ts b/api/src/job.ts index 610b748b..8121f51d 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -60,6 +60,14 @@ export { const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; const AUTO_LOAD_DIRKEEP_RETRIES = 2; +/** Replaying the same sealed grant cannot repair an authorization denial. */ +class InputAuthorizationError extends Error { + constructor(status: number) { + super(`HTTP error: ${status}`); + this.name = 'InputAuthorizationError'; + } +} + /** * Bridges a `fetch` response body to a Node-stream Readable. The types at the * module boundary (Node's `stream/web` vs. lib.dom) don't overlap cleanly, @@ -993,11 +1001,18 @@ export class Job { submissionDir: this.submissionDir, identity: this.jobIdentity, }; + const startedAt = performance.now(); + let started = 0; + let completed = 0; + let cancelled = 0; let firstFailure: { error: unknown } | undefined; const runFileOperation = async (operation: () => Promise): Promise => { + started++; try { await operation(); + completed++; } catch (error) { + if (firstFailure) cancelled++; if (!firstFailure) { firstFailure = { error }; controller.abort(error); @@ -1031,6 +1046,15 @@ export class Job { Array.from({ length: workerCount }, () => runPrimeWorker()), ); if (firstFailure) { + this.log.error({ + inputCount: fileOps.length, + completed, + failed: 1, + cancelled, + notStarted: fileOps.length - started, + durationMs: Math.round(performance.now() - startedAt), + err: firstFailure.error, + }, 'Input preparation batch failed'); if (this.session) { /* A sibling may already have atomically replaced its destination. The * workspace now matches neither the previous checkpoint nor the full @@ -1302,6 +1326,9 @@ export class Job { if (!response.ok) { await response.body?.cancel().catch(() => {}); + if (response.status === 401 || response.status === 403) { + throw new InputAuthorizationError(response.status); + } throw new Error(`HTTP error: ${response.status}`); } @@ -1366,11 +1393,10 @@ export class Job { try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw abortReason(operation.signal); } - /* ValidationError is deterministic — a bad Content-Disposition - * filename will fail identically on every retry. Abort fast - * (cleanup + rethrow) instead of burning ~7.5s on exponential - * backoff and surfacing the error as a generic download failure. */ - if (error instanceof ValidationError) { + /* Invalid filenames and authorization denials cannot recover by + * replaying the same request. Abort the batch before exponential + * backoff amplifies the failure across its remaining files. */ + if (error instanceof ValidationError || error instanceof InputAuthorizationError) { try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw error; } @@ -1383,7 +1409,9 @@ export class Job { } } - this.log.error({ fileId: file.id, maxRetries, err: lastError }, 'Failed to download file'); + if (!context?.signal) { + this.log.error({ fileId: file.id, maxRetries, err: lastError }, 'Failed to download file'); + } try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw lastError ?? new Error(`Failed to download input ${file.id}`); } From 794df9e444acc2a8a2c6a0fbb65ddc0976c8ef41 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 15:47:10 -0400 Subject: [PATCH 079/116] fix: preserve retries for transient egress ledger conflicts (#179) * fix: distinguish retryable ledger contention from scope denials * fix: preserve error classification through marker discovery and relay * test: exercise classified denials through gateway configuration * fix: honor bounded gateway retry hints during object downloads --- api/src/download.test.ts | 79 +++++++++++++++++++++++++++++- api/src/egress.ts | 1 + api/src/job.ts | 23 +++++++-- packages/code/src/relay.test.ts | 35 +++++++++++++ packages/code/src/relay.ts | 6 +++ service/src/egress-gateway.test.ts | 36 +++++++++++++- service/src/egress-gateway.ts | 4 ++ service/src/egress-grant.ts | 4 +- service/src/egress-ledger.ts | 2 +- 9 files changed, 182 insertions(+), 8 deletions(-) diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 2c7248e8..bb8ec7e9 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -55,6 +55,7 @@ function makeRuntime(): Runtime { function makeJob(files: TFile[] = [], session?: SessionWorkspace): Job { return new Job({ session_id: 'test-session', + egress_grant: 'test-grant', runtime: makeRuntime(), files, args: [], @@ -440,10 +441,12 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { }); it.each([401, 403])('does not retry an HTTP %i authorization denial', async status => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; let requests = 0; routes.set('/sessions/previous/objects/denied', { status, + headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' }, onRequest: () => { requests++; }, }); const job = makeJob([file]); @@ -454,7 +457,8 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.readdir(tmpDir)).toEqual([]); }); - it.each([404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + it.each([403, 404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const file: TFile = { id: 'transient', storage_session_id: 'previous', name: 'ready.txt' }; let requests = 0; const route: Route = { @@ -471,7 +475,79 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); }); + it.each([false, true])('honors conflict retry hints with cancellation=%s', async cancel => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; + const controller = new AbortController(); + const timestamps: number[] = []; + let timer: ReturnType | undefined; + const route: Route = { + status: 503, body: 'ready', + headers: { 'X-CodeAPI-Error-Code': 'ledger_conflict', 'Retry-After': '1' }, + onRequest: () => { + timestamps.push(performance.now()); + if (timestamps.length === 2) route.status = 200; + else if (cancel) timer = setTimeout(() => controller.abort(new Error('cancelled retry')), 25); + }, + }; + routes.set('/sessions/previous/objects/retry-hint', route); + const file: TFile = { id: 'retry-hint', storage_session_id: 'previous', name: 'ready.txt' }; + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + try { + const result = job.downloadAndWriteFile(file, 5, 1, { + submissionDir: tmpDir, identity: fallbackSandboxIdentity(), signal: controller.signal, + }); + if (cancel) { + await expect(result).rejects.toThrow('cancelled retry'); + expect(timestamps).toHaveLength(1); + expect(await fsp.readdir(tmpDir)).toEqual([]); + } else { + await expect(result).resolves.toBe('ready.txt'); + expect(timestamps).toHaveLength(2); + expect(timestamps[1] - timestamps[0]).toBeGreaterThanOrEqual(900); + } + } finally { + clearTimeout(timer); + } + }); + + it('does not retry an unclassified direct file-server denial', async () => { + config.egress_gateway_url = ''; + let requests = 0; + const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; + routes.set('/sessions/previous/objects/denied', { + status: 403, onRequest: () => { requests++; }, + }); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + await expect(job.downloadAndWriteFile(file, 5, 1)).rejects.toThrow('HTTP error: 403'); + expect(requests).toBe(1); + }); + + it.each(['legacy', 'classified', 'direct'])('handles %s marker denials before priming', async mode => { + config.egress_gateway_url = mode === 'direct' ? '' : `http://127.0.0.1:${serverPort}`; + let requests = 0; + const route: Route = { + status: 403, body: '[]', + headers: mode === 'classified' ? { 'X-CodeAPI-Error-Code': 'scope_mismatch' } : {}, + onRequest: () => { if (++requests === 2) route.status = 200; }, + }; + routes.set('/sessions/previous/objects', route); + const file: TFile = { id: 'ready', storage_session_id: 'previous', name: 'ready.txt' }; + routes.set('/sessions/previous/objects/ready', { status: 200, body: 'ready' }); + const job = makeJob([file], sessionWorkspaceAt(tmpDir, 'marker-retry')); + if (mode === 'legacy') { + await job.prime(); + expect(requests).toBe(2); + expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); + } else { + await expect(job.prime()).rejects.toThrow('HTTP error loading .dirkeep markers: 403'); + expect(requests).toBe(1); + } + }); + it('accounts for a denied 240-file batch once and stops queued downloads', async () => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const files: TFile[] = Array.from({ length: 240 }, (_, index) => ({ id: `file-${index}`, storage_session_id: 'previous', name: `file-${index}.txt`, })); @@ -480,6 +556,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { for (const file of files) { routes.set(`/sessions/previous/objects/${file.id}`, { status: 403, + headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' }, delayMs: file.id === 'file-0' ? 0 : 30, onRequest: () => { requests++; }, }); diff --git a/api/src/egress.ts b/api/src/egress.ts index 442872ab..69ecf6db 100644 --- a/api/src/egress.ts +++ b/api/src/egress.ts @@ -1 +1,2 @@ export const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +export const EGRESS_ERROR_CODE_HEADER = 'X-CodeAPI-Error-Code'; diff --git a/api/src/job.ts b/api/src/job.ts index 8121f51d..bdcd5061 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -15,7 +15,7 @@ import { getRuntimes } from './runtime'; import { execute } from './nsjail'; import { config } from './config'; import { internalServiceHeaders } from './internal-service-auth'; -import { EGRESS_GRANT_HEADER } from './egress'; +import { EGRESS_GRANT_HEADER, EGRESS_ERROR_CODE_HEADER } from './egress'; import { injectTraceHeaders } from './telemetry'; import { applyReadOnlyInputPermissions, @@ -1196,6 +1196,11 @@ export class Job { } } + private isLegacyGatewayDenial(response: Response): boolean { + return !!config.egress_gateway_url && response.status === 403 && + !response.headers.has(EGRESS_ERROR_CODE_HEADER); + } + /** * Fetches normalized objects for one inherited session and returns the * `.dirkeep` markers belonging to exactly that session. Guards against: @@ -1219,7 +1224,7 @@ export class Job { signal: controller.signal, }, ); - if (res.status === 503 && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { + if ((res.status === 503 || this.isLegacyGatewayDenial(res)) && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { await res.body?.cancel().catch(() => {}); const retryAfterSeconds = Number(res.headers.get('retry-after')); await sleep( @@ -1326,7 +1331,10 @@ export class Job { if (!response.ok) { await response.body?.cancel().catch(() => {}); - if (response.status === 401 || response.status === 403) { + /* Older gateways also used 403 for transient ledger contention. + * Only classify 403 as permanent when the gateway distinguishes it. */ + if (response.status === 401 || + (response.status === 403 && !this.isLegacyGatewayDenial(response))) { throw new InputAuthorizationError(response.status); } throw new Error(`HTTP error: ${response.status}`); @@ -1402,7 +1410,14 @@ export class Job { } lastError = error instanceof Error ? error : new Error(String(error)); if (attempt < maxRetries) { - const delay = retryDelay * Math.pow(2, attempt - 1); + const backoff = retryDelay * Math.pow(2, attempt - 1); + const retryAfterSeconds = response?.status === 503 + ? Number(response.headers.get('retry-after')) : NaN; + /* Use the same bounded retry hint as marker discovery, without + * shortening exponential backoff or bypassing batch cancellation. */ + const delay = Number.isFinite(retryAfterSeconds) + ? Math.max(backoff, Math.min(1000, Math.max(0, retryAfterSeconds * 1000))) + : backoff; this.log.warn({ fileId: file.id, attempt, maxRetries, delay, err: lastError }, 'Download failed, retrying'); await sleep(delay, operation.signal); } diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts index c0814408..a9b81fd0 100644 --- a/packages/code/src/relay.test.ts +++ b/packages/code/src/relay.test.ts @@ -381,3 +381,38 @@ test('file relay rejects plaintext remote upstreams', async () => { /HTTPS unless it is a local development host/, ); }); + +for (const [status, reason] of [[403, 'scope_mismatch'], [503, 'ledger_conflict']] as const) { + test(`file relay preserves ${reason} classification`, async () => { + const upstream = createServer((_req, res) => { + res.writeHead(status, { + 'X-CodeAPI-Error-Code': reason, + 'Retry-After': '1', + 'X-Internal-Secret': 'must-not-forward', + }).end('rejected'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', + maxBytes: 1024, timeoutMs: 1000, + }); + try { + const response = await fetch(`${relay.url}/sessions/storage-1/objects/file-1`, { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }); + assert.equal(response.status, status); + assert.equal(response.headers.get('x-codeapi-error-code'), reason); + assert.equal(response.headers.get('retry-after'), '1'); + assert.equal(response.headers.get('x-internal-secret'), null); + assert.equal(await response.text(), 'rejected'); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close(error => error ? reject(error) : resolve()), + ); + } + }); +} diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 255845e9..14e91280 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -240,6 +240,12 @@ export async function startFileRelay( )!, } : {}), + ...(upstreamResponse.headers.has('x-codeapi-error-code') + ? { 'X-CodeAPI-Error-Code': upstreamResponse.headers.get('x-codeapi-error-code')! } + : {}), + ...(upstreamResponse.headers.has('retry-after') + ? { 'Retry-After': upstreamResponse.headers.get('retry-after')! } + : {}), 'Content-Length': String(body.length), }); response.end(body); diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 99303850..717309d7 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -1,6 +1,6 @@ process.env.CODEAPI_EGRESS_GATEWAY_AUTOSTART = 'false'; -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { afterAll, beforeAll, beforeEach, describe, expect, test, spyOn } from 'bun:test'; import crypto from 'crypto'; import RedisMock from 'ioredis-mock'; import type { Server } from 'http'; @@ -435,6 +435,39 @@ describe('egress gateway routes', () => { } }); + test('reports exhausted ledger conflicts as retryable without forwarding the read', async () => { + const redis = new RedisMock(); + env.EGRESS_LEDGER_REQUIRED = true; + setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); + const duplicate = redis.duplicate.bind(redis); + const duplication = spyOn(redis, 'duplicate').mockImplementation(() => { + const connection = duplicate(); + const transaction = { + set: () => transaction, + exec: async () => null, + }; + spyOn(connection, 'multi').mockImplementation(() => transaction as never); + return connection; + }); + try { + await createEgressLedger(claims()); + const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const response = await gatewayFetch(`/sessions/${readSession}/objects?detail=normalized`, { + headers: grantHeader(), + }); + expect(response.status).toBe(503); + expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('ledger_conflict'); + expect(response.headers.get('Retry-After')).toBe('1'); + expect(upstreamCalls).toHaveLength(0); + expect((await assertEgressGrantActive(claims())).request_count).toBe(0); + } finally { + duplication.mockRestore(); + setEgressLedgerRedisForTest(null); + redis.disconnect(); + env.EGRESS_LEDGER_REQUIRED = false; + } + }); + test('lists only scoped objects and injects internal credentials', async () => { upstreamResponse = Response.json([ { id: 'file_123', name: 'inputs/data.csv', storage_session_id: 'sess_input' }, @@ -557,6 +590,7 @@ describe('egress gateway routes', () => { }); expect(response.status).toBe(403); + expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('scope_mismatch'); expect(upstreamCalls).toHaveLength(0); } finally { await redis.disconnect(); diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index d4d3e713..13a6de95 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -6,6 +6,7 @@ import { Readable } from 'stream'; import { env } from './config'; import { EGRESS_GRANT_HEADER, + EGRESS_ERROR_CODE_HEADER, EgressGrantError, egressGrantFromExecutionClaims, openEgressGrant, @@ -157,6 +158,7 @@ app.use((req: Request, res: Response, next: NextFunction) => { function errorStatus(error: EgressGrantError): number { if (error.reason === 'missing_secret' || error.reason === 'weak_secret') return 500; + if (error.reason === 'ledger_conflict') return 503; if (error.reason === 'malformed') return 400; if (error.reason === 'expired') return 401; return 403; @@ -165,6 +167,8 @@ function errorStatus(error: EgressGrantError): number { function sendEgressError(req: Request, res: Response, error: unknown): Response { if (error instanceof EgressGrantError) { const statusCode = errorStatus(error); + res.setHeader(EGRESS_ERROR_CODE_HEADER, error.reason); + if (error.reason === 'ledger_conflict') res.setHeader('Retry-After', '1'); logger.warn('Rejected egress gateway request', { requestId: requestId(res), reason: error.reason, diff --git a/service/src/egress-grant.ts b/service/src/egress-grant.ts index 8d8b1352..8148bec7 100644 --- a/service/src/egress-grant.ts +++ b/service/src/egress-grant.ts @@ -3,6 +3,7 @@ import type { ExecutionManifestClaims, ExecutionManifestInputFile } from './exec import type * as t from './types'; export const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +export const EGRESS_ERROR_CODE_HEADER = 'X-CodeAPI-Error-Code'; export const EGRESS_GRANT_VERSION = 1; const TOKEN_PREFIX = 'ceg1'; @@ -19,7 +20,8 @@ export type EgressGrantErrorReason = | 'malformed' | 'expired' | 'wrong_type' - | 'scope_mismatch'; + | 'scope_mismatch' + | 'ledger_conflict'; export class EgressGrantError extends Error { readonly reason: EgressGrantErrorReason; diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index 24dda87c..b6bbf4fb 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -264,7 +264,7 @@ async function mutateRecord( }); releaseMutationConnection(client); } - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger update conflicted'); + throw new EgressGrantError('ledger_conflict', 'Egress grant ledger update conflicted'); } export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { From dead07bd466a17dab4bc5b2b3520312fa52b0e03 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 16:36:29 -0400 Subject: [PATCH 080/116] perf: reuse authorized input versions and make egress accounting atomic (#180) * perf: reuse authorized input versions and make egress accounting atomic * perf: resolve authorized input manifests once per execution * test: preserve fetch signature in revocation fixture * fix: isolate shared-grant failures and prevent ledger replay --- .github/workflows/ci.yml | 3 + api/src/config.ts | 3 + api/src/download.test.ts | 44 +++ api/src/http-input-cache.test.ts | 163 +++++++++++ api/src/http-input-cache.ts | 177 +++++++++++ api/src/input-manifest.test.ts | 124 ++++++++ api/src/job.ts | 55 ++++ api/src/metrics.ts | 6 + api/src/session-inputs.ts | 36 ++- docs/INPUT_REUSE.md | 91 ++++++ .../templates/egress-gateway-deployment.yaml | 8 + .../templates/file-server-deployment.yaml | 4 + .../templates/worker-sandbox-deployment.yaml | 8 + helm/codeapi/values.yaml | 11 + launcher/src/main.rs | 4 + packages/code/src/relay.test.ts | 65 +++++ packages/code/src/relay.ts | 16 +- service/src/config.ts | 6 + service/src/egress-gateway.test.ts | 207 ++++++++++--- service/src/egress-gateway.ts | 97 +++++- service/src/egress-ledger-reconnect.test.ts | 63 ++++ service/src/egress-ledger-script.ts | 123 ++++++++ service/src/egress-ledger.test.ts | 81 ++++- service/src/egress-ledger.ts | 276 ++++-------------- service/src/file-download.test.ts | 49 ++++ service/src/file-download.ts | 27 ++ service/src/file-object-resolver.test.ts | 50 ++++ service/src/file-object-resolver.ts | 75 +++++ service/src/file-server.ts | 160 ++++------ service/src/test/redis.ts | 45 +++ 30 files changed, 1690 insertions(+), 387 deletions(-) create mode 100644 api/src/http-input-cache.test.ts create mode 100644 api/src/http-input-cache.ts create mode 100644 api/src/input-manifest.test.ts create mode 100644 docs/INPUT_REUSE.md create mode 100644 service/src/egress-ledger-reconnect.test.ts create mode 100644 service/src/egress-ledger-script.ts create mode 100644 service/src/file-download.test.ts create mode 100644 service/src/file-download.ts create mode 100644 service/src/file-object-resolver.test.ts create mode 100644 service/src/file-object-resolver.ts create mode 100644 service/src/test/redis.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71536caa..f041e2d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,9 @@ jobs: - name: Install dependencies run: bun ci + - name: Install Redis for ledger integration tests + run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Build service run: bun run build diff --git a/api/src/config.ts b/api/src/config.ts index bbc2c184..f3b935ff 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -122,6 +122,9 @@ export const config = { /* Ceiling for the pushed input cache (session-inputs.ts). Eviction is * always safe — a miss simply re-pushes on the next probe — so this is a * disk guard, not a correctness knob. */ + http_input_cache_enabled: process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED === 'true', + http_input_cache_max_objects: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS, 4096), + http_input_cache_max_inflight: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT, 16), input_cache_max_bytes: safeInt( process.env.SANDBOX_INPUT_CACHE_MAX_BYTES, 512 * 1024 * 1024, diff --git a/api/src/download.test.ts b/api/src/download.test.ts index bb8ec7e9..da877dad 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -2,6 +2,8 @@ import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, spyOn import * as fsp from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; +import { createHash, randomUUID } from 'node:crypto'; +import { SESSION_INPUT_CACHE_DIR } from './session-inputs'; import * as semver from 'semver'; import { Job, SessionWorkspaceDirtyError, type TFile } from './job'; import type { Runtime } from './runtime'; @@ -511,6 +513,48 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { } }); + it('reuses versioned bytes in fresh workspaces without bypassing a later denial', async () => { + const previousCache = config.http_input_cache_enabled; + const version = randomUUID(); + const cacheKey = createHash('sha256').update(version).digest('hex'); + const otherDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'codeapi-cache-second-')); + config.http_input_cache_enabled = true; + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; + let reads = 0; + let checks = 0; + const meta: Route = { status: 200, body: JSON.stringify({ cacheable: true, cacheKey, version, size: 8, readOnly: false }), + onRequest: () => { checks++; }, + }; + routes.set('/sessions/previous/objects/cached/metadata', meta); + routes.set('/sessions/previous/objects/cached', { status: 200, body: 'original', + headers: { 'X-CodeAPI-Input-Version': version }, + onRequest: request => { reads++; expect(request.headers.get('x-codeapi-input-version')).toBe(version); }, + }); + const file: TFile = { id: 'cached', storage_session_id: 'previous', name: 'data.txt', input_cache_key: cacheKey }; + try { + const first = makeJob([file]); + asInternals(first).submissionDir = tmpDir; + await first.downloadAndWriteFile(file); + await fsp.writeFile(path.join(tmpDir, 'data.txt'), 'sandbox changed this'); + const second = makeJob([file]); + asInternals(second).submissionDir = otherDir; + await second.downloadAndWriteFile(file); + expect(await fsp.readFile(path.join(otherDir, 'data.txt'), 'utf8')).toBe('original'); + expect(reads).toBe(1); + expect(checks).toBe(2); + meta.status = 403; + meta.headers = { 'X-CodeAPI-Error-Code': 'scope_mismatch' }; + await expect(second.downloadAndWriteFile(file)).rejects.toThrow('HTTP error: 403'); + expect(checks).toBe(3); + expect(reads).toBe(1); + } finally { + config.http_input_cache_enabled = previousCache; + await fsp.rm(otherDir, { recursive: true, force: true }); + await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, cacheKey), { force: true }); + await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, `${cacheKey}.json`), { force: true }); + } + }); + it('does not retry an unclassified direct file-server denial', async () => { config.egress_gateway_url = ''; let requests = 0; diff --git a/api/src/http-input-cache.test.ts b/api/src/http-input-cache.test.ts new file mode 100644 index 00000000..674a03ab --- /dev/null +++ b/api/src/http-input-cache.test.ts @@ -0,0 +1,163 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import { rm } from 'node:fs/promises'; +import path from 'node:path'; +import { fetchCachedHttpInput } from './http-input-cache'; +import { hasCachedInput, SESSION_INPUT_CACHE_DIR } from './session-inputs'; + +const keys = new Set(); +afterEach(async () => { + for (const key of keys) { + await rm(path.join(SESSION_INPUT_CACHE_DIR, key), { force: true }); + await rm(path.join(SESSION_INPUT_CACHE_DIR, `${key}.json`), { force: true }); + } + keys.clear(); +}); +function fixture(body = 'input', principal = 'tenant/user') { + const version = randomUUID(); + const cacheKey = createHash('sha256').update(principal + version).digest('hex'); + keys.add(cacheKey); + const meta = { cacheable: true, version, cacheKey, size: Buffer.byteLength(body), readOnly: false, name: 'input.txt' }; + let reads = 0; + let authorizations = 0; + return { + meta, + counts: () => ({ reads, authorizations }), + args: { + maxBytes: 8192, maxObjects: 2, maxFileBytes: 8192, maxInflight: 4, + metadata: async () => { authorizations++; return Response.json(meta); }, + download: async (expected: string, _signal: AbortSignal) => { + reads++; + expect(expected).toBe(version); + return new Response(body, { headers: { 'X-CodeAPI-Input-Version': version } }); + }, + }, + }; +} + +function gate() { + let release!: () => void; + const promise = new Promise(resolve => { release = resolve; }); + return { promise, release }; +} + +describe('authorized HTTP input cache', () => { + test('fresh executions reuse bytes but authorize every hit', async () => { + const f = fixture(); + for (let i = 0; i < 3; i++) { + const response = await fetchCachedHttpInput(f.args); + expect(await response?.text()).toBe('input'); + expect(response?.headers.get('content-disposition')).toContain('input.txt'); + } + expect(f.counts()).toEqual({ reads: 1, authorizations: 3 }); + expect(await hasCachedInput('', '', f.meta.cacheKey)).toBe(false); // Cannot bypass preflight using a pushed key. + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(true); + const denied = await fetchCachedHttpInput({ ...f.args, + metadata: async () => new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' } }), + }); + expect(denied?.status).toBe(403); + expect(f.counts().reads).toBe(1); + }); + + test('new versions and principals never reuse an existing version key', async () => { + for (const [body, principal] of [['old', 'tenant/user'], ['new', 'tenant/user'], ['private', 'another-tenant/user']]) { + const f = fixture(body, principal); + expect(await (await fetchCachedHttpInput(f.args))?.text()).toBe(body); + expect(f.counts().reads).toBe(1); + } + }); + + test('coalesces misses while one cancelled caller leaves the remaining reader intact', async () => { + const f = fixture(); + const started = gate(); + const finish = gate(); + let downloads = 0; + let sharedSignal: AbortSignal | undefined; + const args = { ...f.args, download: async (version: string, signal: AbortSignal) => { + downloads++; sharedSignal = signal; started.release(); + await finish.promise; + return f.args.download(version, signal); + } }; + const controller = new AbortController(); + const first = fetchCachedHttpInput({ ...args, signal: controller.signal }); + const second = fetchCachedHttpInput(args); + await started.promise; + await new Promise(resolve => setTimeout(resolve, 10)); + controller.abort(new Error('first cancelled')); + await expect(first).rejects.toThrow('first cancelled'); + expect(sharedSignal?.aborted).toBe(false); + finish.release(); + expect(await (await second)?.text()).toBe('input'); + expect(downloads).toBe(1); + expect(f.counts().authorizations).toBe(2); + }); + + test('a shared fill does not propagate its creator grant denial to a valid waiter', async () => { + const f = fixture(); + const started = gate(); + const finish = gate(); + const denied = fetchCachedHttpInput({ ...f.args, download: async () => { + started.release(); + await finish.promise; + return new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'request_budget_exceeded' } }); + } }); + await started.promise; + const valid = fetchCachedHttpInput(f.args); + await Bun.sleep(10); + finish.release(); + expect((await denied)?.status).toBe(403); + // Job.fetchInputObject uses the waiter's own normal download on undefined. + expect(await valid).toBeUndefined(); + expect(await (await f.args.download(f.meta.version, new AbortController().signal)).text()).toBe('input'); + expect(f.counts().authorizations).toBe(2); + }); + + test('last-reader cancellation aborts the upstream fill without publishing', async () => { + const f = fixture(); + const started = gate(); + const aborted = gate(); + const controller = new AbortController(); + const pending = fetchCachedHttpInput({ ...f.args, signal: controller.signal, + download: async (_version, signal) => { + started.release(); + return new Promise((_resolve, reject) => signal.addEventListener('abort', () => { + aborted.release(); reject(signal.reason); + }, { once: true })); + }, + }); + await started.promise; + controller.abort(new Error('cancel fill')); + await expect(pending).rejects.toThrow('cancel fill'); + await aborted.promise; + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + }); + + test('changed-version and oversized responses are never published', async () => { + const f = fixture(); + const changed = await fetchCachedHttpInput({ ...f.args, download: async () => new Response('changed', { + headers: { 'X-CodeAPI-Input-Version': randomUUID() }, + }) }); + expect(changed).toBeUndefined(); + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + await expect(fetchCachedHttpInput({ ...f.args, download: async () => new Response('too many bytes', { + headers: { 'X-CodeAPI-Input-Version': f.meta.version }, + }) })).rejects.toThrow(); + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + }); + + test('cache quotas evict old entries and preserve an already-open reader', async () => { + const first = fixture('a'.repeat(4000)); + const second = fixture('b'.repeat(4000)); + const response = await fetchCachedHttpInput({ ...first.args, maxObjects: 1 }); + expect(await (await fetchCachedHttpInput({ ...second.args, maxObjects: 1 }))?.text()).toBe('b'.repeat(4000)); + expect(await hasCachedInput('', '', first.meta.cacheKey, 'http')).toBe(false); + expect(await response?.text()).toBe('a'.repeat(4000)); + }); + + test('legacy metadata protocols fall back without a cache read or fill', async () => { + const f = fixture(); + expect(await fetchCachedHttpInput({ ...f.args, metadata: async () => new Response(null, { status: 404 }) })).toBeUndefined(); + expect(await fetchCachedHttpInput({ ...f.args, metadata: async () => Response.json({ cacheable: false }) })).toBeUndefined(); + expect(f.counts().reads).toBe(0); + }); +}); diff --git a/api/src/http-input-cache.ts b/api/src/http-input-cache.ts new file mode 100644 index 00000000..c785bdf0 --- /dev/null +++ b/api/src/http-input-cache.ts @@ -0,0 +1,177 @@ +import { Readable } from 'node:stream'; +import { createGzip } from 'node:zlib'; +import { httpInputCacheEvents } from './metrics'; +import { cachedInputResponse, openCachedInput, storeCachedInputs } from './session-inputs'; + +type Metadata = { cacheable: true; cacheKey: string; version: string; size: number; name?: string; readOnly: boolean }; +type FillResult = { stored: boolean; status?: number; headers?: Headers }; +type Fill = { controller: AbortController; users: number; result: Promise }; +const fills = new Map(); + +function validMetadata(value: unknown, maxBytes: number): value is Metadata { + if (!value || typeof value !== 'object') return false; + const m = value as Metadata; + return m.cacheable === true && typeof m.cacheKey === 'string' && /^[0-9a-f]{64}$/.test(m.cacheKey) && + typeof m.version === 'string' && /^[0-9a-f-]{36}$/.test(m.version) && + Number.isSafeInteger(m.size) && m.size >= 0 && m.size + 1024 <= maxBytes && + typeof m.readOnly === 'boolean' && (m.name === undefined || (typeof m.name === 'string' && m.name.length <= 4096)); +} + +function tarHeader(name: string, bytes: number): Buffer { + const header = Buffer.alloc(512); + header.write(name, 0, 100, 'utf8'); + header.write('0000600\0', 100, 8, 'ascii'); + header.write(bytes.toString(8).padStart(11, '0') + '\0', 124, 12, 'ascii'); + header.fill(32, 148, 156); + header[156] = 48; + header.write('ustar\0', 257, 6, 'ascii'); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii'); + return header; +} + +/** Reuse the pushed-cache writer's staging, quota, no-follow and atomic commit + * rules. No workspace pathname or sandbox-visible file is used as cache input. */ +async function fillCache(response: Response, meta: Metadata, maxBytes: number, maxObjects: number): Promise { + if (!response.body) throw new Error('Input response has no body'); + const body = response.body; + const sidecar = Buffer.from(JSON.stringify({ readOnly: meta.readOnly, source: 'http' })); + async function* archive(): AsyncGenerator { + yield tarHeader(meta.cacheKey, meta.size); + const reader = body.getReader(); + let bytes = 0; + try { + for (;;) { + const part = await reader.read(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > meta.size) throw new Error('Input exceeded its authorized metadata size'); + yield Buffer.from(part.value); + } + if (bytes !== meta.size) throw new Error('Input size changed during preparation'); + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } + yield Buffer.alloc((512 - meta.size % 512) % 512); + yield tarHeader(`${meta.cacheKey}.json`, sidecar.length); + yield sidecar; + yield Buffer.alloc((512 - sidecar.length % 512) % 512); + yield Buffer.alloc(1024); + } + const source = Readable.from(archive()); + const compressed = createGzip(); + compressed.on('error', () => {}); // Queue admission checks an already-failed stream. + source.on('error', error => compressed.destroy(error)); + source.pipe(compressed); + try { + await storeCachedInputs(compressed, maxBytes, meta.size + sidecar.length, maxObjects); + } finally { + source.destroy(); + compressed.destroy(); + await body.cancel().catch(() => {}); + } +} + +async function waitForFill(fill: Fill, signal?: AbortSignal): Promise { + if (signal?.aborted) { + if (fill.users === 0) fill.controller.abort(signal.reason); + signal.throwIfAborted(); + } + fill.users++; + let abort: (() => void) | undefined; + try { + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(signal?.reason ?? new Error('Input preparation cancelled')); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + }); + return await Promise.race([fill.result, cancelled]); + } finally { + if (abort) signal?.removeEventListener('abort', abort); + if (--fill.users === 0) fill.controller.abort(new Error('No input-cache consumers remain')); + } +} + +/** Every caller performs its own scoped preflight, even for hits or shared fills. + * Only opaque version keys returned by that authorized gateway enter this cache. */ +export async function fetchCachedHttpInput(args: { + metadata(): Promise; + download(version: string, signal: AbortSignal): Promise; + signal?: AbortSignal; + maxBytes: number; + maxFileBytes: number; + maxInflight: number; + maxObjects: number; +}): Promise { + args.signal?.throwIfAborted(); + const preflight = await args.metadata(); + if (preflight.status === 404 || preflight.status === 405) { + await preflight.body?.cancel(); + httpInputCacheEvents.inc({ event: 'legacy_bypass' }); + return undefined; // Older gateway/relay: retain the uncached protocol. + } + if (!preflight.ok) { httpInputCacheEvents.inc({ event: 'preflight_failure' }); return preflight; } + const value: unknown = await preflight.json(); + if (!validMetadata(value, args.maxBytes) || value.size > args.maxFileBytes) { + httpInputCacheEvents.inc({ event: 'uncacheable' }); + return undefined; + } + const meta = value; + args.signal?.throwIfAborted(); + let cached = await openCachedInput('', '', meta.cacheKey, 'http'); + if (cached) httpInputCacheEvents.inc({ event: 'hit' }); + if (!cached) { + let fill = fills.get(meta.cacheKey); + const joinedExistingFill = fill !== undefined; + if (fill) httpInputCacheEvents.inc({ event: 'coalesced' }); + if (!fill) { + if (fills.size >= args.maxInflight) { + httpInputCacheEvents.inc({ event: 'capacity_bypass' }); + return undefined; + } + httpInputCacheEvents.inc({ event: 'fill' }); + const controller = new AbortController(); + fill = { controller, users: 0, result: Promise.resolve({ stored: false }) }; + const ownFill = fill; + fills.set(meta.cacheKey, ownFill); + ownFill.result = (async (): Promise => { + const response = await args.download(meta.version, controller.signal); + if (!response.ok) { + await response.body?.cancel(); + return { stored: false, status: response.status, headers: response.headers }; + } + if (response.headers.get('x-codeapi-input-version') !== meta.version || + (response.headers.get('x-read-only')?.toLowerCase() === 'true') !== meta.readOnly) { + await response.body?.cancel(); + return { stored: false }; // Old file server or inconsistent metadata: never publish. + } + await fillCache(response, meta, args.maxBytes, args.maxObjects); + return { stored: true }; + })().finally(() => { + if (fills.get(meta.cacheKey) === ownFill) fills.delete(meta.cacheKey); + }); + // A caller can cancel between admission and waiting; avoid an unhandled rejection. + void ownFill.result.catch(() => {}); + } + const result = await waitForFill(fill, args.signal); + if (result.status) { + // The transfer used the creator's grant. Its denial/budget must not reject + // another caller whose own preflight succeeded; use that caller's fetch. + if (joinedExistingFill) return undefined; + const headers = new Headers(result.headers); + headers.delete('content-length'); + return new Response(null, { status: result.status, headers }); + } + if (!result.stored) return undefined; + cached = await openCachedInput('', '', meta.cacheKey, 'http'); + } + if (!cached) return undefined; // Evicted between commit and open: normal download remains correct. + if (args.signal?.aborted) { + await cached.handle.close(); + args.signal.throwIfAborted(); + } + const response = cachedInputResponse(cached); + if (meta.name) response.headers.set('content-disposition', `attachment; filename*=UTF-8''${encodeURIComponent(meta.name)}`); + return response; +} diff --git a/api/src/input-manifest.test.ts b/api/src/input-manifest.test.ts new file mode 100644 index 00000000..db55ca4a --- /dev/null +++ b/api/src/input-manifest.test.ts @@ -0,0 +1,124 @@ +import { afterEach, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import * as fsp from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { config } from './config'; +import { Job } from './job'; +import { SESSION_INPUT_CACHE_DIR } from './session-inputs'; +import { fallbackSandboxIdentity } from './workspace-isolation'; + +const originalFetch = globalThis.fetch; +const originalConfig = { http_input_cache_enabled: config.http_input_cache_enabled, egress_gateway_url: config.egress_gateway_url }; +const dirs: string[] = []; +const keys: string[] = []; +afterEach(async () => { + globalThis.fetch = originalFetch; + Object.assign(config, originalConfig); + await Promise.all(dirs.splice(0).map(dir => fsp.rm(dir, { recursive: true, force: true }))); + await Promise.all(keys.splice(0).flatMap(key => [key, `${key}.json`]).map(key => fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, key), { force: true }))); +}); + +async function fixture(count: number, mode: 'batch' | 'legacy' | 'race' = 'batch') { + config.http_input_cache_enabled = true; + config.egress_gateway_url = 'http://manifest.test'; + let manifests = 0, singlePreflights = 0, downloads = 0; + const version = randomUUID(); + const freshVersion = randomUUID(); + const metadata = (id: string, v = version) => { + const key = createHash('sha256').update(id + v).digest('hex'); + keys.push(key); + return { cacheable: true, cacheKey: key, version: v, size: 5, readOnly: false }; + }; + globalThis.fetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + const pathname = new URL(String(url)).pathname; + if (pathname.endsWith('/objects')) return Response.json([]); + if (pathname === '/input-manifest') { + manifests++; + if (mode === 'legacy') return new Response(null, { status: 404 }); + const request = JSON.parse(init!.body as string) as { files: { objectHandle: string }[] }; + expect(request.files).toHaveLength(count); + return Response.json({ files: request.files.map(file => metadata(file.objectHandle)) }); + } + if (pathname.endsWith('/metadata')) { + singlePreflights++; + return Response.json(metadata(pathname.split('/').slice(-2)[0], mode === 'race' ? freshVersion : version)); + } + downloads++; + if (mode === 'race' && new Headers(init?.headers).get('X-CodeAPI-Input-Version') === version) { + return new Response(null, { status: 409 }); + } + return new Response('bytes', { headers: { 'X-CodeAPI-Input-Version': mode === 'race' ? freshVersion : version } }); + }) as typeof fetch; + async function prime(egressGrant = 'test-grant') { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'manifest-prime-')); + dirs.push(dir); + const session = { + runtimeSessionId: 'test', acquire: async () => ({ dir, workspaceId: 'test', identity: fallbackSandboxIdentity() }), + primedInputId: () => undefined, markPrimed: () => {}, markDirty: () => {}, + }; + const job = new Job({ + session_id: 'test', egress_grant: egressGrant, runtime: { language: 'bash', version: '5.0.0', aliases: [] }, + files: Array.from({ length: count }, (_, i) => ({ id: `f${i}`, storage_session_id: 's', name: `file${i}.txt` })), + args: [], stdin: '', timeouts: { run: 5000, compile: 5000 }, cpu_times: { run: 5000, compile: 5000 }, + memory_limits: { run: 128e6, compile: 128e6 }, session, + } as never); + (job as unknown as { log: { level: string } }).log.level = 'silent'; + await job.prime(); + expect(await fsp.readFile(path.join(dir, 'file0.txt'), 'utf8')).toBe('bytes'); + } + return { prime, counts: () => ({ manifests, singlePreflights, downloads }) }; +} + +test('240 inputs use one authorized manifest per fresh workspace and reuse only content', async () => { + const f = await fixture(240); + await f.prime(); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 2, singlePreflights: 0, downloads: 240 }); +}, 30000); + +test('an older gateway falls back to independently authorized preflights', async () => { + const f = await fixture(2, 'legacy'); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 1, singlePreflights: 2, downloads: 2 }); +}); + +test('a raced version consumes the batch entry and retries against fresh metadata', async () => { + const f = await fixture(1, 'race'); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 1, singlePreflights: 1, downloads: 2 }); +}); + + +test('one execution grant denial cannot fail a coalesced execution with its own valid grant', async () => { + const f = await fixture(1); + const underlying = globalThis.fetch; + let release!: () => void; + let started!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const creatorStarted = new Promise(resolve => { started = resolve; }); + let deniedDownloads = 0, validDownloads = 0; + globalThis.fetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + if (String(url).endsWith('/objects/f0')) { + if (new Headers(init?.headers).get('X-CodeAPI-Egress-Grant') === 'denied') { + deniedDownloads++; + started(); await blocked; + return new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' } }); + } + validDownloads++; + } + return underlying(url, init); + }) as typeof fetch; + const creator = f.prime('denied'); + void creator.catch(() => {}); + await creatorStarted; + const waiter = f.prime('valid'); + try { + while (f.counts().manifests < 2) await Bun.sleep(1); + await Bun.sleep(20); + } finally { release(); } + const result = await Promise.allSettled([creator, waiter]); + expect(result.map(item => item.status)).toEqual(['rejected', 'fulfilled']); + expect(deniedDownloads).toBe(1); + expect(validDownloads).toBe(1); +}); diff --git a/api/src/job.ts b/api/src/job.ts index bdcd5061..d2504de9 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -42,6 +42,7 @@ import { validateFilePath, isValidFilePath, } from './validation'; +import { fetchCachedHttpInput } from './http-input-cache'; import { cachedInputResponse, inputCacheKey, openCachedInput } from './session-inputs'; export { @@ -733,6 +734,7 @@ export class Job { private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; private inputFileHashes = new Map(); + private inputManifest = new Map(); private inputDestinations = new Map(); private entryPointName: string | undefined; private chmoddedDirs = new Set(); @@ -938,6 +940,7 @@ export class Job { async prime(): Promise { this.inputDestinations.clear(); + this.inputManifest.clear(); const requestedDestinations = new Map(); for (const file of this.files) { validateFilePath(file.name, '/tmp/codeapi-request-validation'); @@ -990,6 +993,8 @@ export class Job { await this.autoLoadDirkeep(); } + await this.prepareInputManifest(); + /* Promise.all rejects as soon as one operation fails, while its siblings * keep running. The route's finally then calls cleanup(), which clears the * session path/identity. A delayed sibling used to resume afterward and @@ -1431,6 +1436,38 @@ export class Job { throw lastError ?? new Error(`Failed to download input ${file.id}`); } + private async prepareInputManifest(): Promise { + if (!config.http_input_cache_enabled || !config.egress_gateway_url) return; + const files = this.files.filter(file => file.id && file.storage_session_id); + if (!files.length) return; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), AUTO_LOAD_DIRKEEP_TIMEOUT_MS); + try { + const response = await fetch(`${this.fileEgressBaseUrl()}/input-manifest`, { + method: 'POST', headers: this.fileEgressHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ files: files.map(file => ({ + sessionHandle: file.storage_session_id, objectHandle: file.id, + })) }), signal: controller.signal, + }); + if (!response.ok) { + await response.body?.cancel(); + return; // Older gateways/relays and transient failures use per-file preflight. + } + const manifest = await response.json() as { files?: unknown[] }; + if (!Array.isArray(manifest.files) || manifest.files.length !== files.length) return; + manifest.files.forEach((metadata, index) => { + if (metadata && typeof metadata === 'object' && !('retry' in metadata)) { + this.inputManifest.set(files[index], metadata); + } + }); + } catch { + // This is an optimization. Individual reads still authorize and report failures. + } finally { + clearTimeout(timeout); + controller.abort(); + } + } + /** * Resolves an input object's bytes, preferring the runner-local cache the * control plane pushes into on backends whose sandbox cannot reach the file @@ -1462,6 +1499,24 @@ export class Job { `Input ${file.id} was not delivered to the sandbox and no file server is reachable`, ); } + if (config.http_input_cache_enabled && config.egress_gateway_url) { + const response = await fetchCachedHttpInput({ + metadata: () => { + const metadata = this.inputManifest.get(file); + // A version-race retry must obtain a new authorized storage version. + this.inputManifest.delete(file); + return metadata === undefined + ? fetch(`${this.buildDownloadUrl(file)}/metadata`, { headers: this.fileEgressHeaders(), signal }) + : Promise.resolve(Response.json(metadata)); + }, + download: (version, sharedSignal) => fetch(this.buildDownloadUrl(file), { + headers: this.fileEgressHeaders({ 'X-CodeAPI-Input-Version': version }), signal: sharedSignal, + }), + signal, maxBytes: config.input_cache_max_bytes, maxFileBytes: config.max_file_size, + maxInflight: config.http_input_cache_max_inflight, maxObjects: config.http_input_cache_max_objects, + }); + if (response) return response; + } return fetch(this.buildDownloadUrl(file), { headers: this.fileEgressHeaders(), signal, diff --git a/api/src/metrics.ts b/api/src/metrics.ts index 2da33b27..328b4871 100644 --- a/api/src/metrics.ts +++ b/api/src/metrics.ts @@ -16,6 +16,12 @@ const httpRequestDuration = new Histogram({ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], }); +export const httpInputCacheEvents = new Counter({ + name: 'codeapi_sandbox_http_input_cache_events_total', + help: 'Authorized HTTP input cache events; fills and failures may both occur for one input', + labelNames: ['event'] as const, +}); + export const sandboxExecutions = new Counter({ name: 'codeapi_sandbox_executions_total', help: 'Total number of sandbox execution attempts by outcome', diff --git a/api/src/session-inputs.ts b/api/src/session-inputs.ts index 0f1b7f3a..cc9f4b5a 100644 --- a/api/src/session-inputs.ts +++ b/api/src/session-inputs.ts @@ -147,6 +147,8 @@ export interface CachedInputMeta { * ref resolve to the first ref's path — which then overwrote a file the * sandbox had edited. */ readOnly: boolean; + /** HTTP entries require a fresh gateway preflight and cannot satisfy pushed-input probes. */ + source?: 'http'; } export interface CachedInput { @@ -176,7 +178,9 @@ function parseCachedInputMeta(raw: string): CachedInputMeta | null { ) { return null; } - return { readOnly: (parsed as { readOnly: boolean }).readOnly }; + const source = (parsed as { source?: unknown }).source; + if (source !== undefined && source !== 'http') return null; + return { readOnly: (parsed as { readOnly: boolean }).readOnly, ...(source === 'http' ? { source } : {}) }; } catch { return null; } @@ -186,8 +190,9 @@ export async function hasCachedInput( storageSessionId: string, id: string, cacheKey?: string, + source: 'push' | 'http' = 'push', ): Promise { - const opened = await openCachedInput(storageSessionId, id, cacheKey); + const opened = await openCachedInput(storageSessionId, id, cacheKey, source); if (!opened) return false; await opened.handle.close(); return true; @@ -197,6 +202,7 @@ export async function openCachedInput( storageSessionId: string, id: string, cacheKey?: string, + source: 'push' | 'http' = 'push', ): Promise { const key = cacheKey ?? inputCacheKey(storageSessionId, id); if (!/^[0-9a-f]{64}$/.test(key)) return null; @@ -229,7 +235,7 @@ export async function openCachedInput( await metaHandle?.close().catch(() => {}); } const meta = raw === null ? null : parseCachedInputMeta(raw); - if (!meta) { + if (!meta || (meta.source ?? 'push') !== source) { logger.warn({ key }, 'Ignoring session input with missing or invalid metadata'); await handle.close(); return null; @@ -325,7 +331,13 @@ async function extractInputArchive( }; compressedGuard.on('error', forwardCompressedError); body.once('error', forwardBodyError); - body.pipe(compressedGuard).pipe(gunzip); + const sourceState = body as NodeJS.ReadableStream & { errored?: Error; destroyed?: boolean }; + compressedGuard.pipe(gunzip); + if (sourceState.errored || sourceState.destroyed) { + forwardBodyError(sourceState.errored ?? new Error('Input stream was cancelled before extraction')); + } else { + body.pipe(compressedGuard); + } let buffered = Buffer.alloc(0); let current: @@ -517,7 +529,10 @@ async function storeCachedInputsOnce( body: NodeJS.ReadableStream, maxBytes = Number.MAX_SAFE_INTEGER, expectedBytes?: number, + maxObjects = Number.MAX_SAFE_INTEGER, ): Promise { + const readable = body as NodeJS.ReadableStream & { errored?: Error; destroyed?: boolean }; + if (readable.errored || readable.destroyed) throw readable.errored ?? new Error('Input stream was cancelled before cache admission'); await fsp.mkdir(SESSION_INPUT_CACHE_DIR, { recursive: true, mode: 0o700 }); const cacheStat = await fsp.lstat(SESSION_INPUT_CACHE_DIR); if (!cacheStat.isDirectory() || cacheStat.isSymbolicLink()) { @@ -571,6 +586,8 @@ async function storeCachedInputsOnce( } } + if (keys.length > maxObjects) throw new Error('Input batch exceeds cache object limit'); + await pruneInputCache(Math.max(0, maxBytes - stagedBytes), maxObjects - keys.length); let stored = 0; /* Commit sidecars before data. A new key remains a probe miss until both * exist; replacing an immutable key can only expose its new validated @@ -596,6 +613,7 @@ export async function storeCachedInputs( body: NodeJS.ReadableStream, maxBytes = Number.MAX_SAFE_INTEGER, expectedBytes?: number, + maxObjects = Number.MAX_SAFE_INTEGER, ): Promise { /* Concurrent pushes otherwise each budget only its own staging tree and can * collectively recreate the same transient disk spike. Queue extraction; @@ -607,7 +625,7 @@ export async function storeCachedInputs( }); await previous; try { - return await storeCachedInputsOnce(body, maxBytes, expectedBytes); + return await storeCachedInputsOnce(body, maxBytes, expectedBytes, maxObjects); } finally { release(); } @@ -615,7 +633,7 @@ export async function storeCachedInputs( /** Drops least-recently-used entries until the cache fits `maxBytes`. Eviction * is always safe: a miss simply re-pushes on the next probe. */ -export async function pruneInputCache(maxBytes: number): Promise { +export async function pruneInputCache(maxBytes: number, maxObjects = Number.MAX_SAFE_INTEGER): Promise { const names = await fsp.readdir(SESSION_INPUT_CACHE_DIR).catch(() => [] as string[]); const nameSet = new Set(names.filter(name => ENTRY_PATTERN.test(name))); const pairs: Array<{ key: string; size: number; atime: number }> = []; @@ -644,14 +662,16 @@ export async function pruneInputCache(maxBytes: number): Promise { await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, orphan), { force: true }).catch(() => {}); } } - if (total <= maxBytes) return; + let objects = pairs.length; + if (total <= maxBytes && objects <= maxObjects) return; pairs.sort((a, b) => a.atime - b.atime); for (const pair of pairs) { - if (total <= maxBytes) break; + if (total <= maxBytes && objects <= maxObjects) break; await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, pair.key), { force: true }).catch(() => {}); await fsp .rm(path.join(SESSION_INPUT_CACHE_DIR, `${pair.key}${META_SUFFIX}`), { force: true }) .catch(() => {}); total -= pair.size; + objects -= 1; } } diff --git a/docs/INPUT_REUSE.md b/docs/INPUT_REUSE.md new file mode 100644 index 00000000..40e277c4 --- /dev/null +++ b/docs/INPUT_REUSE.md @@ -0,0 +1,91 @@ +# Bounded input reuse for stateless executions + +Fresh execution workspaces can reuse input contents without retaining a mutable conversation sandbox. Each reader first authorizes a metadata request through the egress gateway. New file-server uploads carry a random `codeapi-version` metadata value that changes on every PUT, including overwrites with identical contents. The gateway binds a cache key to that version, storage identity, tenant, user, size, filename, and read-only flag. + +```mermaid +sequenceDiagram + participant R as Runner + participant G as Egress gateway + participant F as File server + participant C as Protected input cache + R->>G: POST bounded input manifest (one per execution) + G->>G: Verify grant, scope, expiry, revocation, budget + G->>F: Resolve current metadata with bounded concurrency + F-->>G: Current upload version and metadata + G-->>R: Ordered principal-scoped version keys + R->>C: Open authorized version + alt Cache miss + R->>G: Download with expected version + G->>G: Authorize and account download + G->>F: Forward expected version + F-->>R: Exact GET metadata and bytes, or 409 if changed + R->>C: Stage, validate size, atomically publish + end + R->>R: Copy into fresh workspace using existing priming rules +``` + +## Invariants + +- A cache hit never authorizes an input. Every execution performs its own preflight, including readers joining a shared fill. Denied or revoked grants cannot use cached data. +- Every manifest handle is scope-checked before storage access. Revocation is checked again before returning resolved metadata. A deadline and disconnect cancel storage work; failures and older gateways fall back to independently authorized per-file preflights. A version-race retry discards its manifest entry. +- HTTP entries are marked separately from pushed inputs. Supplying an HTTP key in `input_cache_key` cannot bypass preflight through the older pushed-cache path. +- Cache files stay outside execution workspaces and sandbox mounts. Priming copies bytes; it never hard-links a writable workspace to trusted cache contents. Existing no-follow, read-only, hashing, atomic rename, and descriptor-pinning behavior remains in use. +- Concurrent authorized misses for the same version can share one download. Cancelling one reader does not cancel remaining readers; cancelling the last reader aborts the shared request. The number of fills, cached bytes, and object count are bounded. +- The downloader checks the version from the **actual GET**, rather than labeling bytes with metadata from an earlier HEAD. A raced overwrite returns 409 and preparation retries from current metadata. Legacy objects without a version use the uncached path. +- All writers of input objects must assign a fresh version on every overwrite. The file server does so for both upload routes. Checkpoint storage uses a separate path. Direct bucket writes that preserve an old version marker are outside this protocol. +- The optional Redis object-key index stores only a locator hint. It is not an authorization or metadata cache. Preflights still read current storage metadata; indexed keys must match the exact session and object identity. +- Shared download errors belong to the initiating grant. A coalesced caller falls back to its own authorized download rather than inheriting that grant's denial or exhausted budget. +- Redis reconnects never replay unfulfilled ledger mutations. A lost reply fails closed and may leave a conservatively charged counter/reservation until grant expiry; automatically refunding an ambiguous mutation could over-credit its budget. +- Full grant policy is no longer returned to the gateway for each authorization check. Atomic Redis scripts serialize accounting with revocation. Duplicate releases cannot repeatedly refund unrelated counters. Newly created compact ledgers keep immutable policy separate from mutable counters. + +## Configuration + +| Helm value | Environment variable | Default | +|---|---|---| +| `egressGrant.ledgerCompact` | `CODEAPI_EGRESS_LEDGER_COMPACT` | `false` | +| `egressGrant.inputManifestMaxFiles` | `CODEAPI_INPUT_MANIFEST_MAX_FILES` | `512` | +| `egressGrant.inputManifestConcurrency` | `CODEAPI_INPUT_MANIFEST_CONCURRENCY` | `8` | +| `egressGrant.inputManifestTimeoutMs` | `CODEAPI_INPUT_MANIFEST_TIMEOUT_MS` | `10000` | +| `fileServer.objectIndexEnabled` | `CODEAPI_FILE_OBJECT_INDEX_ENABLED` | `false` | +| `fileServer.metadataConcurrency` | `CODEAPI_FILE_METADATA_CONCURRENCY` | `1` | +| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `false` | +| `workerSandbox.sandbox.httpInputCacheMaxInflight` | `SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT` | `16` | +| `workerSandbox.sandbox.httpInputCacheMaxObjects` | `SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS` | `4096` | +| `workerSandbox.sandbox.inputCacheMaxBytes` | `SANDBOX_INPUT_CACHE_MAX_BYTES` | `536870912` | + +HTTP reuse requires a configured egress gateway. Cacheable objects are also bounded by the existing runner maximum file size. Cache capacity is local to each runner; eviction, restart, or routing to another runner causes a safe cache miss. The cache does not require persistent-session affinity. + +The manifest accepts at most 512 entries and a 4 MiB JSON body (protocol safety ceilings), with configured concurrency capped at 64. The runner bounds its opportunistic manifest request to 10 seconds, matching directory preparation, then uses per-file authorization if it cannot obtain a complete response. Oversized batches also fall back. Manifest requests remove repeated grant-header transfer and decoding, but still read current storage metadata for each file. + +Metadata listing concurrency preserves order and is capped at 64. A canary can use 8 after measuring storage load. This applies to directory-marker preparation as well; marker listings still happen and are not a retained conversation manifest. + +## Rollout and rollback + +1. Deploy the new binaries with feature flags off. Atomic accounting supports existing JSON ledgers, and legacy downloads retain metadata compatibility. The new file server stamps future uploads with versions. +2. Update **all** egress-gateway replicas before enabling compact ledgers. New binaries read both formats regardless of the creation flag. Older binaries cannot read compact hashes. To roll back to an older binary, disable compact creation, drain active grants, and wait their maximum TTL plus grace; never delete active ledgers to force a rollback. +3. Update all file-server writers before enabling the object-key index. Otherwise an older writer can change a locator without updating the index. Keep file-server replicas consistent during an indexed rollout. +4. Update the gateway, relay, runner, and launcher before enabling HTTP reuse on a small runner canary. Older gateway/relay metadata routes return 404/405 and fall back safely. Older files return `cacheable: false`. Keep the feature disabled for storage adapters that cannot return user metadata on GET. +5. Observe `codeapi_sandbox_http_input_cache_events_total` (bounded event labels, no identities), cold and warm preparation latency, storage/Redis operations, admission fairness, request budgets, and memory/disk pressure before widening the rollout. A successful manifest consumes one read request for the batch, matching the existing list-request accounting unit. Per-file compatibility preflights each consume a read request; each cold miss consumes an additional download request. Do not disable budget enforcement to accommodate a workload. +6. Disable HTTP reuse to return to normal downloads immediately. Cached files can age out normally; no workspace deletion or migration is needed. + +The creation flags default off. No deployment or object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. + +## Validation + +Ledger tests use an isolated real `redis-server` on a Unix socket with persistence disabled. Install Redis before running service tests. They cover legacy/compact formats, 240 concurrent reads against a strict budget, revocation, expiry, rejected uploads, duplicate releases, and format changes without resetting state. + +Focused commands: + +```sh +cd api +bun test src/input-manifest.test.ts src/http-input-cache.test.ts src/session-inputs.test.ts src/session-inputs.prime.test.ts src/download.test.ts src/inline-prime-atomicity.test.ts src/job-cleanup.test.ts +npx tsc --noEmit +``` + +```sh +cd service +bun test src/egress-ledger.test.ts src/egress-ledger-reconnect.test.ts src/egress-gateway.test.ts src/file-object-resolver.test.ts src/file-download.test.ts src/file-metadata.test.ts +npx tsc --noEmit +``` + +Run the code-package tests with Node (its supported test runner), plus launcher and deployment checks in CI. Cache regressions cover fresh-workspace reuse, cross-principal/version separation, denied preflights, pushed-key bypass prevention, coalesced cancellation, changed or oversized responses, and descriptor-safe eviction. Production latency targets must be validated with real regional storage latency and workload sizes; local synthetic results are not a production SLO. diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 6e394909..49c865f0 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -40,6 +40,14 @@ spec: value: {{ .Values.hardenedSandboxMode | quote }} - name: CODEAPI_EGRESS_LEDGER_REQUIRED value: {{ .Values.egressGrant.ledgerRequired | quote }} + - name: CODEAPI_INPUT_MANIFEST_MAX_FILES + value: {{ .Values.egressGrant.inputManifestMaxFiles | quote }} + - name: CODEAPI_INPUT_MANIFEST_CONCURRENCY + value: {{ .Values.egressGrant.inputManifestConcurrency | quote }} + - name: CODEAPI_INPUT_MANIFEST_TIMEOUT_MS + value: {{ .Values.egressGrant.inputManifestTimeoutMs | quote }} + - name: CODEAPI_EGRESS_LEDGER_COMPACT + value: {{ .Values.egressGrant.ledgerCompact | quote }} - name: CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS value: {{ .Values.egressGrant.ledgerTtlGraceSeconds | quote }} - name: EGRESS_GATEWAY_PORT diff --git a/helm/codeapi/templates/file-server-deployment.yaml b/helm/codeapi/templates/file-server-deployment.yaml index 1dd95166..1a9e8531 100644 --- a/helm/codeapi/templates/file-server-deployment.yaml +++ b/helm/codeapi/templates/file-server-deployment.yaml @@ -51,6 +51,10 @@ spec: containerPort: {{ .Values.fileServer.service.port }} protocol: TCP env: + - name: CODEAPI_FILE_METADATA_CONCURRENCY + value: {{ .Values.fileServer.metadataConcurrency | quote }} + - name: CODEAPI_FILE_OBJECT_INDEX_ENABLED + value: {{ .Values.fileServer.objectIndexEnabled | quote }} {{ include "codeapi.otel.env" (dict "root" . "serviceName" "aiml-codeapi-file-server") | nindent 12 }} {{- if $useS3 }} # AWS S3 configuration (IRSA or static credentials) diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index 39c2e197..2819d998 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -314,6 +314,14 @@ spec: value: {{ (.Values.workerSandbox.sandbox.jobUidCount | default (.Values.workerSandbox.sandbox.maxConcurrentJobs | default (mul (.Values.workerSandbox.launcher.vcpus | default 2) 4))) | quote }} - name: SANDBOX_WORKSPACE_REAPER_MAX_AGE_SECONDS value: {{ (.Values.workerSandbox.sandbox.workspaceReaperMaxAgeSeconds | default 3600) | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_ENABLED + value: {{ .Values.workerSandbox.sandbox.httpInputCacheEnabled | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT + value: {{ .Values.workerSandbox.sandbox.httpInputCacheMaxInflight | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS + value: {{ .Values.workerSandbox.sandbox.httpInputCacheMaxObjects | quote }} + - name: SANDBOX_INPUT_CACHE_MAX_BYTES + value: {{ .Values.workerSandbox.sandbox.inputCacheMaxBytes | quote }} - name: SANDBOX_EXECUTE_BODY_LIMIT value: {{ .Values.workerSandbox.sandbox.executeBodyLimit | quote }} - name: SANDBOX_DISABLE_NETWORKING diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index ee997ce8..3ecfbef8 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -46,6 +46,11 @@ egressGrant: ttlSeconds: 900 ledgerRequired: true ledgerTtlGraceSeconds: 300 + # Enable only after all gateway replicas support compact ledgers. + ledgerCompact: false + inputManifestMaxFiles: 512 + inputManifestConcurrency: 8 + inputManifestTimeoutMs: 10000 # Worker signs sandbox execute requests with this private key; sandbox-runner # receives only the public verifier so a runner compromise cannot mint new @@ -268,6 +273,10 @@ workerSandbox: # Defaults to maxConcurrentJobs when unset. jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 + httpInputCacheEnabled: false + httpInputCacheMaxInflight: 16 + httpInputCacheMaxObjects: 4096 + inputCacheMaxBytes: 536870912 # Language runtime package delivery packages: @@ -327,6 +336,8 @@ workerSandbox: # FILE SERVER (S3/MinIO integration, stateless) # ============================================================================= fileServer: + objectIndexEnabled: false + metadataConcurrency: 1 enabled: true replicaCount: 1 diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 02d26418..7771b9b9 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -421,6 +421,10 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_COMPILE_TIMEOUT", "SANDBOX_DATA_DIRECTORY", "SANDBOX_DISABLE_NETWORKING", + "SANDBOX_HTTP_INPUT_CACHE_ENABLED", + "SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT", + "SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS", + "SANDBOX_INPUT_CACHE_MAX_BYTES", "SANDBOX_EXECUTE_BODY_LIMIT", "SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY", "SANDBOX_FORWARD_TARGET", diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts index a9b81fd0..c4d92288 100644 --- a/packages/code/src/relay.test.ts +++ b/packages/code/src/relay.test.ts @@ -416,3 +416,68 @@ for (const [status, reason] of [[403, 'scope_mismatch'], [503, 'ledger_conflict' } }); } + +test('file relay carries version preflights and download preconditions without opening metadata writes', async () => { + let requests = 0; + const upstream = createServer((req, res) => { + requests++; + if (req.url?.endsWith('/metadata')) { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ cacheable: true, version: 'opaque-version' })); + } else { + assert.equal(req.headers['x-codeapi-input-version'], 'opaque-version'); + res.writeHead(200, { 'X-CodeAPI-Input-Version': 'opaque-version' }).end('bytes'); + } + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', maxBytes: 1024, timeoutMs: 1000 }); + const headers = { 'X-LibreChat-Code-Relay-Token': 'relay-secret', 'X-CodeAPI-Egress-Grant': 'grant' }; + try { + const metadata = await fetch(`${relay.url}/sessions/s/objects/o/metadata`, { headers }); + assert.equal(metadata.status, 200); + assert.equal((await metadata.json() as { version: string }).version, 'opaque-version'); + const input = await fetch(`${relay.url}/sessions/s/objects/o`, { headers: { ...headers, 'X-CodeAPI-Input-Version': 'opaque-version' } }); + assert.equal(input.headers.get('x-codeapi-input-version'), 'opaque-version'); + assert.equal(await input.text(), 'bytes'); + const denied = await fetch(`${relay.url}/sessions/s/objects/o/metadata`, { method: 'PUT', headers, body: '' }); + assert.equal(denied.status, 404); + assert.equal(requests, 2); + } finally { + await relay.close(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); + + +test('file relay forwards bounded manifest POSTs and rejects alternate manifest methods', async () => { + let requests = 0; + const upstream = createServer(async (req, res) => { + requests++; + assert.equal(req.method, 'POST'); + assert.equal(req.url, '/input-manifest'); + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant'); + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + assert.deepEqual(JSON.parse(Buffer.concat(chunks).toString()), { files: [] }); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ files: [] })); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', maxBytes: 128, timeoutMs: 1000 }); + const headers = { 'X-LibreChat-Code-Relay-Token': 'relay-secret', 'X-CodeAPI-Egress-Grant': 'grant', 'Content-Type': 'application/json' }; + try { + const response = await fetch(`${relay.url}/input-manifest`, { method: 'POST', headers, body: JSON.stringify({ files: [] }) }); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { files: [] }); + for (const method of ['GET', 'PUT']) { + const denied = await fetch(`${relay.url}/input-manifest`, { method, headers }); + assert.equal(denied.status, 404); + } + const oversized = await fetch(`${relay.url}/input-manifest`, { method: 'POST', headers, body: 'x'.repeat(129) }); + assert.equal(oversized.status, 413); + assert.equal(requests, 1); + } finally { + await relay.close(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 14e91280..07aa8714 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -20,6 +20,7 @@ export interface FileRelayHandle { } const OBJECT_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+$/; +const OBJECT_METADATA_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+\/metadata$/; const OBJECT_LIST_PATH = /^\/sessions\/[^/]+\/objects$/; const MAX_RELAY_HEADER_BYTES = 512 * 1024; const LOCAL_HTTP_HOSTS = new Set([ @@ -160,16 +161,19 @@ export async function startFileRelay( response.end('{"status":"ok"}'); return; } + const manifestRequest = request.method === 'POST' && requestUrl.pathname === '/input-manifest' && requestUrl.search.length === 0; const objectRequest = OBJECT_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; + const metadataRequest = request.method === 'GET' && + OBJECT_METADATA_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; const normalizedListRequest = request.method === 'GET' && OBJECT_LIST_PATH.test(requestUrl.pathname) && requestUrl.searchParams.size === 1 && requestUrl.searchParams.get('detail') === 'normalized'; if ( - (request.method !== 'GET' && request.method !== 'PUT') || - (!objectRequest && !normalizedListRequest) + (request.method !== 'GET' && request.method !== 'PUT' && !manifestRequest) || + (!objectRequest && !normalizedListRequest && !metadataRequest && !manifestRequest) ) { response.writeHead(404).end(); return; @@ -191,7 +195,7 @@ export async function startFileRelay( }`; target.search = requestUrl.search; const requestBody = - request.method === 'PUT' + (request.method === 'PUT' || manifestRequest) ? await readRequestBody(request, options.maxBytes) : undefined; const upstreamResponse = await fetch(target, { @@ -200,7 +204,9 @@ export async function startFileRelay( ...(typeof grant === 'string' ? { 'X-CodeAPI-Egress-Grant': grant } : {}), - ...(request.method === 'PUT' + ...(typeof request.headers['x-codeapi-input-version'] === 'string' + ? { 'X-CodeAPI-Input-Version': request.headers['x-codeapi-input-version'] } : {}), + ...((request.method === 'PUT' || manifestRequest) ? { 'Content-Length': String(requestBody?.length ?? 0), ...(typeof request.headers['content-type'] === 'string' @@ -246,6 +252,8 @@ export async function startFileRelay( ...(upstreamResponse.headers.has('retry-after') ? { 'Retry-After': upstreamResponse.headers.get('retry-after')! } : {}), + ...(upstreamResponse.headers.has('x-codeapi-input-version') + ? { 'X-CodeAPI-Input-Version': upstreamResponse.headers.get('x-codeapi-input-version')! } : {}), 'Content-Length': String(body.length), }); response.end(body); diff --git a/service/src/config.ts b/service/src/config.ts index 94daf5a3..90df6b58 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -318,6 +318,12 @@ export const env = { EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, EGRESS_LEDGER_REQUIRED: process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + FILE_METADATA_CONCURRENCY: Math.min(64, Math.max(1, Math.floor(Number(process.env.CODEAPI_FILE_METADATA_CONCURRENCY) || 1))), + FILE_OBJECT_INDEX_ENABLED: process.env.CODEAPI_FILE_OBJECT_INDEX_ENABLED === 'true', + INPUT_MANIFEST_MAX_FILES: Math.min(512, Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_MAX_FILES) || 512))), + INPUT_MANIFEST_CONCURRENCY: Math.min(64, Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_CONCURRENCY) || 8))), + INPUT_MANIFEST_TIMEOUT_MS: Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_TIMEOUT_MS) || 10000)), + EGRESS_LEDGER_COMPACT: process.env.CODEAPI_EGRESS_LEDGER_COMPACT === 'true', EGRESS_LEDGER_TTL_GRACE_SECONDS: Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds(process.env.EGRESS_GRANT_TTL_SECONDS, defaultJobTimeoutMs), diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 717309d7..c3cd4da4 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -1,14 +1,15 @@ process.env.CODEAPI_EGRESS_GATEWAY_AUTOSTART = 'false'; -import { afterAll, beforeAll, beforeEach, describe, expect, test, spyOn } from 'bun:test'; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; import crypto from 'crypto'; -import RedisMock from 'ioredis-mock'; +import { startTestRedis } from './test/redis'; import type { Server } from 'http'; import type { AddressInfo } from 'net'; import { env } from './config'; import { assertEgressGrantActive, createEgressLedger, + revokeEgressLedger, setEgressLedgerRedisForTest, } from './egress-ledger'; import { @@ -435,39 +436,171 @@ describe('egress gateway routes', () => { } }); - test('reports exhausted ledger conflicts as retryable without forwarding the read', async () => { - const redis = new RedisMock(); + test('batch preflight scopes every entry before reading and preserves order under a concurrency bound', async () => { + const files = Array.from({ length: 17 }, (_, i) => ({ id: `file_${i}`, session_id: 'sess_input', name: `file_${i}.csv` })); + const grant = claims({ input_files: files }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const body = { files: files.map(file => ({ sessionHandle: sid, objectHandle: objectHandle({ fileId: file.id, name: file.name }) })) }; + let active = 0, peak = 0, calls = 0; + const width = env.INPUT_MANIFEST_CONCURRENCY; + env.INPUT_MANIFEST_CONCURRENCY = 3; + globalThis.fetch = (async (input: RequestInfo | URL) => { + calls++; active++; peak = Math.max(peak, active); + await Bun.sleep(5); + active--; + const id = String(input).split('/').at(-2)!; + return Response.json({ version: crypto.randomUUID(), size: 5, originalFilename: `${id}.csv` }); + }) as typeof fetch; + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); env.EGRESS_LEDGER_REQUIRED = true; - setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); - const duplicate = redis.duplicate.bind(redis); - const duplication = spyOn(redis, 'duplicate').mockImplementation(() => { - const connection = duplicate(); - const transaction = { - set: () => transaction, - exec: async () => null, - }; - spyOn(connection, 'multi').mockImplementation(() => transaction as never); - return connection; + try { + await createEgressLedger(grant); + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + expect(response.status).toBe(200); + const result = await response.json() as { files: { cacheKey: string; name: string }[] }; + expect(result.files.map(file => file.name)).toEqual(files.map(file => file.name)); + expect(result.files.every(file => /^[0-9a-f]{64}$/.test(file.cacheKey))).toBe(true); + expect(calls).toBe(17); + expect(peak).toBe(3); + expect((await assertEgressGrantActive(grant)).request_count).toBe(1); + body.files.push({ sessionHandle: sid, objectHandle: objectHandle({ fileId: 'outside_scope' }) }); + const denied = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + expect(denied.status).toBe(403); + expect(calls).toBe(17); + } finally { + env.INPUT_MANIFEST_CONCURRENCY = width; + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.closeTestServer(); + } + }); + + test('batch preflight withholds resolved metadata if the grant is revoked during storage access', async () => { + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); + env.EGRESS_LEDGER_REQUIRED = true; + try { + const grant = claims(); + await createEgressLedger(grant); + globalThis.fetch = (async (_input: RequestInfo | URL) => { + await revokeEgressLedger(grant.grant_id!, 'test revocation'); + return Response.json({ version: crypto.randomUUID(), size: 5 }); + }) as typeof fetch; + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [{ sessionHandle: sessionHandle({ dir: 'read', sessionId: 'sess_input' }), objectHandle: objectHandle({}) }] }), + }); + expect(response.status).toBe(403); + expect(await response.text()).not.toContain('cacheKey'); + } finally { + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.closeTestServer(); + } + }); + + test('batch deadline aborts in-flight storage requests without starting queued inputs', async () => { + const timeout = env.INPUT_MANIFEST_TIMEOUT_MS; + const width = env.INPUT_MANIFEST_CONCURRENCY; + env.INPUT_MANIFEST_TIMEOUT_MS = 20; + env.INPUT_MANIFEST_CONCURRENCY = 1; + let started = 0, aborted = 0; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + started++; + return new Promise((_resolve, reject) => { + init!.signal!.addEventListener('abort', () => { aborted++; reject(init!.signal!.reason); }, { once: true }); + }); + }) as typeof fetch; + try { + const file = { sessionHandle: sessionHandle({ dir: 'read', sessionId: 'sess_input' }), objectHandle: objectHandle({}) }; + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [file, file] }), + }); + expect(response.ok).toBe(false); + expect(started).toBe(1); + expect(aborted).toBe(1); + } finally { + env.INPUT_MANIFEST_TIMEOUT_MS = timeout; + env.INPUT_MANIFEST_CONCURRENCY = width; + } + }); + + test('batch preflight rejects oversized and malformed manifests before storage access', async () => { + for (const files of [Array.from({ length: env.INPUT_MANIFEST_MAX_FILES + 1 }, () => ({})), [null], [{}]]) { + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(), 'Content-Type': 'application/json' }, body: JSON.stringify({ files }), + }); + expect(response.status).toBe(400); + } + expect(upstreamCalls).toHaveLength(0); + }); + + test('preflight authorizes scope and returns version keys scoped to the principal', async () => { + const version = crypto.randomUUID(); + upstreamResponse = Response.json({ version, size: 5, originalFilename: 'inputs/data.csv', readOnly: true }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + const response = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(response.status).toBe(200); + const metadata = await response.json() as { cacheKey: string; version: string; readOnly: boolean }; + expect(metadata.cacheKey).toMatch(/^[0-9a-f]{64}$/); + expect(metadata.version).toBe(version); + expect(metadata.readOnly).toBe(true); + expect(upstreamCalls[0].url).toEndWith('/sessions/sess_input/objects/file_123/metadata'); + const second = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { + headers: grantHeader(claims({ tenant_id: 'another_tenant' })), }); + expect((await second.json() as { cacheKey: string }).cacheKey).not.toBe(metadata.cacheKey); + const before = upstreamCalls.length; + const denied = await gatewayFetch(`/sessions/${sid}/objects/${objectHandle({ fileId: 'outside_scope' })}/metadata`, { + headers: grantHeader(), + }); + expect(denied.status).toBe(403); + expect(upstreamCalls).toHaveLength(before); + }); + + test('preflight denies revoked grants and old metadata stays uncached', async () => { + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + upstreamResponse = Response.json({ size: 5 }); + const legacy = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(await legacy.json()).toEqual({ cacheable: false }); + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); + env.EGRESS_LEDGER_REQUIRED = true; try { await createEgressLedger(claims()); - const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); - const response = await gatewayFetch(`/sessions/${readSession}/objects?detail=normalized`, { - headers: grantHeader(), - }); - expect(response.status).toBe(503); - expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('ledger_conflict'); - expect(response.headers.get('Retry-After')).toBe('1'); - expect(upstreamCalls).toHaveLength(0); - expect((await assertEgressGrantActive(claims())).request_count).toBe(0); + await redis.del(`codeapi:egress:grant:${claims().grant_id}`); + const before = upstreamCalls.length; + const denied = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(denied.status).toBe(403); + expect(upstreamCalls).toHaveLength(before); } finally { - duplication.mockRestore(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); - redis.disconnect(); env.EGRESS_LEDGER_REQUIRED = false; } }); + test('forwards the authorized input-version precondition on downloads', async () => { + const version = crypto.randomUUID(); + upstreamResponse = new Response('bytes', { headers: { 'X-CodeAPI-Input-Version': version } }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const response = await gatewayFetch(`/sessions/${sid}/objects/${objectHandle({})}`, { + headers: { ...grantHeader(), 'X-CodeAPI-Input-Version': version }, + }); + expect(response.status).toBe(200); + expect(response.headers.get('x-codeapi-input-version')).toBe(version); + expect(new Headers(upstreamCalls[0].init.headers).get('x-codeapi-input-version')).toBe(version); + await response.text(); + }); + test('lists only scoped objects and injects internal credentials', async () => { upstreamResponse = Response.json([ { id: 'file_123', name: 'inputs/data.csv', storage_session_id: 'sess_input' }, @@ -495,7 +628,7 @@ describe('egress gateway routes', () => { }); test('accepts legacy rollout grants and handles while ledger-required mode is enabled', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -526,14 +659,14 @@ describe('egress gateway routes', () => { expect(record.max_output_files).toBe(50); expect(record.max_requests).toBe(1000); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('restores token-only legacy grants and creates ledger state before returning handles', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -569,14 +702,14 @@ describe('egress gateway routes', () => { expect(record.grant_id).toBe(legacyGrant.grant_id); expect(record.exec_id).toBe('exec_123'); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('rejects grantless handles for non-legacy grants in ledger-required mode', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -593,7 +726,7 @@ describe('egress gateway routes', () => { expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('scope_mismatch'); expect(upstreamCalls).toHaveLength(0); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } @@ -849,7 +982,7 @@ describe('egress gateway routes', () => { }); test('rolls back upload reservations when upstream PUT throws', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); const grant = claims({ max_output_files: 1, max_requests: 3 }); @@ -894,14 +1027,14 @@ describe('egress gateway routes', () => { expect(retried.status).toBe(201); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('does not roll back ledger state when upload reservation is rejected', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -933,14 +1066,14 @@ describe('egress gateway routes', () => { expect(await upload('bbbbbbbbbbbbbbbbbbbbb')).toBe(403); expect(upstreamCalls).toHaveLength(1); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('enforces output budgets per turn when grants reuse an output session', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -989,7 +1122,7 @@ describe('egress gateway routes', () => { expect(await upload(secondTurn, 'ddddddddddddddddddddd')).toBe(201); expect(await upload(secondTurn, 'eeeeeeeeeeeeeeeeeeeee')).toBe(403); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 13a6de95..f86ce656 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -26,7 +26,7 @@ import { isSyntheticInternalRequestHeader, } from './internal-synthetic'; import { - assertEgressGrantActive, + checkEgressGrantActive, createEgressLedger, ensureEgressLedger, pingEgressLedger, @@ -44,6 +44,7 @@ import logger from './logger'; import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; import { isOpaqueObjectContentDisposition } from './file-metadata'; +import { mapObjectDetails } from './file-object-resolver'; export const app: Express = express(); app.disable('x-powered-by'); @@ -89,6 +90,8 @@ function routeFamily(req: Request): string { if (req.path === '/tool-call') return 'ptc-tool-call'; if (req.path.startsWith('/sessions/')) { if (req.method === 'PUT') return 'file-upload'; + if (req.method === 'POST' && req.path === '/input-manifest') return 'input-manifest'; + if (req.method === 'GET' && req.path.endsWith('/metadata')) return 'input-metadata'; if (req.method === 'GET' && req.path.includes('/objects/')) return 'file-download'; if (req.method === 'GET' && req.path.endsWith('/objects')) return 'file-list'; return 'file-unknown'; @@ -199,7 +202,7 @@ async function getGrant(req: Request, res: Response): Promise if (grant.legacy_grant) { await ensureEgressLedger(grant); } - await assertEgressGrantActive(grant); + await checkEgressGrantActive(grant); return grant; } @@ -438,7 +441,7 @@ async function restoreInternalSandboxResult(args: { if (grant.legacy_grant) { await ensureEgressLedger(grant); } - await assertEgressGrantActive(grant); + await checkEgressGrantActive(grant); const restored = restoreSandboxExecuteResult( args.result as Parameters[0], args.egressGrantToken, @@ -624,6 +627,88 @@ app.get('/sessions/:sessionHandle/objects', async (req, res) => { } }); +function inputMetadata( + metadata: { version?: unknown; size?: unknown; originalFilename?: unknown; readOnly?: unknown }, + grant: EgressGrantClaims, sessionId: string, objectId: string, +) { + if (typeof metadata.version !== 'string' || !/^[0-9a-f-]{36}$/.test(metadata.version) || + !Number.isSafeInteger(metadata.size) || (metadata.size as number) < 0) { + return { cacheable: false }; + } + const name = typeof metadata.originalFilename === 'string' ? metadata.originalFilename : undefined; + const readOnly = metadata.readOnly === true; + const cacheKey = crypto.createHash('sha256').update(JSON.stringify([ + 'authorized-http-input-v1', grant.tenant_id, grant.user_id, sessionId, objectId, + metadata.version, metadata.size, name, readOnly, + ])).digest('hex'); + return { cacheable: true, cacheKey, version: metadata.version, size: metadata.size, name, readOnly }; +} + +/** One budgeted HTTP read, with every handle checked before any storage access. + * Results belong only to this execution; the manifest is not an auth token. */ +app.post('/input-manifest', express.json({ limit: '4mb' }), async (req, res) => { + const controller = new AbortController(); + const cancel = () => controller.abort(); + const timeout = setTimeout(cancel, env.INPUT_MANIFEST_TIMEOUT_MS); + res.once('close', cancel); + try { + if (Object.keys(req.query).length || !Array.isArray(req.body?.files) || + req.body.files.length > env.INPUT_MANIFEST_MAX_FILES || + req.body.files.some((file: unknown) => !file || typeof file !== 'object' || + typeof (file as { sessionHandle?: unknown }).sessionHandle !== 'string' || + typeof (file as { objectHandle?: unknown }).objectHandle !== 'string')) { + return res.status(400).json({ error: 'Invalid input manifest' }); + } + const grant = await getGrant(req, res); + const files: Array<{ sessionId: string; objectId: string }> = req.body.files.map((file: { sessionHandle: string; objectHandle: string }) => { + const sessionId = openSessionParam(file.sessionHandle, grant, 'read'); + const object = openObjectParam(file.objectHandle, grant, sessionId); + return { sessionId, objectId: object.id }; + }); + await recordEgressRead(grant); + async function* inputs() { yield* files; } + const metadata = await mapObjectDetails(inputs(), async ({ sessionId, objectId }) => { + controller.signal.throwIfAborted(); + const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, + `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(objectId)}/metadata`), + { headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); + if (!upstream.ok) { + await upstream.body?.cancel(); + // Recheck failures individually through the existing retry/classification path. + return { cacheable: false, retry: true }; + } + return inputMetadata(await upstream.json(), grant, sessionId, objectId); + }, env.INPUT_MANIFEST_CONCURRENCY); + // Do not publish a manifest after revocation/expiry during storage resolution. + await checkEgressGrantActive(grant); + return res.json({ files: metadata }); + } catch (error) { + return sendEgressError(req, res, error); + } finally { + clearTimeout(timeout); + res.removeListener('close', cancel); + controller.abort(); + } +}); + +/** Cache preflight is a scoped, budgeted read, never a reusable authorization grant. */ +app.get('/sessions/:sessionHandle/objects/:objectHandle/metadata', async (req, res) => { + try { + if (Object.keys(req.query).length) return res.status(400).json({ error: 'Metadata query parameters are not supported' }); + const grant = await getGrant(req, res); + const sessionId = openSessionParam(req.params.sessionHandle, grant, 'read'); + const object = openObjectParam(req.params.objectHandle, grant, sessionId); + await recordEgressRead(grant); + const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, + `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}/metadata`), + { headers: injectTraceHeaders(internalServiceHeaders()) }); + if (!upstream.ok) return pipeFetchResponse(upstream, res); + return res.json(inputMetadata(await upstream.json(), grant, sessionId, object.id)); + } catch (error) { + return sendEgressError(req, res, error); + } +}); + app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { try { if (Object.keys(req.query).length > 0) { @@ -633,12 +718,16 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { const sessionId = openSessionParam(req.params.sessionHandle, grant, 'read'); const object = openObjectParam(req.params.objectHandle, grant, sessionId); await recordEgressRead(grant); + const expectedVersion = req.header('x-codeapi-input-version'); + if (expectedVersion && !/^[0-9a-f-]{36}$/.test(expectedVersion)) { + return res.status(400).json({ error: 'Invalid input version' }); + } const upstream = await fetch( forwardUrl( env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}`, ), - { headers: injectTraceHeaders(internalServiceHeaders()) }, + { headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, ); const headerOverrides = isOpaqueObjectContentDisposition( upstream.headers.get('content-disposition'), diff --git a/service/src/egress-ledger-reconnect.test.ts b/service/src/egress-ledger-reconnect.test.ts new file mode 100644 index 00000000..0cf4bb78 --- /dev/null +++ b/service/src/egress-ledger-reconnect.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from 'bun:test'; +import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net'; +import IORedis from 'ioredis'; +import { env } from './config'; +import { startTestRedis } from './test/redis'; +import { + EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, recordEgressRead, + assertEgressGrantActive, setEgressLedgerRedisForTest, +} from './egress-ledger'; +import type { EgressGrantClaims } from './egress-grant'; + +test('a lost Redis mutation reply rejects without replaying its applied counter after reconnect', async () => { + const redis = await startTestRedis(); + const sockets = new Set(); + let dropReply = false; + const proxy = createServer(downstream => { + const upstream = createConnection(redis.options.path!); + sockets.add(downstream); sockets.add(upstream); + downstream.pipe(upstream); + upstream.on('data', data => { + if (dropReply) { + dropReply = false; + downstream.destroy(); upstream.destroy(); + } else downstream.write(data); + }); + downstream.on('error', () => {}); + upstream.on('error', () => downstream.destroy()); + downstream.on('close', () => { sockets.delete(downstream); upstream.destroy(); }); + upstream.on('close', () => { sockets.delete(upstream); downstream.destroy(); }); + }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const client = new IORedis({ host: '127.0.0.1', port: (proxy.address() as AddressInfo).port, + ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, retryStrategy: () => 10, lazyConnect: true }); + client.on('error', () => {}); + const required = env.EGRESS_LEDGER_REQUIRED; + const compact = env.EGRESS_LEDGER_COMPACT; + env.EGRESS_LEDGER_REQUIRED = true; + env.EGRESS_LEDGER_COMPACT = true; + setEgressLedgerRedisForTest(client); + try { + await client.connect(); + const now = Math.floor(Date.now() / 1000); + const grant: EgressGrantClaims = { v: 1, typ: 'grant', grant_id: 'reconnect', exec_id: 'exec', + tenant_id: 'tenant', user_id: 'user', session_key: 'session', input_files: [], read_sessions: [], + output_session_id: 'output', max_upload_bytes: 100, max_output_files: 10, max_requests: 10, iat: now, exp: now + 300 }; + await createEgressLedger(grant); + const reconnected = new Promise(resolve => client.once('ready', resolve)); + dropReply = true; + await expect(recordEgressRead(grant)).rejects.toThrow(); + await reconnected; + expect((await assertEgressGrantActive(grant)).request_count).toBe(1); + await recordEgressRead(grant); + expect((await assertEgressGrantActive(grant)).request_count).toBe(2); + } finally { + env.EGRESS_LEDGER_REQUIRED = required; + env.EGRESS_LEDGER_COMPACT = compact; + setEgressLedgerRedisForTest(null); + client.disconnect(); + for (const socket of sockets) socket.destroy(); + await new Promise(resolve => proxy.close(() => resolve())); + await redis.closeTestServer(); + } +}, 10000); diff --git a/service/src/egress-ledger-script.ts b/service/src/egress-ledger-script.ts new file mode 100644 index 00000000..c63346ad --- /dev/null +++ b/service/src/egress-ledger-script.ts @@ -0,0 +1,123 @@ +/** One-key transactions work on Redis Cluster and serialize admission with revocation. + * Legacy JSON is retained for mixed-version rollout; compact hashes avoid decoding + * the immutable input policy on the hot path. Never retry an ambiguous EVAL result: + * the operation may already have consumed its budget. */ +export const EGRESS_LEDGER_SCRIPT = ` +local key = KEYS[1] +local op = ARGV[1] +local kind = redis.call('TYPE', key) +if type(kind) == 'table' then kind = kind.ok end +local now = tonumber(ARGV[3]) +local function denied(message) return {'error', 'scope_mismatch', message} end +if op == 'create' then + if kind ~= 'none' then return {'ok'} end + local policy = cjson.decode(ARGV[4]) + if ARGV[5] == 'compact' then + redis.call('HSET', key, 'policy', ARGV[4], 'status', 'active', + 'exec_id', policy.exec_id, 'exp', policy.exp, + 'max_requests', policy.max_requests, 'max_upload_bytes', policy.max_upload_bytes, + 'max_output_files', policy.max_output_files, + 'request_count', 0, 'read_count', 0, 'upload_count', 0, 'tool_call_count', 0, 'uploaded_bytes', 0) + else + redis.call('SET', key, ARGV[4]) + end + redis.call('EXPIRE', key, tonumber(ARGV[6])) + return {'ok'} +end +if kind == 'none' then + if op == 'revoke' then return {'ok'} end + return denied('Egress grant ledger record is missing') +end +if kind ~= 'hash' and kind ~= 'string' then return denied('Invalid egress ledger representation') end +local compact = kind == 'hash' +local record = nil +if not compact then record = cjson.decode(redis.call('GET', key)) end +local function get(field) + if compact then return redis.call('HGET', key, field) end + return record[field] +end +local function number(field) return tonumber(get(field)) end +local function put(field, value) + if compact then redis.call('HSET', key, field, value) else record[field] = value end +end +local function add(field, value) + if compact then redis.call('HINCRBY', key, field, value) else record[field] = record[field] + value end +end +local function encodeRecord(value) + local encoded = cjson.encode(value) + -- Preserve the array contract for old gateways when cjson sees empty lists. + for _, field in ipairs({'input_files', 'read_sessions', 'output_file_ids'}) do + encoded = string.gsub(encoded, '"' .. field .. '":{}', '"' .. field .. '":[]') + end + return encoded +end +local function save() + if not compact then + -- Keep the original expiration, including a revocation tombstone's lifetime. + local ttl = redis.call('PTTL', key) + redis.call('SET', key, encodeRecord(record)) + if ttl >= 0 then redis.call('PEXPIRE', key, math.max(1, ttl)) end + end +end +if op == 'revoke' then + put('status', 'revoked') + put('revoked_at', now) + put('revoke_reason', ARGV[4]) + save() + return {'ok'} +end +if get('exec_id') ~= ARGV[2] then return denied('Egress grant ledger record does not match token') end +if get('status') ~= 'active' then return denied('Egress grant has been revoked') end +if number('exp') <= now then return {'error', 'expired', 'Egress grant is expired'} end +if op == 'check' then return {'ok'} end +if op == 'snapshot' then + if compact then + record = cjson.decode(redis.call('HGET', key, 'policy')) + for _, field in ipairs({'request_count', 'read_count', 'upload_count', 'tool_call_count', 'uploaded_bytes'}) do + record[field] = number(field) + end + record.output_file_ids = {} + local fields = redis.call('HKEYS', key) + for _, field in ipairs(fields) do + if string.sub(field, 1, 7) == 'output:' then table.insert(record.output_file_ids, string.sub(field, 8)) end + end + end + return {'ok', encodeRecord(record)} +end +local file = ARGV[4] +local bytes = tonumber(ARGV[5]) +local outputField = 'output:' .. file +local outputIndex = nil +if not compact and (op == 'reserve' or op == 'release') then + for i, id in ipairs(record.output_file_ids) do if id == file then outputIndex = i end end +end +local existing = compact and redis.call('HGET', key, outputField) or outputIndex +if op == 'release' then + -- A retried release must not refund another operation's request or byte budget. + if not existing then return {'ok'} end + local reservedBytes = compact and tonumber(existing) or bytes + if compact and reservedBytes ~= bytes then return denied('Upload release does not match reservation') end + add('uploaded_bytes', -math.min(number('uploaded_bytes'), reservedBytes)) + add('upload_count', -1) + add('request_count', -1) + if compact then redis.call('HDEL', key, outputField) else table.remove(record.output_file_ids, outputIndex) end +elseif op == 'reserve' or op == 'read' or op == 'tool' then + if number('request_count') >= number('max_requests') then return denied('Egress grant request budget exceeded') end + if op == 'reserve' then + local maxBytes = math.min(number('max_upload_bytes'), tonumber(ARGV[6])) + if not bytes or bytes < 0 or bytes ~= math.floor(bytes) or bytes > maxBytes then + return denied('Upload exceeds per-file egress byte limit') + end + if existing then return denied('Output file id has already been used for this grant') end + if number('upload_count') >= number('max_output_files') then return denied('Output file count budget exceeded') end + if number('uploaded_bytes') + bytes > maxBytes * number('max_output_files') then return denied('Aggregate upload byte budget exceeded') end + add('uploaded_bytes', bytes) + add('upload_count', 1) + if compact then redis.call('HSET', key, outputField, bytes) else table.insert(record.output_file_ids, file) end + elseif op == 'read' then add('read_count', 1) + else add('tool_call_count', 1) end + add('request_count', 1) +else return denied('Unknown egress ledger operation') end +save() +return {'ok'} +`; diff --git a/service/src/egress-ledger.test.ts b/service/src/egress-ledger.test.ts index 48b69d71..87fbec8d 100644 --- a/service/src/egress-ledger.test.ts +++ b/service/src/egress-ledger.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import RedisMock from 'ioredis-mock'; +import { startTestRedis } from './test/redis'; import { env } from './config'; import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; @@ -9,6 +9,9 @@ import { ensureEgressLedger, releaseEgressUpload, reserveEgressUpload, + recordEgressRead, + recordEgressToolCall, + checkEgressGrantActive, revokeEgressLedger, setEgressLedgerRedisForTest, } from './egress-ledger'; @@ -49,26 +52,30 @@ function expectEgressError(fn: () => Promise, reason: EgressGrantError[ ); } -describe('egress Redis ledger', () => { - let redis: InstanceType; +describe.each([false, true])('egress Redis ledger compact=%s', compact => { + let redis: Awaited>; let previousRequired: boolean; + let previousCompact: boolean; let previousMaxFileBytes: number; let previousTtlGraceSeconds: number; - beforeEach(() => { + beforeEach(async () => { previousRequired = env.EGRESS_LEDGER_REQUIRED; + previousCompact = env.EGRESS_LEDGER_COMPACT; + env.EGRESS_LEDGER_COMPACT = compact; previousMaxFileBytes = env.EGRESS_GATEWAY_MAX_FILE_BYTES; previousTtlGraceSeconds = env.EGRESS_LEDGER_TTL_GRACE_SECONDS; env.EGRESS_LEDGER_REQUIRED = true; env.EGRESS_GATEWAY_MAX_FILE_BYTES = 10; - redis = new RedisMock(); + redis = await startTestRedis(); setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); }); afterEach(async () => { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = previousRequired; + env.EGRESS_LEDGER_COMPACT = previousCompact; env.EGRESS_GATEWAY_MAX_FILE_BYTES = previousMaxFileBytes; env.EGRESS_LEDGER_TTL_GRACE_SECONDS = previousTtlGraceSeconds; }); @@ -103,7 +110,7 @@ describe('egress Redis ledger', () => { ); }); - test('clears Redis WATCH after rejected mutations so later valid updates can proceed', async () => { + test('leaves counters unchanged after a rejected mutation', async () => { const claims = grant({ max_output_files: 2, max_requests: 5 }); await createEgressLedger(claims); @@ -143,7 +150,7 @@ describe('egress Redis ledger', () => { await expectEgressError(() => assertEgressGrantActive(claims), 'scope_mismatch'); }); - test('keeps concurrent WATCH mutations isolated on dedicated Redis connections', async () => { + test('accounts concurrent operations without WATCH connections', async () => { const claims = grant({ max_output_files: 16, max_requests: 16, @@ -170,7 +177,7 @@ describe('egress Redis ledger', () => { ); const record = await assertEgressGrantActive(claims); - expect(duplicateCount).toBe(8); + expect(duplicateCount).toBe(0); expect(record.request_count).toBe(12); expect(record.upload_count).toBe(12); expect(record.uploaded_bytes).toBe(12); @@ -184,4 +191,60 @@ describe('egress Redis ledger', () => { redis.duplicate = duplicate as typeof redis.duplicate; } }); + test('admits exactly the request budget under a 240-file burst', async () => { + const claims = grant({ max_requests: 100, input_files: Array.from({ length: 240 }, (_, i) => ({ + id: `file_${i}`, session_id: 'inputs', name: `${i}.txt`, + })) }); + await createEgressLedger(claims); + const results = await Promise.allSettled(Array.from({ length: 240 }, () => recordEgressRead(claims))); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(100); + expect((await assertEgressGrantActive(claims)).request_count).toBe(100); + expect((await assertEgressGrantActive(claims)).read_count).toBe(100); + }); + + test('rejects wrong execution, expired grants and mutations after revocation', async () => { + const claims = grant({ max_requests: 1000 }); + await createEgressLedger(claims); + await expectEgressError(() => recordEgressRead({ ...claims, exec_id: 'wrong' }), 'scope_mismatch'); + await recordEgressToolCall(claims.grant_id, claims.exec_id); + await Promise.all([ + ...Array.from({ length: 40 }, () => recordEgressRead(claims).catch(() => {})), + revokeEgressLedger(claims.grant_id, 'done'), + ]); + await createEgressLedger(claims); + await expectEgressError(() => checkEgressGrantActive(claims), 'scope_mismatch'); + await expectEgressError(() => recordEgressRead(claims), 'scope_mismatch'); + const expired = grant({ grant_id: 'expired', exp: nowSeconds() - 1 }); + await createEgressLedger(expired); + await expectEgressError(() => checkEgressGrantActive(expired), 'expired'); + }); + + test('duplicate releases cannot refund another upload or read', async () => { + const claims = grant({ max_requests: 10, max_output_files: 2 }); + await createEgressLedger(claims); + await recordEgressRead(claims); + await reserveEgressUpload({ grant: claims, fileId: 'a', bytes: 3 }); + await reserveEgressUpload({ grant: claims, fileId: 'b', bytes: 4 }); + await Promise.all(Array.from({ length: 10 }, () => releaseEgressUpload({ grant: claims, fileId: 'a', bytes: 3 }))); + expect(await assertEgressGrantActive(claims)).toMatchObject({ + request_count: 2, upload_count: 1, uploaded_bytes: 4, output_file_ids: ['b'], + }); + }); + + test('format selection affects new grants only and never resets existing state', async () => { + const claims = grant(); + await createEgressLedger(claims); + await recordEgressRead(claims); + env.EGRESS_LEDGER_COMPACT = !compact; + await ensureEgressLedger(claims); + await recordEgressRead(claims); + expect(await redis.type(`codeapi:egress:grant:${claims.grant_id}`)).toBe(compact ? 'hash' : 'string'); + expect((await assertEgressGrantActive(claims)).request_count).toBe(2); + if (!compact) { + const legacy = JSON.parse((await redis.get(`codeapi:egress:grant:${claims.grant_id}`))!); + expect(legacy.output_file_ids).toEqual([]); + expect(Array.isArray(legacy.input_files)).toBe(true); + } + }); + }); diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index b6bbf4fb..c635dcc4 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -6,6 +6,7 @@ import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; import logger from './logger'; import { redisKeepAliveOptions } from './redis-options'; +import { EGRESS_LEDGER_SCRIPT } from './egress-ledger-script'; type LedgerStatus = 'active' | 'revoked'; @@ -30,21 +31,17 @@ export interface EgressLedgerRecord { output_file_ids: string[]; } -let redis: IORedis | null = null; -const LEDGER_MUTATION_ATTEMPTS = 32; -const LEDGER_MUTATION_POOL_SIZE = Math.max(1, Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32); - -type MutationConnectionWaiter = { - resolve: (client: IORedis) => void; - reject: (error: Error) => void; -}; - -const mutationConnections = new Set(); -let idleMutationConnections: IORedis[] = []; -let mutationConnectionWaiters: MutationConnectionWaiter[] = []; +/** A lost reply is ambiguous: never replay a possibly applied mutation. + * maxRetries=0 also rejects the pending promise on disconnect instead of leaving + * it unresolved when ioredis discards its unfulfilled-command queue. */ +export const EGRESS_LEDGER_REDIS_RETRY_OPTIONS = { + autoResendUnfulfilledCommands: false, + maxRetriesPerRequest: 0, +} as const; +let redis: IORedis | null = null; +const scriptClients = new WeakSet(); export function setEgressLedgerRedisForTest(client: IORedis | null): void { - resetMutationConnections(); redis = client; } @@ -66,7 +63,7 @@ function redisConnection(): IORedis { host: process.env.REDIS_HOST ?? 'redis', port: Number(process.env.REDIS_PORT) || 6379, password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: 1, + ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, retryStrategy, enableReadyCheck: true, connectTimeout: 10000, @@ -82,71 +79,6 @@ function redisConnection(): IORedis { return redis; } -function resetMutationConnections(): void { - const resetError = new Error('Egress ledger Redis connection reset'); - for (const waiter of mutationConnectionWaiters) { - waiter.reject(resetError); - } - mutationConnectionWaiters = []; - idleMutationConnections = []; - for (const client of mutationConnections) { - client.disconnect(); - } - mutationConnections.clear(); -} - -async function dedicatedMutationConnection(): Promise { - while (idleMutationConnections.length > 0) { - const client = idleMutationConnections.pop()!; - if (client.status !== 'end') { - return client; - } - mutationConnections.delete(client); - } - - if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { - return createMutationConnection(); - } - - return new Promise((resolve, reject) => { - mutationConnectionWaiters.push({ resolve, reject }); - }); -} - -function createMutationConnection(): IORedis { - const client = redisConnection().duplicate(); - mutationConnections.add(client); - client.on('error', error => logger.error('Egress ledger mutation Redis error', { error })); - return client; -} - -function releaseMutationConnection(client: IORedis): void { - if (!mutationConnections.has(client) || client.status === 'end') { - mutationConnections.delete(client); - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - try { - waiter.resolve(createMutationConnection()); - } catch (error) { - waiter.reject(error instanceof Error ? error : new Error(String(error))); - } - } - return; - } - - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - waiter.resolve(client); - return; - } - - idleMutationConnections.push(client); -} - -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - export async function pingEgressLedger(): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; await redisConnection().ping(); @@ -173,171 +105,79 @@ function recordFromGrant(grant: EgressGrantClaims): EgressLedgerRecord { }; } -export async function createEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); +async function executeLedger( + operation: string, + grantId: string, + executionId = '', + extra: Array = [], +): Promise { + const client = redisConnection() as IORedis & { + executeEgressLedger: (...args: Array) => Promise; + }; + if (!scriptClients.has(client)) { + client.defineCommand('executeEgressLedger', { numberOfKeys: 1, lua: EGRESS_LEDGER_SCRIPT }); + scriptClients.add(client); } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - ); + const result = await client.executeEgressLedger( + ledgerKey(grantId), operation, executionId, + Math.floor(Date.now() / 1000), ...extra, + ) as string[]; + if (result[0] === 'error') { + throw new EgressGrantError(result[1] as EgressGrantError['reason'], result[2]); + } + return result[1]; } -export async function ensureEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } +export async function createEgressLedger(grant: EgressGrantClaims): Promise { + if (!grant.grant_id) throw new EgressGrantError('malformed', 'Egress grant id is required'); if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - 'NX', - ); + await executeLedger('create', grant.grant_id, grant.exec_id, [ + JSON.stringify(recordFromGrant(grant)), env.EGRESS_LEDGER_COMPACT ? 'compact' : 'legacy', ttlSeconds(grant.exp), + ]); } -async function loadRecord(grantId: string): Promise { - const raw = await redisConnection().get(ledgerKey(grantId)); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - return JSON.parse(raw) as EgressLedgerRecord; -} +/** Admission is idempotent: neither replay nor rolling deployment resets budgets or revocation. */ +export const ensureEgressLedger = createEgressLedger; -function assertActive(record: EgressLedgerRecord, grant: Pick): void { - if (record.grant_id !== grant.grant_id || record.exec_id !== grant.exec_id) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record does not match token'); - } - if (record.status !== 'active') { - throw new EgressGrantError('scope_mismatch', 'Egress grant has been revoked'); - } - if (record.exp <= Math.floor(Date.now() / 1000)) { - throw new EgressGrantError('expired', 'Egress grant is expired'); - } -} - -async function mutateRecord( - grant: EgressGrantClaims, - mutate: (record: EgressLedgerRecord) => void, -): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) { - return recordFromGrant(grant); - } - const client = await dedicatedMutationConnection(); - const key = ledgerKey(grant.grant_id); - try { - for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { - await client.watch(key); - let record: EgressLedgerRecord; - try { - const raw = await client.get(key); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - record = JSON.parse(raw) as EgressLedgerRecord; - assertActive(record, grant); - mutate(record); - if (record.request_count > record.max_requests) { - throw new EgressGrantError('scope_mismatch', 'Egress grant request budget exceeded'); - } - } catch (error) { - await client.unwatch().catch(unwatchError => { - logger.warn('Failed to clear egress ledger WATCH after rejected mutation', { error: unwatchError }); - }); - throw error; - } - const result = await client.multi() - .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) - .exec(); - if (result) return record; - if (i < LEDGER_MUTATION_ATTEMPTS - 1) { - await sleep(Math.min(25, i + 1)); - } - } - } finally { - await client.unwatch().catch(error => { - logger.warn('Failed to clear egress ledger WATCH before returning mutation connection', { error }); - }); - releaseMutationConnection(client); - } - throw new EgressGrantError('ledger_conflict', 'Egress grant ledger update conflicted'); +/** Authorization hot path deliberately does not return the potentially large input policy. */ +export async function checkEgressGrantActive(grant: Pick): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + await executeLedger('check', grant.grant_id, grant.exec_id); } export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); - const record = await loadRecord(grant.grant_id); - assertActive(record, grant); + const record = JSON.parse((await executeLedger('snapshot', grant.grant_id, grant.exec_id))!) as EgressLedgerRecord; + // Redis cjson represents empty Lua arrays as objects. + if (!Array.isArray(record.output_file_ids)) record.output_file_ids = []; + if (!Array.isArray(record.input_files)) record.input_files = []; + if (!Array.isArray(record.read_sessions)) record.read_sessions = []; return record; } export async function recordEgressRead(grant: EgressGrantClaims): Promise { - await mutateRecord(grant, record => { - record.request_count += 1; - record.read_count += 1; - }); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await executeLedger('read', grant.grant_id, grant.exec_id, ['', 0]); } -export async function reserveEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; -}): Promise { - await mutateRecord(args.grant, record => { - if (args.bytes > Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES)) { - throw new EgressGrantError('scope_mismatch', 'Upload exceeds per-file egress byte limit'); - } - if (record.output_file_ids.includes(args.fileId)) { - throw new EgressGrantError('scope_mismatch', 'Output file id has already been used for this grant'); - } - if (record.output_file_ids.length >= record.max_output_files) { - throw new EgressGrantError('scope_mismatch', 'Output file count budget exceeded'); - } - const aggregateLimit = Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) * record.max_output_files; - if (record.uploaded_bytes + args.bytes > aggregateLimit) { - throw new EgressGrantError('scope_mismatch', 'Aggregate upload byte budget exceeded'); - } - record.request_count += 1; - record.upload_count += 1; - record.uploaded_bytes += args.bytes; - record.output_file_ids.push(args.fileId); - }); +export async function reserveEgressUpload(args: { grant: EgressGrantClaims; fileId: string; bytes: number }): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + if (!Number.isSafeInteger(args.bytes) || args.bytes < 0) throw new EgressGrantError('scope_mismatch', 'Invalid upload byte count'); + await executeLedger('reserve', args.grant.grant_id, args.grant.exec_id, [args.fileId, args.bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES]); } -export async function releaseEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; -}): Promise { +export async function releaseEgressUpload(args: { grant: EgressGrantClaims; fileId: string; bytes: number }): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; - await mutateRecord(args.grant, record => { - record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); - record.upload_count = Math.max(0, record.upload_count - 1); - record.request_count = Math.max(0, record.request_count - 1); - record.output_file_ids = record.output_file_ids.filter(id => id !== args.fileId); - }); + if (!Number.isSafeInteger(args.bytes) || args.bytes < 0) throw new EgressGrantError('scope_mismatch', 'Invalid upload byte count'); + await executeLedger('release', args.grant.grant_id, args.grant.exec_id, [args.fileId, args.bytes]); } export async function recordEgressToolCall(grantId: string | undefined, executionId: string): Promise { if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; - const grant = { grant_id: grantId, exec_id: executionId } as EgressGrantClaims; - await mutateRecord(grant, record => { - record.request_count += 1; - record.tool_call_count += 1; - }); + await executeLedger('tool', grantId, executionId, ['', 0]); } export async function revokeEgressLedger(grantId: string, reason: string): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; - const key = ledgerKey(grantId); - const raw = await redisConnection().get(key); - if (!raw) return; - const record = JSON.parse(raw) as EgressLedgerRecord; - record.status = 'revoked'; - record.revoked_at = Math.floor(Date.now() / 1000); - record.revoke_reason = reason; - await redisConnection().set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)); + await executeLedger('revoke', grantId, '', [reason]); } diff --git a/service/src/file-download.test.ts b/service/src/file-download.test.ts new file mode 100644 index 00000000..cb32d771 --- /dev/null +++ b/service/src/file-download.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from 'bun:test'; +import { Readable, Writable } from 'node:stream'; +import { createServer } from 'node:http'; +import express from 'express'; +import { sendFileDownload } from './file-download'; + +async function serverFor(stream: Readable) { + const app = express(); + app.get('/', (req, res) => { void sendFileDownload(stream, res, req.header('x-codeapi-input-version')).catch(() => res.destroy()); }); + const server = createServer(app); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as { port: number }; + return { url: `http://127.0.0.1:${address.port}`, close: () => new Promise(resolve => server.close(() => resolve())) }; +} + +test('serves metadata from the downloaded version and rejects a stale preflight', async () => { + for (const expected of ['current', 'stale']) { + const stream = Object.assign(Readable.from(['bytes']), { headers: { + 'x-amz-meta-codeapi-version': 'current', 'x-amz-meta-read-only': 'true', + 'x-amz-meta-original-filename': 'verified.txt', + } }); + const server = await serverFor(stream); + try { + const response = await fetch(server.url, { headers: { 'X-CodeAPI-Input-Version': expected } }); + expect(response.status).toBe(expected === 'current' ? 200 : 409); + if (expected === 'current') { + expect(response.headers.get('x-read-only')).toBe('true'); + expect(response.headers.get('content-disposition')).toContain('verified.txt'); + expect(await response.text()).toBe('bytes'); + } else await response.text(); + } finally { await server.close(); } + } +}); + +test('downstream cancellation stops the storage stream under backpressure', async () => { + let produced = 0; + const stream = new Readable({ read() { if (++produced <= 1000) this.push(Buffer.alloc(64 * 1024)); else this.push(null); } }); + const response = Object.assign(new Writable({ + highWaterMark: 1, + write(_chunk, _encoding, callback) { setTimeout(callback, 10); }, + }), { setHeader() {} }); + const transfer = sendFileDownload(stream, response as unknown as express.Response); + const timer = setTimeout(() => response.destroy(), 25); + try { + await expect(transfer).rejects.toThrow(); + expect(stream.destroyed).toBe(true); + expect(produced).toBeLessThan(1000); + } finally { clearTimeout(timer); } +}); diff --git a/service/src/file-download.ts b/service/src/file-download.ts new file mode 100644 index 00000000..c46f80a1 --- /dev/null +++ b/service/src/file-download.ts @@ -0,0 +1,27 @@ +import type { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { Response } from 'express'; +import { contentDispositionForOriginalFilename, originalFilenameFromMetadata } from './file-metadata'; + +export async function sendFileDownload(dataStream: Readable, res: Response, expectedVersion?: string): Promise { + // MinIO returns the HTTP response stream. Read metadata from this exact GET, + // avoiding both a redundant HEAD and metadata/content races on overwrite. + const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; + if (expectedVersion && headers['x-amz-meta-codeapi-version'] !== expectedVersion) { + dataStream.destroy(); + res.status(409).json({ error: 'Input changed during preparation; retry with current metadata' }); + return; + } + const metadata: Record = {}; + for (const [key, value] of Object.entries(headers)) { + if (key.startsWith('x-amz-meta-')) metadata[key.slice(11)] = value; + } + res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilenameFromMetadata(metadata))); + if (headers['content-type']) res.setHeader('Content-Type', headers['content-type']); + if (headers['content-length']) res.setHeader('Content-Length', headers['content-length']); + if (metadata['read-only'] === 'true') res.setHeader('X-Read-Only', 'true'); + if (metadata['codeapi-version']) res.setHeader('X-CodeAPI-Input-Version', metadata['codeapi-version']); + const cancel = (): void => { if (!res.writableFinished) dataStream.destroy(new Error('Download client disconnected')); }; + res.once('close', cancel); + try { await pipeline(dataStream, res); } finally { res.off('close', cancel); } +} diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts new file mode 100644 index 00000000..deeb3265 --- /dev/null +++ b/service/src/file-object-resolver.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'bun:test'; +import { FileObjectResolver, mapObjectDetails } from './file-object-resolver'; +import type { BucketItemStat } from 'minio'; + +describe('storage object resolution', () => { + test('indexes exact identities while reading fresh version metadata on every request', async () => { + const index = new Map(); + let lists = 0; + let heads = 0; + let version = 'first'; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { lists++; yield { name: 's/identifier.txt' }; yield { name: 's/id.txt' }; }, + stat: async key => { heads++; expect(key).toBe('s/id.txt'); return { size: 5, etag: 'etag', lastModified: new Date(), metaData: { 'codeapi-version': version } } as BucketItemStat; }, + index: { get: async k => index.get(k) ?? null, set: async (k, v) => index.set(k, v), forget: async k => index.delete(k) }, + }); + expect((await resolver.metadata('s', 'id'))?.stat.metaData['codeapi-version']).toBe('first'); + version = 'second'; + expect((await resolver.metadata('s', 'id'))?.stat.metaData['codeapi-version']).toBe('second'); + expect(lists).toBe(1); + expect(heads).toBe(2); + }); + + test('ignores foreign-session index entries and does not cache absence', async () => { + let present = false; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's2/id.txt' }; if (present) yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { get: async () => 's2/id.txt', set: async () => {}, forget: async () => {} }, + }); + expect(await resolver.resolve('s', 'id')).toBeUndefined(); + present = true; + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + }); +}); + +test('metadata listing stays bounded and ordered across 240 objects', async () => { + let active = 0; + let maximum = 0; + async function* objects() { for (let i = 0; i < 240; i++) yield i; } + const result = await mapObjectDetails(objects(), async value => { + maximum = Math.max(maximum, ++active); + await new Promise(resolve => setTimeout(resolve, value % 3)); + active--; + return value; + }, 8); + expect(maximum).toBe(8); + expect(result).toEqual(Array.from({ length: 240 }, (_, i) => i)); +}); diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts new file mode 100644 index 00000000..a41e0dfc --- /dev/null +++ b/service/src/file-object-resolver.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import type { BucketItemStat } from 'minio'; + +export interface ObjectResolverDependencies { + bucket: string; + list(prefix: string): AsyncIterable<{ name?: string }>; + stat(key: string): Promise; + index?: { + get(key: string): Promise; + set(key: string, value: string, replace: boolean): Promise; + forget(key: string, value: string): Promise; + }; +} + +/** Storage-key index is a hint, never metadata or authorization. A fresh HEAD + * proves existence and supplies the current version even on index/cache hits. */ +export class FileObjectResolver { + constructor(private readonly deps: ObjectResolverDependencies) {} + + private indexKey(session: string, id: string): string { + return `codeapi:file-key:${createHash('sha256').update(JSON.stringify([this.deps.bucket, session, id])).digest('hex')}`; + } + + private matches(key: string, session: string, id: string): boolean { + return path.posix.dirname(key) === session && + (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); + } + + async remember(session: string, id: string, key: string, replace = true): Promise { + if (!this.matches(key, session, id)) throw new Error('Object key does not match storage identity'); + await this.deps.index?.set(this.indexKey(session, id), key, replace); + } + + async resolve(session: string, id: string): Promise { + const cached = await this.deps.index?.get(this.indexKey(session, id)); + if (cached && this.matches(cached, session, id)) return cached; + for await (const object of this.deps.list(`${session}/${id}`)) { + if (object.name && this.matches(object.name, session, id)) { + await this.remember(session, id, object.name, false); + return object.name; + } + } + return undefined; + } + + async metadata(session: string, id: string): Promise<{ key: string; stat: BucketItemStat } | undefined> { + const key = await this.resolve(session, id); + if (!key) return undefined; + try { + return { key, stat: await this.deps.stat(key) }; + } catch (error) { + if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; + await this.deps.index?.forget(this.indexKey(session, id), key); + // Do not cache absence: a later upload can publish this identity again. + return undefined; + } + } +} + +/** Bound storage metadata requests while preserving listing order. */ +export async function mapObjectDetails(objects: AsyncIterable, describe: (object: T) => Promise, concurrency: number): Promise { + const results: R[] = []; + const batch: T[] = []; + const width = Math.max(1, Math.min(64, Math.floor(concurrency) || 1)); + for await (const object of objects) { + batch.push(object); + if (batch.length === width) { + results.push(...await Promise.all(batch.map(describe))); + batch.length = 0; + } + } + results.push(...await Promise.all(batch.map(describe))); + return results; +} diff --git a/service/src/file-server.ts b/service/src/file-server.ts index f9293e48..9b326ca2 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,4 +1,8 @@ import b from 'busboy'; +import { randomUUID } from 'node:crypto'; +import { mapObjectDetails } from './file-object-resolver'; +import { sendFileDownload } from './file-download'; +import { FileObjectResolver } from './file-object-resolver'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; @@ -18,7 +22,6 @@ import logger from './fileServerLogger'; import { env } from './config'; import { redisKeepAliveOptions } from './redis-options'; import { - contentDispositionForOriginalFilename, decodeOriginalFilename, originalFilenameFromMetadata, } from './file-metadata'; @@ -144,6 +147,21 @@ redisClient.on('ready', () => { logger.info('Redis Client Ready'); }); +const objectResolver = new FileObjectResolver({ + bucket: bucketName, + list: prefix => minioClient.listObjects(bucketName, prefix, true), + stat: key => minioClient.statObject(bucketName, key), + ...(env.FILE_OBJECT_INDEX_ENABLED ? { index: { + get: (key: string) => redisClient.get(key), + set: (key: string, value: string, replace: boolean) => replace + ? redisClient.set(key, value, 'EX', env.SESSION_CACHE_TTL) + : redisClient.set(key, value, 'EX', env.SESSION_CACHE_TTL, 'NX'), + forget: (key: string, value: string) => redisClient.eval( + "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0", 1, key, value, + ), + } } : {}), +}); + const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { @@ -250,6 +268,8 @@ async function uploadFile( * `getObject` / `statObject` without a separate Redis lookup. */ const metaData: Record = { 'Content-Type': mimetype, + // New marker on every PUT, including same-ID overwrites and metadata changes. + 'X-Amz-Meta-Codeapi-Version': randomUUID(), 'X-Amz-Meta-Original-Filename': encodedFilename, 'X-Amz-Meta-Original-Filename-Encoded': 'base64', }; @@ -267,6 +287,7 @@ async function uploadFile( } else { await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); } + await objectResolver.remember(session_id, fileId, objectName); logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); fileUploads.inc(); @@ -443,30 +464,14 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => const { session_id, objectId } = req.params; try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - }); - } - - const stat: Partial = await minioClient.statObject(bucketName, objectName); + const resolved = await objectResolver.metadata(session_id, objectId); + if (!resolved) return res.status(404).json({ error: 'File not found' }); + const { key: objectName, stat } = resolved; const originalFilename = originalFilenameFromMetadata(stat.metaData); return res.status(200).json({ name: objectName, + version: stat.metaData?.['codeapi-version'], ...(originalFilename ? { originalFilename } : {}), size: stat.size, lastModified: stat.lastModified, @@ -487,87 +492,33 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const { session_id, objectId } = req.params; try { - // List objects to find the correct file with extension - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - logger.warn('File not found', { session_id, objectId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - bucketName - }); - } - - logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); - - const stat: Partial = await minioClient.statObject(bucketName, objectName); - - const originalFilename = originalFilenameFromMetadata(stat.metaData); - - logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); - - // Explicitly remove problematic headers that might be duplicated - res.removeHeader('Transfer-Encoding'); - res.removeHeader('Date'); - - /* An object-key basename is only a storage identifier, not an original - * filename. If an S3-compatible backend drops user metadata, retain - * attachment semantics but omit the filename so the runner uses its - * caller-supplied destination. */ - res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilename)); - if (stat.metaData?.['content-type'] != null) { - res.setHeader('Content-Type', stat.metaData['content-type']); - } - /* Surface the read-only flag on download so the sandbox can plumb it - * onto its in-memory file metadata without a separate metadata fetch. - * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ - if (stat.metaData?.['read-only'] === 'true') { - res.setHeader('X-Read-Only', 'true'); - } - + const objectName = await objectResolver.resolve(session_id, objectId); + if (!objectName) return res.status(404).json({ error: 'File not found' }); const dataStream = await minioClient.getObject(bucketName, objectName); - fileDownloads.inc(); - - dataStream.on('data', (chunk) => { - res.write(chunk); - }); - - dataStream.on('end', () => { - res.end(); - }); - - dataStream.on('error', (err) => { - logger.error('Error streaming file:', { error: err, session_id, objectId, bucketName }); - // Only send error if headers haven't been sent yet - if (!res.headersSent) { - res.status(500).json({ - error: 'Error streaming file', - details: err.message - }); - } else { - res.end(); + try { + const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; + if (!headers['x-amz-meta-codeapi-version'] || !headers['x-amz-meta-original-filename']) { + // Preserve legacy/S3-compatible metadata behavior without promoting a + // later HEAD's version marker onto bytes from an earlier GET. + const stat = await minioClient.statObject(bucketName, objectName); + if (headers.etag?.replace(/^"|"$/g, '') !== stat.etag) { + return res.status(409).json({ error: 'Input changed during metadata lookup' }); + } + for (const [key, value] of Object.entries(stat.metaData ?? {})) { + if (key !== 'codeapi-version') headers[`x-amz-meta-${key}`] ??= value; + } } - }); + fileDownloads.inc(); + await sendFileDownload(dataStream, res, req.header('x-codeapi-input-version')); + } finally { + dataStream.destroy(); + } } catch (err) { - logger.error('Error downloading file:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error downloading file', - details: (err as Error | undefined)?.message, - session_id, - objectId, - bucketName - }); + logger.error('Error downloading file', { error: err, session_id, objectId }); + if (!res.headersSent && !res.destroyed) { + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); + return res.status(missing ? 404 : 500).json({ error: 'Error downloading file' }); + } } }); @@ -585,7 +536,7 @@ function parseObjectName(objectName: string | undefined): { session_id: string; return { session_id, file_id }; } -const detailLevels: Record Promise> | undefined> = { +const detailLevels: Record Promise>> = { simple: async (obj: BucketItem): Promise> => obj.name ?? '', summary: async (obj: BucketItem): Promise> => ({ name: obj.name, @@ -639,14 +590,9 @@ app.get('/sessions/:session_id/objects', async (req, res) => { const { detail = 'simple' } = req.query; try { - const stream = minioClient.listObjects(bucketName, session_id, true); - const objects: (t.ObjectTypes | Partial | undefined)[] = []; - + const stream = minioClient.listObjects(bucketName, `${session_id}/`, true); const getDetail = detailLevels[detail as string] ?? detailLevels.simple; - - for await (const obj of stream) { - objects.push(await getDetail(obj)); - } + const objects = await mapObjectDetails(stream, getDetail, env.FILE_METADATA_CONCURRENCY); res.json(objects); } catch (err) { diff --git a/service/src/test/redis.ts b/service/src/test/redis.ts new file mode 100644 index 00000000..f243f657 --- /dev/null +++ b/service/src/test/redis.ts @@ -0,0 +1,45 @@ +import { spawn } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import IORedis from 'ioredis'; + +/** Real Lua semantics, isolated Unix socket, no TCP listener or durable data. */ +export async function startTestRedis(): Promise }> { + const dir = await mkdtemp(path.join(tmpdir(), 'codeapi-redis-')); + const socket = path.join(dir, 'redis.sock'); + const process = spawn('redis-server', [ + '--port', '0', '--unixsocket', socket, '--unixsocketperm', '700', + '--save', '', '--appendonly', 'no', '--dir', dir, + ], { stdio: 'ignore' }); + let failure: Error | undefined; + process.on('error', error => { failure = error; }); + const exited = new Promise(resolve => { + process.once('exit', () => resolve()); + process.once('error', () => resolve()); + }); + const client = new IORedis(socket, { lazyConnect: true, retryStrategy: () => null, maxRetriesPerRequest: 0 }); + client.on('error', () => {}); + const closeTestServer = async (): Promise => { + client.disconnect(); + process.kill('SIGTERM'); + await exited; + await rm(dir, { recursive: true, force: true }); + }; + try { + for (let attempt = 0; attempt < 100; attempt++) { + if (failure) throw failure; + try { + await client.connect(); + await client.ping(); + return Object.assign(client, { closeTestServer }); + } catch { + await new Promise(resolve => setTimeout(resolve, 20)); + } + } + throw new Error('Test Redis did not start; install redis-server'); + } catch (error) { + await closeTestServer(); + throw error; + } +} From a992ec0b4bf4c4a46caf9ef38b49f0a07010eb6b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:45:37 -0400 Subject: [PATCH 081/116] =?UTF-8?q?=F0=9F=A7=B9=20fix:=20Evict=20Stale=20F?= =?UTF-8?q?ile-Object=20Index=20Entries=20(#182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: evict stale file-object index entries Forget cached locators after successful deletion and missing-object downloads so replacement keys resolve immediately. Reuse exact resolver matching in the delete route to avoid prefix collisions. Fixes #181 * fix: keep file-object deletion storage-authoritative * fix: retire superseded upload objects * fix: canonicalize replacement object keys * fix: collapse legacy object-key siblings * fix: recover reads from stale locators * fix: namespace canonical object identities --- service/src/file-object-resolver.test.ts | 143 ++++++++++++++++++++++- service/src/file-object-resolver.ts | 124 +++++++++++++++++--- service/src/file-server.ts | 74 +++++++++--- 3 files changed, 306 insertions(+), 35 deletions(-) diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts index deeb3265..2342d87a 100644 --- a/service/src/file-object-resolver.test.ts +++ b/service/src/file-object-resolver.test.ts @@ -1,8 +1,27 @@ import { describe, expect, test } from 'bun:test'; -import { FileObjectResolver, mapObjectDetails } from './file-object-resolver'; +import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; import type { BucketItemStat } from 'minio'; describe('storage object resolution', () => { + test('replacement uploads converge on one stable object key', () => { + expect(storageKeyForUpload('s', 'id', '.csv', true)).toBe('s/.codeapi-objects/aWQ'); + expect(storageKeyForUpload('s', 'id', '.pdf', true)).toBe('s/.codeapi-objects/aWQ'); + expect(canonicalObjectId(storageKeyForUpload('s', 'report.csv', '', true))).toBe('report.csv'); + expect(storageKeyForUpload('s', 'generated', '.csv', false)).toBe('s/generated.csv'); + }); + + test('canonical dotted identities cannot match another legacy identity', async () => { + const dottedKey = storageKeyForUpload('s', 'report.csv', '', true); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: dottedKey }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + }); + + expect(await resolver.listFresh('s', 'report.csv')).toEqual([dottedKey]); + expect(await resolver.listFresh('s', 'report')).toEqual([]); + }); + test('indexes exact identities while reading fresh version metadata on every request', async () => { const index = new Map(); let lists = 0; @@ -33,6 +52,128 @@ describe('storage object resolution', () => { present = true; expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); }); + + test('falls back to storage when the advisory index is unavailable', async () => { + const failures: string[] = []; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + onIndexError: operation => failures.push(operation), + index: { + get: async () => { throw new Error('redis unavailable'); }, + set: async () => { throw new Error('redis unavailable'); }, + forget: async () => { throw new Error('redis unavailable'); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + expect(await resolver.listFresh('s', 'id')).toEqual(['s/id.txt']); + expect(failures).toEqual(['get', 'set']); + }); + + test('fresh listing ignores a stale locator and returns every exact sibling', async () => { + const index = new Map([['locator', 's/id.txt']]); + let lists = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { + lists++; + yield { name: 's/identifier.txt' }; + yield { name: 's/id.csv' }; + yield { name: 's/id.pdf' }; + }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async () => index.get('locator') ?? null, + set: async (_key, value) => index.set('locator', value), + forget: async (_key, value) => { if (index.get('locator') === value) index.delete('locator'); }, + }, + }); + + expect(await resolver.listFresh('s', 'id')).toEqual(['s/id.csv', 's/id.pdf']); + expect(index.get('locator')).toBe('s/id.txt'); + expect(lists).toBe(2); + }); + + test('metadata re-resolves storage after a cached locator is missing', async () => { + const index = new Map([['locator', 's/id.txt']]); + let heads = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's/.codeapi-objects/aWQ' }; }, + stat: async key => { + heads++; + if (key === 's/id.txt') throw Object.assign(new Error('missing'), { code: 'NoSuchKey' }); + return { + size: 1, + etag: 'current', + lastModified: new Date(), + metaData: { 'codeapi-version': 'current' }, + } as BucketItemStat; + }, + index: { + get: async () => index.get('locator') ?? null, + set: async (_key, value) => index.set('locator', value), + forget: async (_key, value) => { if (index.get('locator') === value) index.delete('locator'); }, + }, + }); + + const metadata = await resolver.metadata('s', 'id'); + expect(metadata?.key).toBe('s/.codeapi-objects/aWQ'); + expect(metadata?.stat.metaData['codeapi-version']).toBe('current'); + expect(index.get('locator')).toBe('s/.codeapi-objects/aWQ'); + expect(heads).toBe(2); + }); + + test('forgets a deleted locator so a replacement key for the same identity resolves', async () => { + const index = new Map(); + const stored = new Set(['s/id.txt']); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* (prefix) { for (const name of stored) if (name.startsWith(prefix)) yield { name }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async k => index.get(k) ?? null, + set: async (k, v, replace) => { if (replace || !index.has(k)) index.set(k, v); }, + forget: async (k, v) => { if (index.get(k) === v) index.delete(k); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + stored.delete('s/id.txt'); + await resolver.forget('s', 'id', 's/id.txt'); + expect(index.size).toBe(0); + + stored.add('s/id.csv'); + expect(await resolver.resolve('s', 'id')).toBe('s/id.csv'); + }); + + test('eviction is scoped to the identity and never drops a newer cached key', async () => { + const index = new Map(); + let lists = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { lists++; yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async k => index.get(k) ?? null, + set: async (k, v, replace) => { if (replace || !index.has(k)) index.set(k, v); }, + forget: async (k, v) => { if (index.get(k) === v) index.delete(k); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + // A concurrent upload republished the identity before the delete evicted it. + await resolver.remember('s', 'id', 's/id.csv'); + await resolver.forget('s', 'id', 's/id.txt'); + // Keys outside the identity can never reach its entry. + await resolver.forget('s', 'id', 's2/id.txt'); + await resolver.forget('s', 'id', 's/other.txt'); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.csv'); + expect(lists).toBe(1); + }); }); test('metadata listing stays bounded and ordered across 240 objects', async () => { diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts index a41e0dfc..bc4b4ce5 100644 --- a/service/src/file-object-resolver.ts +++ b/service/src/file-object-resolver.ts @@ -2,10 +2,28 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import type { BucketItemStat } from 'minio'; +const CANONICAL_OBJECT_DIRECTORY = '.codeapi-objects'; + +export function canonicalObjectKey(session: string, id: string): string { + return `${session}/${CANONICAL_OBJECT_DIRECTORY}/${Buffer.from(id, 'utf8').toString('base64url')}`; +} + +export function canonicalObjectId(key: string): string | undefined { + const parts = key.split('/'); + if (parts.length !== 3 || parts[1] !== CANONICAL_OBJECT_DIRECTORY || parts[2] === '') return undefined; + try { + const id = Buffer.from(parts[2], 'base64url').toString('utf8'); + return canonicalObjectKey(parts[0], id) === key ? id : undefined; + } catch { + return undefined; + } +} + export interface ObjectResolverDependencies { bucket: string; list(prefix: string): AsyncIterable<{ name?: string }>; stat(key: string): Promise; + onIndexError?(operation: 'get' | 'set' | 'forget', error: unknown): void; index?: { get(key: string): Promise; set(key: string, value: string, replace: boolean): Promise; @@ -13,48 +31,122 @@ export interface ObjectResolverDependencies { }; } +/** Caller-supplied identities keep one stable storage key across replacement + * filenames. This gives concurrent PUTs one last-writer-wins S3 object without + * requiring a distributed lock or leaving extension-keyed siblings behind. */ +export function storageKeyForUpload( + session: string, + id: string, + extension: string, + replacing: boolean, +): string { + return replacing ? canonicalObjectKey(session, id) : `${session}/${id}${extension}`; +} + /** Storage-key index is a hint, never metadata or authorization. A fresh HEAD * proves existence and supplies the current version even on index/cache hits. */ export class FileObjectResolver { constructor(private readonly deps: ObjectResolverDependencies) {} + private reportIndexError(operation: 'get' | 'set' | 'forget', error: unknown): void { + this.deps.onIndexError?.(operation, error); + } + private indexKey(session: string, id: string): string { return `codeapi:file-key:${createHash('sha256').update(JSON.stringify([this.deps.bucket, session, id])).digest('hex')}`; } private matches(key: string, session: string, id: string): boolean { + if (key === canonicalObjectKey(session, id)) return true; return path.posix.dirname(key) === session && (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); } async remember(session: string, id: string, key: string, replace = true): Promise { if (!this.matches(key, session, id)) throw new Error('Object key does not match storage identity'); - await this.deps.index?.set(this.indexKey(session, id), key, replace); + try { + await this.deps.index?.set(this.indexKey(session, id), key, replace); + } catch (error) { + this.reportIndexError('set', error); + } } - async resolve(session: string, id: string): Promise { - const cached = await this.deps.index?.get(this.indexKey(session, id)); - if (cached && this.matches(cached, session, id)) return cached; - for await (const object of this.deps.list(`${session}/${id}`)) { - if (object.name && this.matches(object.name, session, id)) { - await this.remember(session, id, object.name, false); - return object.name; + /** Evict a cached locator once its object is known to be gone. Scoped to the + * requested identity, and conditional on the stored value so a replacement + * key published concurrently for the same identity is never dropped. */ + async forget(session: string, id: string, key: string): Promise { + if (!this.matches(key, session, id)) return; + try { + await this.deps.index?.forget(this.indexKey(session, id), key); + } catch (error) { + this.reportIndexError('forget', error); + } + } + + private async cached(session: string, id: string): Promise { + try { + const key = await this.deps.index?.get(this.indexKey(session, id)); + return key && this.matches(key, session, id) ? key : undefined; + } catch (error) { + this.reportIndexError('get', error); + return undefined; + } + } + + private async findInStorage(session: string, id: string, replaceIndex: boolean): Promise { + for (const prefix of [canonicalObjectKey(session, id), `${session}/${id}`]) { + for await (const object of this.deps.list(prefix)) { + if (object.name && this.matches(object.name, session, id)) { + await this.remember(session, id, object.name, replaceIndex); + return object.name; + } } } return undefined; } + /** List every exact storage key for an identity without consulting its + * locator. Used to collapse legacy siblings and delete the whole identity. */ + async listFresh(session: string, id: string): Promise { + const keys = new Set(); + for (const prefix of [canonicalObjectKey(session, id), `${session}/${id}`]) { + for await (const object of this.deps.list(prefix)) { + if (object.name && this.matches(object.name, session, id)) keys.add(object.name); + } + } + return [...keys]; + } + + async resolve(session: string, id: string): Promise { + return await this.cached(session, id) ?? await this.findInStorage(session, id, false); + } + + /** Recover once a cached key is proven missing. Eviction and publication are + * advisory; the authoritative storage listing determines the replacement. */ + async recover(session: string, id: string, missingKey: string): Promise { + await this.forget(session, id, missingKey); + const [current] = await this.listFresh(session, id); + if (current) await this.remember(session, id, current); + return current; + } + async metadata(session: string, id: string): Promise<{ key: string; stat: BucketItemStat } | undefined> { - const key = await this.resolve(session, id); + let key = await this.resolve(session, id); if (!key) return undefined; - try { - return { key, stat: await this.deps.stat(key) }; - } catch (error) { - if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; - await this.deps.index?.forget(this.indexKey(session, id), key); - // Do not cache absence: a later upload can publish this identity again. - return undefined; + for (let attempt = 0; attempt < 2; attempt++) { + try { + return { key, stat: await this.deps.stat(key) }; + } catch (error) { + if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; + if (attempt === 1) { + await this.forget(session, id, key); + return undefined; + } + key = await this.recover(session, id, key); + if (!key) return undefined; + } } + return undefined; } } diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 9b326ca2..3dfa0a10 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,8 +1,7 @@ import b from 'busboy'; import { randomUUID } from 'node:crypto'; -import { mapObjectDetails } from './file-object-resolver'; +import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; import { sendFileDownload } from './file-download'; -import { FileObjectResolver } from './file-object-resolver'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; @@ -151,6 +150,7 @@ const objectResolver = new FileObjectResolver({ bucket: bucketName, list: prefix => minioClient.listObjects(bucketName, prefix, true), stat: key => minioClient.statObject(bucketName, key), + onIndexError: (operation, error) => logger.warn('File-object index operation failed', { operation, error }), ...(env.FILE_OBJECT_INDEX_ENABLED ? { index: { get: (key: string) => redisClient.get(key), set: (key: string, value: string, replace: boolean) => replace @@ -162,6 +162,16 @@ const objectResolver = new FileObjectResolver({ } } : {}), }); +/** Index eviction is best effort: the index is only a hint, so a Redis failure + * must never turn a completed delete or a missing-object 404 into a 500. */ +async function forgetObjectKey(session_id: string, objectId: string, objectName: string): Promise { + try { + await objectResolver.forget(session_id, objectId, objectName); + } catch (error) { + logger.warn('Failed to evict file-object index entry', { error, session_id, objectId, objectName }); + } +} + const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { @@ -257,7 +267,14 @@ async function uploadFile( ): Promise { const fileId = existingFileId ?? nanoid(); const fileExtension = path.extname(filename); - const objectName = `${session_id}/${fileId}${fileExtension}`; + // Caller-supplied identities use one canonical key, so concurrent writers + // converge on S3's last-writer semantics regardless of filename extension. + const objectName = storageKeyForUpload( + session_id, + fileId, + fileExtension, + existingFileId != null, + ); const encodedFilename = Buffer.from(filename).toString('base64'); @@ -287,6 +304,15 @@ async function uploadFile( } else { await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); } + if (existingFileId != null) { + // Retire every extension-keyed sibling left by older replacement behavior. + // Concurrent replacement writers share objectName and never delete it. + for (const sibling of await objectResolver.listFresh(session_id, fileId)) { + if (sibling === objectName) continue; + await minioClient.removeObject(bucketName, sibling); + await objectResolver.forget(session_id, fileId, sibling); + } + } await objectResolver.remember(session_id, fileId, objectName); logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); @@ -490,11 +516,21 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const { session_id, objectId } = req.params; + let objectName: string | undefined; try { - const objectName = await objectResolver.resolve(session_id, objectId); + objectName = await objectResolver.resolve(session_id, objectId); if (!objectName) return res.status(404).json({ error: 'File not found' }); - const dataStream = await minioClient.getObject(bucketName, objectName); + let dataStream: Readable; + try { + dataStream = await minioClient.getObject(bucketName, objectName); + } catch (error) { + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? ''); + if (!missing) throw error; + objectName = await objectResolver.recover(session_id, objectId, objectName); + if (!objectName) return res.status(404).json({ error: 'File not found' }); + dataStream = await minioClient.getObject(bucketName, objectName); + } try { const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; if (!headers['x-amz-meta-codeapi-version'] || !headers['x-amz-meta-original-filename']) { @@ -515,8 +551,11 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { } } catch (err) { logger.error('Error downloading file', { error: err, session_id, objectId }); + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); + // A locator that no longer names bytes must not shadow a replacement object + // published for the same identity until the index TTL expires. + if (missing && objectName) await forgetObjectKey(session_id, objectId, objectName); if (!res.headersSent && !res.destroyed) { - const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); return res.status(missing ? 404 : 500).json({ error: 'Error downloading file' }); } } @@ -527,6 +566,10 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { */ function parseObjectName(objectName: string | undefined): { session_id: string; file_id: string } | null { if (objectName == null || objectName === '') return null; + const canonicalId = canonicalObjectId(objectName); + if (canonicalId != null) { + return { session_id: objectName.split('/', 1)[0], file_id: canonicalId }; + } const parts = objectName.split('/'); if (parts.length < 2) return null; const session_id = parts[0]; @@ -605,17 +648,9 @@ app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { const { session_id, fileId } = req.params; try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${fileId}`, true); - let objectName = ''; + const objectNames = await objectResolver.listFresh(session_id, fileId); - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { + if (objectNames.length === 0) { logger.warn('File not found for deletion', { session_id, fileId, bucketName }); return res.status(404).json({ error: 'File not found', @@ -626,8 +661,11 @@ app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { }); } - await minioClient.removeObject(bucketName, objectName); - logger.info(`[${INSTANCE_ID}] File deleted successfully: ${objectName}`); + for (const objectName of objectNames) { + await minioClient.removeObject(bucketName, objectName); + await forgetObjectKey(session_id, fileId, objectName); + } + logger.info(`[${INSTANCE_ID}] File identity deleted successfully`, { session_id, fileId, objectNames }); return res.status(200).json({ message: 'File deleted successfully', session_id, From c3fd558195c8c4513977b83963c6aad495a4874d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:45:48 -0400 Subject: [PATCH 082/116] perf: enable authorized input reuse by default (#183) --- api/src/config-defaults.test.ts | 21 +++++++++++++++++++++ api/src/config.ts | 3 ++- docs/INPUT_REUSE.md | 8 ++++---- helm/codeapi/values.yaml | 2 +- 4 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 api/src/config-defaults.test.ts diff --git a/api/src/config-defaults.test.ts b/api/src/config-defaults.test.ts new file mode 100644 index 00000000..281f5679 --- /dev/null +++ b/api/src/config-defaults.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from 'bun:test'; +import path from 'node:path'; + +function readHttpInputCacheDefault(value?: string): boolean { + const env = { ...process.env }; + if (value === undefined) delete env.SANDBOX_HTTP_INPUT_CACHE_ENABLED; + else env.SANDBOX_HTTP_INPUT_CACHE_ENABLED = value; + const script = [ + `const { config } = await import(${JSON.stringify(path.resolve(process.cwd(), 'src/config.ts'))})`, + 'process.stdout.write(JSON.stringify(config.http_input_cache_enabled))', + ].join(';'); + const result = Bun.spawnSync({ cmd: [process.execPath, '--eval', script], env }); + expect(result.exitCode).toBe(0); + return JSON.parse(result.stdout.toString()) as boolean; +} + +test('authorized HTTP input reuse defaults on and retains an explicit rollback switch', () => { + expect(readHttpInputCacheDefault()).toBe(true); + expect(readHttpInputCacheDefault('false')).toBe(false); + expect(readHttpInputCacheDefault('true')).toBe(true); +}); diff --git a/api/src/config.ts b/api/src/config.ts index f3b935ff..a5d0c993 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -122,7 +122,8 @@ export const config = { /* Ceiling for the pushed input cache (session-inputs.ts). Eviction is * always safe — a miss simply re-pushes on the next probe — so this is a * disk guard, not a correctness knob. */ - http_input_cache_enabled: process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED === 'true', + http_input_cache_enabled: + (process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED ?? 'true') === 'true', http_input_cache_max_objects: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS, 4096), http_input_cache_max_inflight: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT, 16), input_cache_max_bytes: safeInt( diff --git a/docs/INPUT_REUSE.md b/docs/INPUT_REUSE.md index 40e277c4..f87afa89 100644 --- a/docs/INPUT_REUSE.md +++ b/docs/INPUT_REUSE.md @@ -48,7 +48,7 @@ sequenceDiagram | `egressGrant.inputManifestTimeoutMs` | `CODEAPI_INPUT_MANIFEST_TIMEOUT_MS` | `10000` | | `fileServer.objectIndexEnabled` | `CODEAPI_FILE_OBJECT_INDEX_ENABLED` | `false` | | `fileServer.metadataConcurrency` | `CODEAPI_FILE_METADATA_CONCURRENCY` | `1` | -| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `false` | +| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `true` | | `workerSandbox.sandbox.httpInputCacheMaxInflight` | `SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT` | `16` | | `workerSandbox.sandbox.httpInputCacheMaxObjects` | `SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS` | `4096` | | `workerSandbox.sandbox.inputCacheMaxBytes` | `SANDBOX_INPUT_CACHE_MAX_BYTES` | `536870912` | @@ -61,14 +61,14 @@ Metadata listing concurrency preserves order and is capped at 64. A canary can u ## Rollout and rollback -1. Deploy the new binaries with feature flags off. Atomic accounting supports existing JSON ledgers, and legacy downloads retain metadata compatibility. The new file server stamps future uploads with versions. +1. Deploy the new binaries with HTTP input reuse enabled by default. Mixed-version requests remain compatible: older gateways and relays fall back to normal downloads, while older unversioned objects return `cacheable: false`. The new file server stamps future uploads with versions. Set `workerSandbox.sandbox.httpInputCacheEnabled=false` only when a staged rollout requires the immediate rollback path. 2. Update **all** egress-gateway replicas before enabling compact ledgers. New binaries read both formats regardless of the creation flag. Older binaries cannot read compact hashes. To roll back to an older binary, disable compact creation, drain active grants, and wait their maximum TTL plus grace; never delete active ledgers to force a rollback. 3. Update all file-server writers before enabling the object-key index. Otherwise an older writer can change a locator without updating the index. Keep file-server replicas consistent during an indexed rollout. -4. Update the gateway, relay, runner, and launcher before enabling HTTP reuse on a small runner canary. Older gateway/relay metadata routes return 404/405 and fall back safely. Older files return `cacheable: false`. Keep the feature disabled for storage adapters that cannot return user metadata on GET. +4. Canary the default-on HTTP reuse path after updating the gateway, relay, runner, and launcher. Keep the feature explicitly disabled for storage adapters that cannot return user metadata on GET. 5. Observe `codeapi_sandbox_http_input_cache_events_total` (bounded event labels, no identities), cold and warm preparation latency, storage/Redis operations, admission fairness, request budgets, and memory/disk pressure before widening the rollout. A successful manifest consumes one read request for the batch, matching the existing list-request accounting unit. Per-file compatibility preflights each consume a read request; each cold miss consumes an additional download request. Do not disable budget enforcement to accommodate a workload. 6. Disable HTTP reuse to return to normal downloads immediately. Cached files can age out normally; no workspace deletion or migration is needed. -The creation flags default off. No deployment or object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. +HTTP input reuse defaults on. Compact-ledger creation and the object-key index remain off until their mixed-version rollout requirements are satisfied. No object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. ## Validation diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 3ecfbef8..bafed305 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -273,7 +273,7 @@ workerSandbox: # Defaults to maxConcurrentJobs when unset. jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 - httpInputCacheEnabled: false + httpInputCacheEnabled: true httpInputCacheMaxInflight: 16 httpInputCacheMaxObjects: 4096 inputCacheMaxBytes: 536870912 From 76129e15e5f5a62d193074524459c80708c0d741 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:58:40 -0400 Subject: [PATCH 083/116] fix: honor requested input destinations (#184) --- api/openapi.yaml | 1 + api/src/download.test.ts | 111 +++++++++-------------- api/src/job-helpers.test.ts | 169 +++--------------------------------- api/src/job.ts | 86 ++++-------------- 4 files changed, 67 insertions(+), 300 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index e4c04c14..23e1224c 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -67,6 +67,7 @@ components: properties: name: type: string + description: Relative path where the file is mounted in the sandbox. id: type: string content: diff --git a/api/src/download.test.ts b/api/src/download.test.ts index da877dad..d0b6ed6e 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -18,12 +18,8 @@ import { /** * Integration tests for `Job.downloadAndWriteFile` against a real HTTP - * listener. These exercise the cross-repo round-trip: the file-server - * (codeapi/service) emits `Content-Disposition: attachment; - * filename*=UTF-8''` for nested artifacts, and the - * sandbox-side parser must recover the path so the file lands at the same - * nested location on the next prime(). Hitting a real listener (not a - * mocked Response) catches anything fetch-level that a unit test would miss. + * listener. Hitting a real listener verifies that response metadata cannot + * redirect a caller-validated sandbox destination. */ interface DownloadInternals { @@ -176,21 +172,16 @@ afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); }); -describe('downloadAndWriteFile / RFC 5987 round-trip', () => { - it('writes a nested-path artifact at the encoded location', async () => { - /* Simulates the matplotlib-bug shape: codeapi previously returned a - * flat `file.name` and the original path was carried only by the - * server's `filename*=` header. The fix is: parser recovers the path, - * `mkdir { recursive: true }` creates the parent dir, file ends up - * where the user expects to `cat` it on the next turn. */ +describe('downloadAndWriteFile destinations', () => { + it('writes a nested-path artifact at the requested location', async () => { const file: TFile = { id: 'nested-id', storage_session_id: 'prev-session', - name: 'flat-fallback.txt', + name: 'proj/notes.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, - contentDisposition: "attachment; filename*=UTF-8''proj%2Fnotes.txt", + contentDisposition: "attachment; filename*=UTF-8''stored-original.txt", body: 'hello from a nested artifact\n', }); @@ -212,7 +203,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { const file: TFile = { id: 'opaque-object-handle', storage_session_id: 'opaque-session-handle', - name: 'gateway-fallback.txt', + name: 'gateway.txt', }; let sawGrantHeader = false; let sawRelayToken = false; @@ -275,14 +266,11 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect((await fsp.stat(path.join(tmpDir, 'readonly.txt'))).mode & 0o777).toBe(SANDBOX_READONLY_FILE_MODE); }); - it('falls back to the legacy filename= form when filename*= is absent', async () => { - /* Backwards-compat: older file-servers (or proxies that strip RFC - * 5987 extended-form headers) still send the legacy quoted form. The - * parser must still find a name and write the file. */ + it('downloads when a legacy filename header matches the requested name', async () => { const file: TFile = { id: 'legacy-id', storage_session_id: 'prev-session', - name: 'ignored.txt', + name: 'legacy.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -323,7 +311,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.stat(path.join(tmpDir, 'opaque-storage-id.xlsx')).catch(() => null)).toBeNull(); }); - it('resolves concurrent header destinations without provisional-name false conflicts', async () => { + it('keeps concurrent inputs at their requested destinations', async () => { const renamed: TFile = { id: 'renamed-id', storage_session_id: 'prev-session', @@ -338,8 +326,6 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { status: 200, contentDisposition: 'attachment; filename="actual.txt"', body: 'renamed bytes', - /* Make the other ref resolve `vacated.txt` while this ref's requested - * name would still be provisional under the old reservation scheme. */ delayMs: 75, }); routes.set(`/sessions/${encodeURIComponent(replacement.storage_session_id!)}/objects/${encodeURIComponent(replacement.id!)}`, { @@ -356,75 +342,62 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { await job.prime(); const submissionDir = asInternals(job).submissionDir; expect(await fsp.readFile(path.join(submissionDir, 'actual.txt'), 'utf8')) - .toBe('renamed bytes'); - expect(await fsp.readFile(path.join(submissionDir, 'vacated.txt'), 'utf8')) .toBe('replacement bytes'); + expect(await fsp.readFile(path.join(submissionDir, 'vacated.txt'), 'utf8')) + .toBe('renamed bytes'); } finally { config.prime_concurrency = originalPrimeConcurrency; await job.cleanup(); } }); - it('rejects concurrent refs that resolve to the same destination before either can overwrite', async () => { - const slower: TFile = { - id: 'same-slower-id', + it('keeps distinct requested names when stored objects share an original filename', async () => { + const original: TFile = { + id: 'original-id', storage_session_id: 'prev-session', - name: 'slower-fallback.txt', + name: 'data.xlsx', }; - const faster: TFile = { - id: 'same-faster-id', + const aliased: TFile = { + id: 'aliased-id', storage_session_id: 'prev-session', - name: 'faster-fallback.txt', + name: 'data-3f9a2c.xlsx', }; - routes.set(`/sessions/${encodeURIComponent(slower.storage_session_id!)}/objects/${encodeURIComponent(slower.id!)}`, { + routes.set(`/sessions/${encodeURIComponent(original.storage_session_id!)}/objects/${encodeURIComponent(original.id!)}`, { status: 200, - contentDisposition: 'attachment; filename="same.txt"', - body: 'slower bytes', + contentDisposition: 'attachment; filename="data.xlsx"', + body: 'original bytes', delayMs: 75, }); - routes.set(`/sessions/${encodeURIComponent(faster.storage_session_id!)}/objects/${encodeURIComponent(faster.id!)}`, { + routes.set(`/sessions/${encodeURIComponent(aliased.storage_session_id!)}/objects/${encodeURIComponent(aliased.id!)}`, { status: 200, - contentDisposition: 'attachment; filename="same.txt"', - body: 'faster bytes', + contentDisposition: 'attachment; filename="data.xlsx"', + body: 'aliased bytes', }); - let dirty = false; const job = makeJob( - [slower, faster], - sessionWorkspaceAt(tmpDir, 'rt_concurrent_same_destination', () => { dirty = true; }), + [original, aliased], + sessionWorkspaceAt(tmpDir, 'rt_shared_original_filename'), ); const originalPrimeConcurrency = config.prime_concurrency; config.prime_concurrency = 2; - let deadlockTimer: ReturnType | undefined; try { - const outcome = await Promise.race([ - job.prime().then( - () => ({ status: 'fulfilled' as const }), - error => ({ status: 'rejected' as const, error }), - ), - new Promise<{ status: 'timeout' }>(resolve => { - deadlockTimer = setTimeout(() => resolve({ status: 'timeout' }), 2_000); - }), - ]); - if (deadlockTimer) clearTimeout(deadlockTimer); - expect(outcome.status).toBe('rejected'); - if (outcome.status === 'rejected') { - expect(outcome.error).toBeInstanceOf(SessionWorkspaceDirtyError); - } - expect(dirty).toBe(true); - expect(await fsp.readFile(path.join(tmpDir, 'same.txt'), 'utf8')).toBe('faster bytes'); + await job.prime(); + const submissionDir = asInternals(job).submissionDir; + expect(await fsp.readFile(path.join(submissionDir, 'data.xlsx'), 'utf8')) + .toBe('original bytes'); + expect(await fsp.readFile(path.join(submissionDir, 'data-3f9a2c.xlsx'), 'utf8')) + .toBe('aliased bytes'); } finally { - if (deadlockTimer) clearTimeout(deadlockTimer); config.prime_concurrency = originalPrimeConcurrency; await job.cleanup(); } }); - it('decodes UTF-8 percent-encoded names with non-ASCII characters', async () => { + it('keeps a Unicode requested name when the header is percent encoded', async () => { const file: TFile = { id: 'utf8-id', storage_session_id: 'prev-session', - name: 'ignored.txt', + name: '你好.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -645,11 +618,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { await expect(fsp.access(path.join(tmpDir, 'should-not-exist.txt'))).rejects.toThrow(); }); - it('rejects a server-supplied filename that escapes the submission dir', async () => { - /* Companion guarantee for the path-preserving sanitizer on the - * LibreChat side: if a malicious / misconfigured server tries to - * smuggle a `..` traversal via Content-Disposition, the codeapi-side - * `validateFilePath` aborts before any write happens. */ + it('ignores a server-supplied filename that escapes the submission dir', async () => { const file: TFile = { id: 'evil-id', storage_session_id: 'prev-session', @@ -658,16 +627,14 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, contentDisposition: "attachment; filename*=UTF-8''..%2F..%2Fescape.txt", - body: 'should never be written', + body: 'safe bytes', }); const job = makeJob([file]); asInternals(job).submissionDir = tmpDir; - /* downloadAndWriteFile rethrows ValidationError fast (no retries) so - * `expect(...).rejects` is the right assertion. */ - await expect(job.downloadAndWriteFile(file)).rejects.toThrow(); - /* Defensive: nothing escaped to a parent dir. */ + await expect(job.downloadAndWriteFile(file)).resolves.toBe('innocent.txt'); + expect(await fsp.readFile(path.join(tmpDir, 'innocent.txt'), 'utf8')).toBe('safe bytes'); const parent = path.dirname(tmpDir); await expect(fsp.access(path.join(parent, 'escape.txt'))).rejects.toThrow(); }); diff --git a/api/src/job-helpers.test.ts b/api/src/job-helpers.test.ts index 0c31f9d3..460902da 100644 --- a/api/src/job-helpers.test.ts +++ b/api/src/job-helpers.test.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { - resolveOriginalName, + resolveInputDestination, isNormalizedObjectForSession, markerConflictsWithExplicitFile, aggregateBashExtras, @@ -41,168 +41,19 @@ function makeRuntime(overrides: Partial & { language: string; pkgdir: s }; } -describe('resolveOriginalName', () => { - function responseWithHeader(value?: string): Response { - const headers = new Headers(); - if (value !== undefined) headers.set('content-disposition', value); - return new Response(null, { headers }); - } - - it('returns file.name when no Content-Disposition is present', () => { - expect( - resolveOriginalName(responseWithHeader(), { name: 'script.py', id: 'abc' }), - ).toBe('script.py'); - }); - - it('extracts quoted filename from Content-Disposition', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="server-name.py"'), - { name: 'client-name.py', id: 'abc' }, - ), - ).toBe('server-name.py'); - }); - - it('extracts unquoted filename from Content-Disposition', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=plain.txt'), - { name: 'ignored.txt', id: 'abc' }, - ), - ).toBe('plain.txt'); - }); - - it('falls back to file.id when file.name is empty and no header exists', () => { - expect( - resolveOriginalName(responseWithHeader(), { name: '', id: 'file-id-123' }), - ).toBe('file-id-123'); - }); - - it('falls back to file.name when header is malformed (no filename token)', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment'), - { name: 'fallback.py', id: 'abc' }, - ), - ).toBe('fallback.py'); - }); - - it('stops at the closing quote when the quoted filename is followed by more params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="foo.txt"; size=123'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('stops at a semicolon when the unquoted filename is followed by more params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=foo.txt; size=123'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('stops at whitespace when the unquoted filename is followed by whitespace-separated params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=foo.txt extra'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('returns empty string when both name and id are absent', () => { - expect(resolveOriginalName(responseWithHeader(), { name: '' })).toBe(''); +describe('resolveInputDestination', () => { + it('uses the caller-requested sandbox path', () => { + expect(resolveInputDestination({ name: 'nested/script.py', id: 'abc' })) + .toBe('nested/script.py'); }); - it('returns empty string when name is empty, id is absent, and header is malformed', () => { - expect( - resolveOriginalName(responseWithHeader('attachment'), { name: '' }), - ).toBe(''); - }); - - it('decodes RFC 5987 filename*= preserving slashes for nested artifact paths', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''test_folder%2Ftest_file.txt"), - { name: 'test_file.txt', id: 'abc' }, - ), - ).toBe('test_folder/test_file.txt'); - }); - - it('decodes RFC 5987 filename*= with a UTF-8 charset that includes a language tag', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8'en'foo%20bar.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo bar.txt'); - }); - - it('decodes RFC 5987 filename*= with non-ASCII characters', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''%E4%BD%A0%E5%A5%BD.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('你好.txt'); - }); - - it('tolerates a filename*= form missing the UTF-8 prefix', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename*=plain.txt'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('plain.txt'); + it('falls back to the object id when no name exists', () => { + expect(resolveInputDestination({ name: '', id: 'file-id-123' })) + .toBe('file-id-123'); }); - it('falls through to legacy filename= when filename*= is malformed', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''bad%ZZ; filename=\"legacy.txt\""), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('legacy.txt'); - }); - - it('prefers filename*= over a legacy filename= present in the same header', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename=\"legacy.txt\"; filename*=UTF-8''nested%2Ffile.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('nested/file.txt'); - }); - - it('keeps the requested name when an old file server advertises the opaque object basename', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''storage-id.xlsx"), - { name: 'Sample_-_Superstore.xlsx', id: 'storage-id' }, - ), - ).toBe('Sample_-_Superstore.xlsx'); - }); - - it('keeps the requested name for a legacy opaque filename header', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="storage-id.csv"'), - { name: 'original.csv', id: 'storage-id' }, - ), - ).toBe('original.csv'); - }); - - it('keeps an authoritative nested filename even when its basename matches the object id', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''exports%2Fstorage-id.csv"), - { name: 'original.csv', id: 'storage-id' }, - ), - ).toBe('exports/storage-id.csv'); + it('returns an empty path when both name and id are absent', () => { + expect(resolveInputDestination({ name: '' })).toBe(''); }); }); diff --git a/api/src/job.ts b/api/src/job.ts index d2504de9..b7b82148 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -173,52 +173,12 @@ export function ensureNodeModulesSymlink( } /** - * Extracts the on-disk filename from a Content-Disposition response header, - * falling back to the request-supplied `file.name` (or `file.id` if no name - * was provided). Pure; exported for unit testing. - * - * Matches RFC 5987 / 8187 `filename*=UTF-8''` first because - * the file server emits that form for UTF-8-safe transport of arbitrary - * names — including paths with `/` separators that the legacy `filename=` - * form would mangle. Falls back to the legacy quoted (`filename="..."`) or - * unquoted (`filename=...`) forms, each stopping at the closing quote or - * the first whitespace/semicolon so trailing params like - * `attachment; filename="foo.txt"; size=123` correctly yield `foo.txt`. + * Resolves the on-disk destination for a by-reference input. The request owns + * the sandbox path; object response metadata must not redirect the write. + * Pure; exported for unit testing. */ -export function resolveOriginalName(response: Response, file: TFile): string { - const fallback = file.name || (file.id ?? ''); - const header = response.headers.get('content-disposition'); - if (!header) return fallback; - - const preferRequestedName = (candidate: string): string => { - /* Older file servers advertised path.basename(objectName) when an - * S3-compatible backend omitted original-filename user metadata. That - * basename is ``, so it is a storage identifier rather - * than an authoritative destination. Preserve the caller's requested name - * during rolling upgrades instead of exposing the opaque id in /mnt/data. */ - const opaqueStem = path.basename(candidate, path.extname(candidate)); - const isFlatObjectBasename = candidate === path.basename(candidate); - return file.name && file.id && isFlatObjectBasename && opaqueStem === file.id - ? file.name - : candidate; - }; - - const star = header.match(/filename\*=(?:UTF-8'[^']*')?([^;]+)/i); - if (star) { - const raw = star[1].trim(); - try { - return preferRequestedName(decodeURIComponent(raw)); - } catch { - /* Malformed percent-encoding (e.g. `%ZZ`) — fall through to the legacy - * forms. The same header may emit both `filename*=` and a legacy - * `filename=` per RFC 5987 §4.3, so a corrupt extended form should - * not poison a valid fallback. */ - } - } - - const match = header.match(/filename="([^"]+)"/i) - ?? header.match(/filename=([^\s;]+)/i); - return match ? preferRequestedName(match[1]) : fallback; +export function resolveInputDestination(file: TFile): string { + return file.name || (file.id ?? ''); } /** @@ -959,12 +919,9 @@ export class Job { ); } requestedDestinations.set(file.name, file); - /* Inline destinations are final, so keep them reserved while reference - * downloads resolve their authoritative Content-Disposition names. - * A ref's requested name is only a fallback, not a real destination yet: - * reserving every ref here makes concurrent swaps/order-dependent - * renames falsely conflict before the owning response has resolved. */ - if (!file.id) this.inputDestinations.set(file.name, file); + /* The request owns every sandbox destination. Reserve it before parallel + * priming begins so object metadata cannot redirect a later write. */ + this.inputDestinations.set(file.name, file); } if (this.session) { @@ -1083,10 +1040,8 @@ export class Job { ): Promise { throwIfAborted(context.signal); if (this.session && file.id && (await this.reusePrimedInput(file, context))) { - /* Reuse has no response header to pass through downloadAndWriteFile, so - * its requested name becomes authoritative only after the on-disk copy - * has been verified. Reserve it before another concurrent ref can claim - * and overwrite that path. */ + /* Inherited markers are registered after prime's initial reservation + * pass, so reserve the verified requested path here as well. */ this.reserveInputDestination(file, file.name); return; } @@ -1345,10 +1300,10 @@ export class Job { throw new Error(`HTTP error: ${response.status}`); } - const originalName = resolveOriginalName(response, file); - validateFilePath(originalName, operation.submissionDir); - this.reserveInputDestination(file, originalName); - const finalPath = path.join(operation.submissionDir, originalName); + const destination = resolveInputDestination(file); + validateFilePath(destination, operation.submissionDir); + this.reserveInputDestination(file, destination); + const finalPath = path.join(operation.submissionDir, destination); const finalParent = path.dirname(finalPath); /* Persistent-session workspaces can hold a prior turn's symlink, so build * ancestors no-follow; a fresh per-job workspace can use plain mkdir -p. */ @@ -1376,7 +1331,7 @@ export class Job { operation.signal, ); const readOnly = response.headers.get('x-read-only')?.toLowerCase() === 'true'; - this.inputFileHashes.set(originalName, { + this.inputFileHashes.set(destination, { originalId: file.id, originalSessionId: file.storage_session_id!, hash, @@ -1389,15 +1344,8 @@ export class Job { await applyReadOnlyInputPermissions(finalPath); } - /* Keep the in-memory TFile in sync with the on-disk name so that - * inputByName lookups in handleSessionFiles match walkDir's - * path.relative() output. Otherwise a Content-Disposition override - * would leave file.name pointing at the client-submitted name while - * the file lives under originalName on disk. */ - if (originalName !== file.name) file.name = originalName; - - this.log.info({ file: originalName, hash: hash.substring(0, 8) }, 'Downloaded file'); - return originalName; + this.log.info({ file: destination, hash: hash.substring(0, 8) }, 'Downloaded file'); + return destination; } catch (error: unknown) { if (response?.body && !response.bodyUsed) { await response.body.cancel().catch(() => {}); From 9d3936fa73e1b447f2e9a3a3961cdef8b411da18 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 08:39:20 -0400 Subject: [PATCH 084/116] fix: disambiguate legacy dotted object identities (#186) --- service/src/file-object-resolver.test.ts | 37 +++++++++++++++++++++++- service/src/file-object-resolver.ts | 15 ++++++++-- service/src/file-server.ts | 13 ++++++--- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts index 2342d87a..69ad3c2f 100644 --- a/service/src/file-object-resolver.test.ts +++ b/service/src/file-object-resolver.test.ts @@ -1,5 +1,12 @@ import { describe, expect, test } from 'bun:test'; -import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; +import { + canonicalObjectId, + canonicalObjectKey, + FileObjectResolver, + legacyObjectId, + mapObjectDetails, + storageKeyForUpload, +} from './file-object-resolver'; import type { BucketItemStat } from 'minio'; describe('storage object resolution', () => { @@ -22,6 +29,34 @@ describe('storage object resolution', () => { expect(await resolver.listFresh('s', 'report')).toEqual([]); }); + test('legacy extension keys map to exactly one dotted or undotted identity', async () => { + const canonicalDotted = canonicalObjectKey('s', 'report.csv'); + const objects = new Set([ + 's/report.csv', + 's/report.csv.txt', + canonicalDotted, + ]); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* (prefix) { + for (const name of objects) if (name.startsWith(prefix)) yield { name }; + }, + stat: async () => ({ metaData: {} } as BucketItemStat), + }); + + expect(legacyObjectId('s/report.csv', 's')).toBe('report'); + expect(legacyObjectId('s/report.csv.txt', 's')).toBe('report.csv'); + expect(legacyObjectId('s/report', 's')).toBe('report'); + expect(legacyObjectId('other/report.csv', 's')).toBeUndefined(); + await expect(resolver.remember('s', 'report.csv', 's/report.csv')) + .rejects.toThrow('Object key does not match storage identity'); + expect(await resolver.listFresh('s', 'report')).toEqual(['s/report.csv']); + expect(await resolver.listFresh('s', 'report.csv')).toEqual([ + canonicalDotted, + 's/report.csv.txt', + ]); + }); + test('indexes exact identities while reading fresh version metadata on every request', async () => { const index = new Map(); let lists = 0; diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts index bc4b4ce5..1665941a 100644 --- a/service/src/file-object-resolver.ts +++ b/service/src/file-object-resolver.ts @@ -19,6 +19,18 @@ export function canonicalObjectId(key: string): string | undefined { } } +/** Legacy objects were stored as `/`. + * Derive exactly one identity by removing that final extension when present. + * In particular, `session/report.csv` belongs to `report`, while a legacy + * `report.csv` identity would be stored as e.g. `session/report.csv.txt`. + * Dotted identities without a filename extension use canonical storage. */ +export function legacyObjectId(key: string, session: string): string | undefined { + if (path.posix.dirname(key) !== session) return undefined; + const basename = path.posix.basename(key); + const extension = path.posix.extname(basename); + return extension === '' ? basename : basename.slice(0, -extension.length); +} + export interface ObjectResolverDependencies { bucket: string; list(prefix: string): AsyncIterable<{ name?: string }>; @@ -58,8 +70,7 @@ export class FileObjectResolver { private matches(key: string, session: string, id: string): boolean { if (key === canonicalObjectKey(session, id)) return true; - return path.posix.dirname(key) === session && - (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); + return legacyObjectId(key, session) === id; } async remember(session: string, id: string, key: string, replace = true): Promise { diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 3dfa0a10..22302042 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,6 +1,12 @@ import b from 'busboy'; import { randomUUID } from 'node:crypto'; -import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; +import { + canonicalObjectId, + FileObjectResolver, + legacyObjectId, + mapObjectDetails, + storageKeyForUpload, +} from './file-object-resolver'; import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; @@ -573,9 +579,8 @@ function parseObjectName(objectName: string | undefined): { session_id: string; const parts = objectName.split('/'); if (parts.length < 2) return null; const session_id = parts[0]; - const fileNameWithExt = parts[1]; - // Remove extension to get file_id - const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); + const file_id = legacyObjectId(objectName, session_id); + if (file_id == null) return null; return { session_id, file_id }; } From 3946ffb6e8664c152a589f37dd8c0feb4dccb6f8 Mon Sep 17 00:00:00 2001 From: "Ignaz \"Ian\" Kraft" Date: Sat, 12 Sep 2026 20:26:48 +0200 Subject: [PATCH 085/116] fix: helm egress deployment getting stuck on install (#176) * fix: helm egress deployment getting stuck on install * only wait for redis if the ledger is required * Update helm/codeapi/templates/egress-gateway-deployment.yaml Co-authored-by: Danny Avila --------- Co-authored-by: Danny Avila --- helm/codeapi/templates/egress-gateway-deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 49c865f0..941844ce 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -24,6 +24,12 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- if or .Values.egressGrant.ledgerRequired .Values.hardenedSandboxMode }} + initContainers: + - name: wait-for-redis + image: busybox:1.36.1 + command: ['sh', '-c', 'until nc -z {{ include "codeapi.redis.host" . }} {{ include "codeapi.redis.port" . }}; do echo waiting for redis; sleep 2; done'] + {{- end }} containers: - name: egress-gateway image: "{{ .Values.egressGateway.image.repository }}:{{ .Values.egressGateway.image.tag }}" From 8a90f6d1ba01c97ed5c4e01f835a783958e581b7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 16:17:36 -0400 Subject: [PATCH 086/116] feat: Add Trusted VM Command Policy (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🛰️ feat: Add trusted VM command policy * docs: clarify trusted VM socket boundary --- docs/adr/001-stateful-code-environments.md | 13 +++- packages/code/README.md | 35 +++++++++ packages/code/package.json | 4 + packages/code/src/cli.test.ts | 40 ++++++++++ packages/code/src/cli.ts | 24 +++++- packages/code/src/index.ts | 1 + packages/code/src/native-policy.test.ts | 63 ++++++++++++++++ packages/code/src/native-policy.ts | 86 ++++++++++++++++++++++ packages/code/src/native-process.test.ts | 30 ++++++++ packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.test.ts | 50 ++++++++++++- packages/code/src/native-sandbox.ts | 29 ++++++-- 12 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 packages/code/src/native-policy.test.ts create mode 100644 packages/code/src/native-policy.ts diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index 8b9830df..cfa44bf4 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -56,8 +56,14 @@ worker replacement; the UI and operator documentation must not imply otherwise. revocation. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. -- Sandbox isolation and default-deny egress remain mandatory; pairing secures - the transport identity but does not make the host a sandbox. +- Sandbox isolation and default-deny egress remain the mandatory default; + pairing secures the transport identity but does not make the host a sandbox. + An operator may explicitly delegate network and local-socket restrictions to + an approved outer VM boundary through a named, digested worker policy. That + delegation retains direct workspace filesystem rules, cancellation, and + resource limits. The operator is responsible for preventing permitted host + services (for example, a privileged container socket) from bypassing those + rules and exposing worker identity or credential material. - A compromised worker can lie about advertised capabilities. Capability labels and policy digests are audit signals until enforcement is coupled to an attested sandbox or trusted host policy. @@ -65,7 +71,8 @@ worker replacement; the UI and operator documentation must not imply otherwise. ## Consequences - `@librechat/code` owns the provider-neutral protocol, identity handling, and - worker CLI; Code API owns enrollment, scheduling, and execution policy. + worker CLI, including machine-local execution-policy presets; Code API owns + enrollment, scheduling, and execution policy. - LibreChat owns environment persistence, ownership, RBAC, and user experience. - The Agents SDK keeps only its adapter until a second concrete consumer proves which coding-tool abstractions are genuinely provider neutral. diff --git a/packages/code/README.md b/packages/code/README.md index 819d9f5c..e46d6fd6 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -211,6 +211,41 @@ LIBRECHAT_CODE_COMMAND_SANDBOX=native-srt librechat-code run \ --worker-dir /path/to/project --allow-workspace-commands ``` +### Trusted VM command policy + +The native SRT backend can be made intentionally permissive when the selected +machine already supplies an administrator-approved outer security boundary. +The `trusted-vm` preset keeps SRT's direct filesystem rules, credential +masking, private scratch storage, cancellation, time limits, and output limits, +while allowing unmatched outbound destinations, local port binding, and Unix +sockets: + +```bash +librechat-code run \ + --worker-dir /home/ubuntu/src \ + --allow-workspace-writes \ + --allow-workspace-commands \ + --command-policy-preset trusted-vm +``` + +`LIBRECHAT_CODE_COMMAND_POLICY_PRESET=trusted-vm` is the environment equivalent. +The default is `restricted`, which preserves the default-deny network policy. +The preset configures `native-srt`; it is not an unsandboxed host-shell +backend. It is rejected unless native workspace commands are enabled. Its +normalized effective controls are included in the worker policy digest, and +the worker advertises `anthropic-srt:trusted-vm` unless an operator supplied a +custom sandbox profile label. + +Treat this preset as delegation to the machine's outer security controls. Any +outbound destination can receive workspace data, local listeners can accept +connections reachable under host policy, and Unix socket access may expose +powerful host services such as a container daemon. A socket that grants host +privilege can bypass SRT's filesystem rules and reach worker or GitHub identity +material; the outer VM boundary must prevent that path or explicitly accept +that trust. Register only the intended source root. Worker identity, +mutation-quarantine state, and configured GitHub App key files must remain +outside it. + ## Docker runtime supervisor (optional hardened adapter) `DockerRuntimeSupervisor` is the first self-contained local OCI adapter. It diff --git a/packages/code/package.json b/packages/code/package.json index a819af9d..452a8be7 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -35,6 +35,10 @@ "types": "./dist/native-sandbox.d.ts", "import": "./dist/native-sandbox.js" }, + "./native-policy": { + "types": "./dist/native-policy.d.ts", + "import": "./dist/native-policy.js" + }, "./github": { "types": "./dist/github.d.ts", "import": "./dist/github.js" diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f978fce5..18456072 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -94,6 +94,46 @@ test('CLI rejects an unknown command sandbox before entering the run loop', () = ); }); +test('CLI rejects an unknown native SRT command policy preset', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'host-shell', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /must be restricted or trusted-vm/); +}); + +test('CLI refuses a permissive policy when native commands are unavailable', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires native-srt workspace commands/); +}); + test('CLI rejects incomplete GitHub App authentication before worker registration', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 27a8d2d9..fdd28e9f 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -29,6 +29,10 @@ import { import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { + resolveNativeSrtCommandPolicy, + serializeNativeSrtCommandPolicy, +} from './native-policy.js'; import { workspaceMutationGuard } from './workspace-guards.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { LocalWorkspaceConfig } from './workspace.js'; @@ -404,6 +408,19 @@ async function run( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + const commandPolicy = resolveNativeSrtCommandPolicy( + option(args, '--command-policy-preset') ?? + process.env.LIBRECHAT_CODE_COMMAND_POLICY_PRESET?.trim().toLowerCase() ?? + 'restricted', + ); + if ( + commandPolicy.preset !== 'restricted' && + (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') + ) { + throw new Error( + 'A permissive command policy preset requires native-srt workspace commands', + ); + } const github = runtimeSessionId == null ? githubCredentials() @@ -756,6 +773,7 @@ async function run( }); const nativeOptions: NativeProcessSandboxOptions = { workspaceRoot: canonicalWorkerDirectory!, + commandPolicy, protectedPaths: [ identityPath, ...rootQuarantinePaths.values(), @@ -820,7 +838,9 @@ async function run( sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? (allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? 'anthropic-srt' + ? commandPolicy.preset === 'restricted' + ? 'anthropic-srt' + : `anthropic-srt:${commandPolicy.preset}` : runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), @@ -829,7 +849,7 @@ async function run( .update(policy) .update( allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? `\0native-srt\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` + ? `\0native-srt\0${serializeNativeSrtCommandPolicy(commandPolicy)}\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` : '', ) .digest('hex'), diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 712e7487..5363f0c0 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -5,6 +5,7 @@ export * from './storage.js'; export * from './runtime.js'; export * from './workspace.js'; export * from './workspace-runtime.js'; +export * from './native-policy.js'; export * from './native-sandbox.js'; export * from './native-process.js'; export * from './github.js'; diff --git a/packages/code/src/native-policy.test.ts b/packages/code/src/native-policy.test.ts new file mode 100644 index 00000000..4e9f4691 --- /dev/null +++ b/packages/code/src/native-policy.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + normalizeNativeSrtCommandPolicy, + resolveNativeSrtCommandPolicy, + serializeNativeSrtCommandPolicy, +} from './native-policy.js'; + +test('restricted remains the default native SRT command policy', () => { + assert.deepEqual(resolveNativeSrtCommandPolicy(), { + version: 1, + preset: 'restricted', + network: { + outbound: 'allowlist', + allowLocalBinding: false, + allowAllUnixSockets: false, + }, + }); +}); + +test('trusted-vm resolves to explicit permissive network controls', () => { + assert.deepEqual(resolveNativeSrtCommandPolicy('trusted-vm'), { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }); +}); + +test('unknown and forged native policies fail closed', () => { + assert.throws( + () => resolveNativeSrtCommandPolicy('host-shell'), + /must be restricted or trusted-vm/, + ); + assert.throws( + () => + normalizeNativeSrtCommandPolicy({ + ...resolveNativeSrtCommandPolicy('restricted'), + network: { + ...resolveNativeSrtCommandPolicy('restricted').network, + allowAllUnixSockets: true, + }, + }), + /does not match its preset/, + ); +}); + +test('serialized policy is stable and includes effective controls', () => { + const first = serializeNativeSrtCommandPolicy( + resolveNativeSrtCommandPolicy('trusted-vm'), + ); + const second = serializeNativeSrtCommandPolicy( + resolveNativeSrtCommandPolicy('trusted-vm'), + ); + assert.equal(first, second); + assert.match(first, /"outbound":"unrestricted"/); + assert.match(first, /"allowLocalBinding":true/); + assert.match(first, /"allowAllUnixSockets":true/); +}); diff --git a/packages/code/src/native-policy.ts b/packages/code/src/native-policy.ts new file mode 100644 index 00000000..5d289bfb --- /dev/null +++ b/packages/code/src/native-policy.ts @@ -0,0 +1,86 @@ +export const NATIVE_SRT_COMMAND_POLICY_PRESETS = [ + 'restricted', + 'trusted-vm', +] as const; + +export type NativeSrtCommandPolicyPreset = + typeof NATIVE_SRT_COMMAND_POLICY_PRESETS[number]; + +export interface NativeSrtCommandPolicy { + version: 1; + preset: NativeSrtCommandPolicyPreset; + network: { + outbound: 'allowlist' | 'unrestricted'; + allowLocalBinding: boolean; + allowAllUnixSockets: boolean; + }; +} + +const PRESETS: Record = { + restricted: { + version: 1, + preset: 'restricted', + network: { + outbound: 'allowlist', + allowLocalBinding: false, + allowAllUnixSockets: false, + }, + }, + 'trusted-vm': { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, +}; + +function isPreset(value: unknown): value is NativeSrtCommandPolicyPreset { + return ( + typeof value === 'string' && + NATIVE_SRT_COMMAND_POLICY_PRESETS.some((preset) => preset === value) + ); +} + +/** Resolve a named convenience preset into the explicit policy SRT enforces. */ +export function resolveNativeSrtCommandPolicy( + preset: unknown = 'restricted', +): NativeSrtCommandPolicy { + if (!isPreset(preset)) { + throw new Error( + 'Native SRT command policy preset must be restricted or trusted-vm', + ); + } + const policy = PRESETS[preset]; + return { ...policy, network: { ...policy.network } }; +} + +/** Validate a programmatic policy and return canonical preset-owned values. */ +export function normalizeNativeSrtCommandPolicy( + policy?: NativeSrtCommandPolicy, +): NativeSrtCommandPolicy { + const normalized = resolveNativeSrtCommandPolicy( + policy?.preset ?? 'restricted', + ); + if ( + policy !== undefined && + (policy.version !== normalized.version || + policy.network?.outbound !== normalized.network.outbound || + policy.network?.allowLocalBinding !== + normalized.network.allowLocalBinding || + policy.network?.allowAllUnixSockets !== + normalized.network.allowAllUnixSockets) + ) { + throw new Error('Native SRT command policy does not match its preset'); + } + return normalized; +} + +/** Stable policy material used in the bridge capability digest. */ +export function serializeNativeSrtCommandPolicy( + policy: NativeSrtCommandPolicy, +): string { + return JSON.stringify(normalizeNativeSrtCommandPolicy(policy)); +} diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index d77ec559..8dcfd4c1 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -108,6 +108,36 @@ test('executor bootstrap excludes bridge credentials and Node injection variable await sandbox.close(); }); +test('executor forwards the resolved command policy without worker credentials', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + commandPolicy: { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, + }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.messages[0].options.commandPolicy, { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }); + await sandbox.close(); +}); + test('executor hands credentials over IPC only for the current command', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index fe4e1aa8..96d89355 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -182,6 +182,7 @@ export class NativeProcessWorkspaceCommandSandbox child.on('disconnect', lost); const { workspaceRoot, + commandPolicy, protectedPaths, allowedDomains, homeDirectory, @@ -192,6 +193,7 @@ export class NativeProcessWorkspaceCommandSandbox { options: { workspaceRoot, + commandPolicy, protectedPaths, allowedDomains, homeDirectory, diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index e2dc4bda..cf790965 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -19,7 +19,10 @@ import { join } from 'node:path'; import { PassThrough } from 'node:stream'; import test from 'node:test'; -import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + SandboxAskCallback, + SandboxRuntimeConfig, +} from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; @@ -46,6 +49,7 @@ function fakeManager( } = {}, ) { let config: SandboxRuntimeConfig | undefined; + let askCallback: SandboxAskCallback | undefined; let reset = false; let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; @@ -55,8 +59,12 @@ function fakeManager( async checkDependenciesAsync() { return { warnings: [], errors: options.dependencyErrors ?? [] }; }, - async initialize(value: SandboxRuntimeConfig) { + async initialize( + value: SandboxRuntimeConfig, + callback?: SandboxAskCallback, + ) { config = value; + askCallback = callback; if (options.initializeError) throw options.initializeError; }, async wrapWithSandboxArgv(command: string) { @@ -107,6 +115,9 @@ function fakeManager( get config() { return config; }, + get askCallback() { + return askCallback; + }, get reset() { return reset; }, @@ -276,6 +287,8 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.deepEqual(fake.config?.network.allowedDomains, []); assert.equal(fake.config?.network.strictAllowlist, true); assert.equal(fake.config?.network.allowAllUnixSockets, false); + assert.equal(fake.config?.network.allowLocalBinding, false); + assert.equal(fake.askCallback, undefined); assert.deepEqual(fake.config?.filesystem.allowRead, [ canonicalRoot, scratchDirectory, @@ -305,6 +318,39 @@ test('initializes SRT with a default-deny network and scrubbed worker credential await assert.rejects(access(scratchDirectory!)); }); +test('trusted-vm permits unmatched egress and local development sockets', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + + await sandbox.prepare(); + + assert.equal(fake.config?.network.strictAllowlist, false); + assert.equal(fake.config?.network.allowLocalBinding, true); + assert.equal(fake.config?.network.allowAllUnixSockets, true); + assert.equal( + await fake.askCallback?.({ host: 'packages.example', port: 443 }), + true, + ); + assert.deepEqual(fake.config?.filesystem.allowWrite.slice(0, 1), [ + await realpath(root), + ]); +}); + test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 139a10d6..171ea663 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -41,7 +41,12 @@ import type { ChildProcessWithoutNullStreams, SpawnOptionsWithoutStdio, } from 'node:child_process'; -import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + SandboxAskCallback, + SandboxRuntimeConfig, +} from '@anthropic-ai/sandbox-runtime'; +import { normalizeNativeSrtCommandPolicy } from './native-policy.js'; +import type { NativeSrtCommandPolicy } from './native-policy.js'; import type { WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, @@ -124,7 +129,10 @@ const HOST_TEMPORARY_ROOT = tmpdir(); interface NativeSandboxManager { isSupportedPlatform(): boolean; checkDependenciesAsync(): Promise<{ warnings: string[]; errors: string[] }>; - initialize(config: SandboxRuntimeConfig): Promise; + initialize( + config: SandboxRuntimeConfig, + sandboxAskCallback?: SandboxAskCallback, + ): Promise; wrapWithSandboxArgv( command: string, binShell?: string, @@ -153,6 +161,7 @@ type SpawnCommand = ( export interface NativeSrtWorkspaceCommandSandboxOptions { workspaceRoot: string; + commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ protectedPaths?: string[]; allowedDomains?: string[]; @@ -369,13 +378,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + const commandPolicy = normalizeNativeSrtCommandPolicy( + this.options.commandPolicy, + ); + const unrestrictedNetwork = + commandPolicy.network.outbound === 'unrestricted'; const config: SandboxRuntimeConfig = { network: { allowedDomains: [...(this.options.allowedDomains ?? [])], deniedDomains: [], - strictAllowlist: true, - allowAllUnixSockets: false, - allowLocalBinding: false, + strictAllowlist: !unrestrictedNetwork, + allowAllUnixSockets: commandPolicy.network.allowAllUnixSockets, + allowLocalBinding: commandPolicy.network.allowLocalBinding, ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), }, filesystem: { @@ -439,7 +453,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox enableWeakerNetworkIsolation: false, git: { safeDirectories: [root] }, }; - await this.manager.initialize(config); + await this.manager.initialize( + config, + unrestrictedNetwork ? async () => true : undefined, + ); this.canonicalRoot = root; } From 31def177aa3ca2b4651bc413f96cf63d6522d707 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 20:31:55 -0400 Subject: [PATCH 087/116] fix: Retry Clean Cancelled BYOM Settlements Through Stop Grace (#188) A clean atomic workspace mutation rejection was settled with retries cut off at the original execution deadline. When Stop arrived near that deadline, process-tree termination finished after it, so the first settlement attempt was aborted immediately while Code API was still draining the cancellation. The client received ASSIGNMENT_EXPIRED and the durable mutation guard stayed armed. Route clean workspace mutation rejections through the known-clean rejection recovery path: a transient-retrying heartbeat and settlement retries through the rejection acknowledgement grace, floored at the bridge cancellation settlement grace. Worker shutdown still fails closed. Share the grace constant from the protocol module so the bridge and worker stay aligned. Closes #173 --- packages/code/src/protocol.ts | 2 + packages/code/src/worker.ts | 26 ++- packages/code/src/workspace-worker.test.ts | 202 +++++++++++++++++++++ service/src/bridge/store.ts | 4 +- 4 files changed, 227 insertions(+), 7 deletions(-) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 44cfdfd0..c1dad949 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -20,6 +20,8 @@ export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; +/** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ +export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index fde84768..19f219da 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'node:crypto'; import { + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, @@ -1619,7 +1620,13 @@ export class BridgeWorker { assignment.runtimeSessionId != null && settlement.status === 'rejected' && (!sandboxStarted || sandboxRejectedExecution); - if (knownCleanStatefulRejection) { + // An armed mutation reaches settlement as rejected only after an atomic + // failure that does not require quarantine. Code API accepts that + // rejection after expiry and drains it for its own grace after Stop, so a + // Stop near the deadline must not cut off retries at the deadline. + const knownCleanWorkspaceRejection = + workspaceMutationArmed && settlement.status === 'rejected'; + if (knownCleanStatefulRejection || knownCleanWorkspaceRejection) { heartbeatController.abort(); await heartbeat; const recoveryHeartbeatController = new AbortController(); @@ -1627,15 +1634,24 @@ export class BridgeWorker { recoveryHeartbeatController.signal, true, ).catch(() => undefined); + const rejectionAckGraceMs = Math.max( + 0, + this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, + ); try { await this.settleWithRetry( assignment, settlement, localDeadlineAtMs + - Math.max( - 0, - this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, - ), + (knownCleanStatefulRejection + ? rejectionAckGraceMs + : Math.max( + rejectionAckGraceMs, + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, + )), + // Stateful rejections outlive shutdown; workspace guards still + // fail closed when the worker itself stops. + knownCleanStatefulRejection ? undefined : signal, ); } finally { recoveryHeartbeatController.abort(); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 14bf2d61..8f9fcd45 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1480,6 +1480,208 @@ test('worker clears quarantine after a command cancellation confirms process ter assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); }); +test('worker retries a clean Stop rejection near its deadline through the cancellation grace', async () => { + const lifecycle: string[] = []; + const settlements: Array> = []; + const remainingMs = 100; + const startedAt = Date.now(); + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute(_request, signal) { + lifecycle.push('execute'); + await new Promise((resolve) => { + if (signal?.aborted) return resolve(); + signal?.addEventListener('abort', () => resolve(), { once: true }); + }); + lifecycle.push('stop'); + // Process-group termination is confirmed after the original deadline. + await new Promise((resolve) => setTimeout(resolve, remainingMs)); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + cancellationPollIntervalMs: 5, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + return Response.json({ + protocolVersion: 1, + cancelled: Date.now() >= startedAt + remainingMs / 2, + }); + } + if (!String(input).endsWith('/settle')) { + return Response.json({ protocolVersion: 1, accepted: true }); + } + lifecycle.push('settle'); + settlements.push({ + ...(JSON.parse(String(init?.body)) as Record), + attemptedAt: Date.now(), + }); + // Settlement delivery takes a real transport turn and honors its deadline. + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 10); + init?.signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + if (settlements.length === 1) { + return Response.json( + { error: 'Bridge settlement temporarily unavailable' }, + { status: 503 }, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-command-stopped-near-deadline', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(startedAt + remainingMs).toISOString(), + remainingMs, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30; touch delayed.txt', + }, + }); + + assert.deepEqual(lifecycle, [ + 'arm', + 'execute', + 'stop', + 'settle', + 'settle', + 'clear', + ]); + assert.ok(Number(settlements[0]?.attemptedAt) > startedAt + remainingMs); + assert.equal(settlements[1]?.status, 'rejected'); + assert.equal(settlements[1]?.errorCode, 'EXECUTION_ABORTED'); +}); + +test('worker keeps quarantine armed when shutdown interrupts a clean command rejection', async () => { + const lifecycle: string[] = []; + const controller = new AbortController(); + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + controller.abort(new Error('shutdown')); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'assignment-command-shutdown-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30', + }, + }, + controller.signal, + ), + /shutdown/, + ); + assert.deepEqual(lifecycle, ['arm', 'execute']); +}); + test('worker retains quarantine when an atomic executor cannot confirm durability', async () => { const lifecycle: string[] = []; const workspaceCapabilities = { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 32205571..5e4249a9 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -11,6 +11,7 @@ import type { } from '../../../packages/code/src/protocol'; import { + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -23,7 +24,6 @@ import { BridgeWorkspaceSlots } from './slots'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; -const CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; const DEFAULT_WORKER_TTL_SECONDS = 60; const DEFAULT_REDIS_COMMAND_TIMEOUT_MS = 1_000; @@ -1924,7 +1924,7 @@ export class RedisBridgeStore { // Give Stop its own grace so a near-timeout cancellation is not // misclassified as an ambiguous timeout. const cancellationDeadlineAtMs = - Date.now() + CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; + Date.now() + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; let cancellationPollMs = POLL_INTERVAL_MS; while (Date.now() < cancellationDeadlineAtMs) { const raw = await boundedCommand( From 118eb9b3ad704e752acc6ffa4b50fb9f1b3bb0ee Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 21:15:39 -0400 Subject: [PATCH 088/116] fix: Release Unassigned Workspace Slots When Dispatch Cleanup Fails (#189) Closes #170 --- service/src/bridge/concurrent-store.test.ts | 40 +++++++++++++++++++++ service/src/bridge/store.ts | 40 +++++++++++++++------ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index fe53c44f..736b7216 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -384,6 +384,46 @@ test('queued cancellation never leases and does not block another root', async ( ).toBeUndefined(); }); +test('unassigned slot releases even when dispatch cleanup fails', async () => { + await register(); + const originalIncr = redis.incr.bind(redis); + const originalSet = redis.set.bind(redis); + const set = originalSet as (...args: unknown[]) => unknown; + redis.incr = ((key: string) => + key.endsWith(':generation') + ? Promise.reject(new Error('injected generation outage')) + : originalIncr(key)) as typeof redis.incr; + redis.set = ((key: string, ...args: unknown[]) => + key.endsWith(':cancelled') + ? Promise.reject(new Error('injected cancellation outage')) + : set(key, ...args)) as typeof redis.set; + try { + // The reservation succeeds, then dispatch fails before storing an assignment. + await expect(dispatch('a')).rejects.toThrow('injected cancellation outage'); + } finally { + redis.incr = originalIncr; + redis.set = originalSet; + } + expect( + await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`), + ).toBe(0); + expect( + await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`), + ).toBeNull(); + const next = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(assignment.request).toMatchObject({ workspaceId: 'a' }); + await settle(assignment); + await expect(next).resolves.toMatchObject({ status: 'rejected' }); +}); + test('late quarantine releases its slot after caller cancellation and retains only its root fence', async () => { await register(); const controller = new AbortController(); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 5e4249a9..b09d3792 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -1060,20 +1060,15 @@ export class RedisBridgeStore { // already committed result rather than turning cleanup availability // into a client-visible failure that could prompt duplicate work. } + } else if (workspaceSlots != null && assignment == null) { + await this.cleanupUnassignedSlot( + args.workerId, + lockIncarnationId, + assignmentId, + ); } else { await this.cleanupDispatch(args.workerId, assignmentId, assignment); } - if (workspaceSlots != null && assignment == null) { - await boundedCommand( - workspaceSlots.release( - args.workerId, - lockIncarnationId, - assignmentId, - ), - this.redisCommandTimeoutMs, - 'Bridge unassigned slot cleanup', - ); - } } } @@ -2138,6 +2133,29 @@ export class RedisBridgeStore { ]); } + private async cleanupUnassignedSlot( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + // No stored assignment owns this reservation, so a cancellation outage + // must not leave the slot and its root busy until TTL expiry. + const [cleanup, release] = await Promise.allSettled([ + this.cleanupDispatch(workerId, assignmentId, undefined), + boundedCommand( + new BridgeWorkspaceSlots(this.redis).release( + workerId, + incarnationId, + assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge unassigned slot cleanup', + ), + ]); + if (cleanup.status === 'rejected') throw cleanup.reason; + if (release.status === 'rejected') throw release.reason; + } + private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, From 8764d019ecb4a503d6be34d8b8234f031b1f77e5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 13 Sep 2026 04:05:29 -0400 Subject: [PATCH 089/116] fix: Exit Cleanly When Native Executor Shuts Down Concurrently (#191) * fix: Exit Cleanly When Native Executor Shuts Down Concurrently A native BYOM worker under systemd KillMode=control-group receives SIGTERM at the same time as its forked SRT executor. The child ignores IPC once it is shutting down, so the parent's close handshake is left pending until the child exits, which rejects it with 'Native executor is unavailable'. That rejection escaped the CLI finally block and turned an idle administrative stop into exit status 1. Treat the close handshake as best-effort: the executor is terminated in finally regardless, and the active command has already drained, so a lost or stalled reply carries no mutation risk. Also make the child report exit status 0 when its own SRT teardown succeeded. Closes #190 * fix: Surface Explicit Executor Cleanup Failures During Close Only a lost, refused, or stalled close handshake is benign at shutdown. A negative close reply from the executor is a real cleanup failure and still rejects so pool shutdown can aggregate it. --- .../code/src/native-process-child.test.ts | 2 +- packages/code/src/native-process-child.ts | 5 +- packages/code/src/native-process.test.ts | 70 +++++++++++++++++++ packages/code/src/native-process.ts | 22 ++++-- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/packages/code/src/native-process-child.test.ts b/packages/code/src/native-process-child.test.ts index 1d4866ae..8aca3652 100644 --- a/packages/code/src/native-process-child.test.ts +++ b/packages/code/src/native-process-child.test.ts @@ -33,7 +33,7 @@ for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM'] as const) { // installation finished without requiring platform SRT dependencies. child.once('message', () => child.kill(signal)); child.send({ id: 'startup-probe', type: 'probe' }); - assert.deepEqual(await exited, { code: 1, signal: null }); + assert.deepEqual(await exited, { code: 0, signal: null }); }, ); } diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index b5221731..05ffedfe 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -25,7 +25,10 @@ const shutdown = () => { if (shuttingDown) return; shuttingDown = true; active?.controller.abort(); - void (sandbox?.close() ?? Promise.resolve()).finally(() => process.exit(1)); + void (sandbox?.close() ?? Promise.resolve()).then( + () => process.exit(0), + () => process.exit(1), + ); setTimeout(() => process.exit(1), 5000); }; process.on('disconnect', shutdown); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 8dcfd4c1..800dcf11 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -301,6 +301,76 @@ test('executor close drains an active command before closing IPC', async () => { await assert.rejects(sandbox.execute(request), /unavailable/); }); +test('executor close resolves when the child exits during the close handshake', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { + send(message: Record, callback: (error: null) => void) { + fake.messages.push(message); + callback(null); + queueMicrotask(() => { + Object.assign(fake.child, { connected: false }); + fake.child.emit('exit', 1, null); + fake.child.emit('disconnect'); + }); + return true; + }, + }); + await sandbox.close(); + assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor close still reports a cleanup failure the child replies with', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { + send(message: Record, callback: (error: null) => void) { + fake.messages.push(message); + callback(null); + queueMicrotask(() => + fake.child.emit('message', { + id: message.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'scratch cleanup failed', + mutation: false, + requiresQuarantine: false, + }), + ); + return true; + }, + }); + await assert.rejects(sandbox.close(), /scratch cleanup failed/); + assert.equal(fake.killCalls, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor close skips the handshake once the child is already lost', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { connected: false }); + fake.child.emit('exit', 1, null); + await sandbox.close(); + assert.equal( + fake.messages.some((m) => m.type === 'close'), + false, + ); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + test('executor startup loss is not reported as an applied mutation', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 96d89355..e1a02498 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -73,6 +73,15 @@ export function nativeExecutorEnvironment( ); } +/** The executor process was lost, refused a send, or stalled past its + * deadline, as opposed to a failure the executor reported explicitly. */ +class NativeExecutorUnavailableError extends WorkspaceToolError { + constructor(mutation: boolean) { + super('Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation); + this.name = 'NativeExecutorUnavailableError'; + } +} + /** One persistent, process-isolated SRT manager per workspace. No automatic * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ export class NativeProcessWorkspaceCommandSandbox @@ -109,11 +118,7 @@ export class NativeProcessWorkspaceCommandSandbox } private unavailable(mutation: boolean): WorkspaceToolError { - return new WorkspaceToolError( - 'Native executor is unavailable', - 'COMMAND_UNAVAILABLE', - mutation, - ); + return new NativeExecutorUnavailableError(mutation); } private async start(): Promise { @@ -340,12 +345,17 @@ export class NativeProcessWorkspaceCommandSandbox return this.closing; } + /** An executor that exits, disconnects, or stalls while closing is + * terminated in `finally` regardless, and the active command has already + * drained, so only a failure the executor reports explicitly is surfaced. */ private async stop(): Promise { await this.active?.catch(() => undefined); await this.ready?.catch(() => undefined); try { if (this.child?.connected && !this.failed) - await this.rpc('close', {}, 10_000, false); + await this.rpc('close', {}, 10_000, false).catch((error: unknown) => { + if (!(error instanceof NativeExecutorUnavailableError)) throw error; + }); } finally { this.failed = true; this.terminate(); From 25f3841c64422c289f30743520cb3ca88cc471fb Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 13 Sep 2026 23:21:49 -0400 Subject: [PATCH 090/116] fix: authenticate GitHub App Git operations (#192) --- packages/code/src/github.test.ts | 9 +++++++-- packages/code/src/github.ts | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index bd62acfd..78d9751d 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -137,12 +137,17 @@ test('builds process-scoped Git HTTPS authorization without embedding credential const provider = new StaticGitHubCredentialProvider( 'github_pat_abcdefghijklmnopqrstuvwxyz', ); + const encodedCredential = Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'); assert.deepEqual( gitHubCredentialEnvironment(await provider.getCredential()), { - [GITHUB_CREDENTIAL_ENV_NAME]: 'github_pat_abcdefghijklmnopqrstuvwxyz', + [GITHUB_CREDENTIAL_ENV_NAME]: encodedCredential, }, ); + assert.ok(!encodedCredential.includes('github_pat_')); }); test('composes the masked credential with SRT Git configuration inside the sandbox', () => { @@ -155,7 +160,7 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); - assert.equal(wrapped.match(/Authorization: Bearer/g)?.length, 1); + assert.equal(wrapped.match(/Authorization: Basic/g)?.length, 1); assert.ok(!wrapped.includes('github_pat_')); }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 7b1f12b1..b8046897 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -197,7 +197,10 @@ export function gitHubCredentialEnvironment( credential: GitHubCredential, ): Record { return { - [GITHUB_CREDENTIAL_ENV_NAME]: credential.value, + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + `x-access-token:${credential.value}`, + 'utf8', + ).toString('base64'), }; } @@ -250,14 +253,14 @@ export function wrapGitHubCredentialCommand( return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', - `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Bearer %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, command, ].join(' && '); } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', - `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Bearer \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, command, ].join(';\n'); From 1c7af888c774a52d3a8d4170590c6c409b03ae6e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 01:17:13 -0400 Subject: [PATCH 091/116] fix: isolate file deletion rate limits (#193) --- service/src/config.ts | 6 +++ service/src/middleware/limits.test.ts | 56 ++++++++++++++++++++++++ service/src/middleware/limits.ts | 11 +++++ service/src/service/exec-timeout.test.ts | 2 +- service/src/service/router.ts | 12 +++-- 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/service/src/config.ts b/service/src/config.ts index 90df6b58..d025831e 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -346,6 +346,12 @@ export const env = { // Files List Rate Limits FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute + // File Delete Rate Limits. Fall back to the fetch settings so existing + // deployments keep their current limits while using an independent bucket. + DELETE_LIMIT_WINDOW: + Number(process.env.DELETE_LIMIT_WINDOW) || Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, + DELETE_MAX_REQUESTS: + Number(process.env.DELETE_MAX_REQUESTS) || Number(process.env.FETCH_MAX_REQUESTS) || 120, // Redis Key Cache Config SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, /** TTL for the durable `session-owner:` record that backs diff --git a/service/src/middleware/limits.test.ts b/service/src/middleware/limits.test.ts index e7c1c46f..12f3daf4 100644 --- a/service/src/middleware/limits.test.ts +++ b/service/src/middleware/limits.test.ts @@ -111,6 +111,40 @@ async function startRateLimitedApp(max: number, windowMs: number): Promise { + const redis = new TestRedisRateLimitStore(); + setRateLimitRedisForTests(redis); + + const app = express(); + app.use((req, _res, next) => { + applyPrincipal(req as AuthenticatedRequest, { + userId: 'user-a', + tenantId: 'tenant-a', + principalSource: 'librechat_jwt', + }); + next(); + }); + app.get( + '/v1/files/session-a', + createRateLimiter('test-fetch', windowMs, max, { message: 'Too many file list requests.' }), + (_req, res) => res.status(200).json({ ok: true }), + ); + app.delete( + '/v1/files/session-a/file-a', + createRateLimiter('test-delete', windowMs, max, { + message: 'Too many file deletion requests.', + structuredBody: true, + }), + (_req, res) => res.status(200).json({ ok: true }), + ); + + const server = app.listen(0, '127.0.0.1'); + servers.push(server); + await once(server, 'listening'); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + function postExec(url: string, headers: Record = {}): Promise { return fetch(`${url}/v1/exec`, { method: 'POST', @@ -228,3 +262,25 @@ describe('execution rate limiting', () => { expect((await postExec(url)).status).toBe(200); }); }); + +describe('file operation rate limiting', () => { + test('keeps deletion traffic out of the file-list bucket and returns structured retry guidance', async () => { + const url = await startIndependentFileLimiterApp(1, 30_000); + + expect((await fetch(`${url}/v1/files/session-a`)).status).toBe(200); + expect((await fetch(`${url}/v1/files/session-a/file-a`, { method: 'DELETE' })).status).toBe(200); + + const rejectedDelete = await fetch(`${url}/v1/files/session-a/file-a`, { method: 'DELETE' }); + const body = await rejectedDelete.json() as ReturnType; + expect(rejectedDelete.status).toBe(429); + expect(rejectedDelete.headers.get('retry-after')).not.toBeNull(); + expect(body.error).toBe('rate_limited'); + expect(body.message).toContain('Too many file deletion requests.'); + + const rejectedList = await fetch(`${url}/v1/files/session-a`); + expect(rejectedList.status).toBe(429); + expect(await rejectedList.json()).toEqual({ + error: expect.stringContaining('Too many file list requests.'), + }); + }); +}); diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index b16ed196..099261a1 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -202,3 +202,14 @@ export const fetchLimiter = createRateLimiter( env.FETCH_MAX_REQUESTS, { message: 'Too many file list requests.' } ); + +export const deleteLimiter = createRateLimiter( + 'delete', + env.DELETE_LIMIT_WINDOW, + env.DELETE_MAX_REQUESTS, + { + message: 'Too many file deletion requests.', + structuredBody: true, + logRejections: true, + } +); diff --git a/service/src/service/exec-timeout.test.ts b/service/src/service/exec-timeout.test.ts index 70e00d17..b10a94c6 100644 --- a/service/src/service/exec-timeout.test.ts +++ b/service/src/service/exec-timeout.test.ts @@ -11,7 +11,7 @@ test('/exec validates timeout before enqueue and forwards its cap to both langua mock.module('./src/middleware/auth', () => ({ sessionAuth: passthrough })); mock.module('./src/middleware/limits', () => ({ executionLimiter: passthrough, uploadLimiter: passthrough, - downloadLimiter: passthrough, fetchLimiter: passthrough, + downloadLimiter: passthrough, fetchLimiter: passthrough, deleteLimiter: passthrough, })); mock.module('./src/lifecycle', () => ({ checkServiceStartUp: () => false, checkServiceShutDown: () => false, diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 2c42f60c..43d0ff4b 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -7,7 +7,13 @@ import { Readable } from 'stream'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { sessionAuth } from '../middleware/auth'; -import { executionLimiter, uploadLimiter, downloadLimiter, fetchLimiter } from '../middleware/limits'; +import { + executionLimiter, + uploadLimiter, + downloadLimiter, + fetchLimiter, + deleteLimiter, +} from '../middleware/limits'; import { internalServiceHeaders } from '../internal-service-auth'; import { resolveSessionKey, resolveOutputBucketSessionKey, SessionKeyResolutionError, parseUploadSessionKeyInput, type SessionKeyInput } from '../session-key'; import { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection } from '../queue'; @@ -997,7 +1003,7 @@ const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) = } }; -router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); +router.delete('/files/:session_id/:fileId', deleteLimiter, sessionAuth, deleteSessionObject); /** * Alias of the route above, on the path LibreChat's `deleteCodeEnvFile` @@ -1013,6 +1019,6 @@ router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSes * * GET on this same path is the metadata proxy above. */ -router.delete('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); +router.delete('/sessions/:session_id/objects/:fileId', deleteLimiter, sessionAuth, deleteSessionObject); export default router; From 737f498ebedc3b9b26da3e5f206d7e0c49b4e901 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 01:27:39 -0400 Subject: [PATCH 092/116] feat: broker GitHub CLI authentication (#194) --- packages/code/README.md | 19 +++++---- packages/code/src/cli.ts | 15 +++---- packages/code/src/github.test.ts | 73 ++++++++++++++++++++++++++++++++ packages/code/src/github.ts | 40 +++++++++++++++++ 4 files changed, 129 insertions(+), 18 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index e46d6fd6..d06fbf9a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -180,14 +180,17 @@ verification are implemented; use macOS, Linux, or WSL2. This also applies to Gi private keys. Git receives authentication through process-scoped `GIT_CONFIG_*` variables. -The same isolated config supplies the standard Git LFS filters; hosts using LFS -must install `git-lfs`, and checkout fails instead of silently leaving pointer -files when it is unavailable. -SRT replaces only the bearer-token portion with a sentinel inside the sandbox -and substitutes the real value in its host proxy only for `github.com` HTTPS -traffic. TLS termination is enabled for that substitution. The worker restores -the parent environment immediately after constructing the sandbox command; it -never writes credentials into the repository, a remote URL, or Git config. +When the GitHub CLI is installed, `gh api`, pull-request, issue, and workflow +commands receive the same installation scope through `GH_TOKEN` (or +`GH_ENTERPRISE_TOKEN` for GHES). The same isolated Git config supplies the +standard Git LFS filters; hosts using LFS must install `git-lfs`, and checkout +fails instead of silently leaving pointer files when it is unavailable. +SRT replaces each real credential with a sentinel inside the sandbox and +substitutes the real value in its host proxy only for the corresponding Git or +GitHub API host. TLS termination is enabled for that substitution. The worker +restores the parent environment immediately after constructing the sandbox +command; it never writes credentials into the repository, a remote URL, Git +config, or the GitHub CLI credential store. GitHub's required domains are added to the command egress allowlist only when authentication is configured. The worker identity, GitHub App key path, token source variables, and mutation-quarantine record remain denied to sandboxed diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index fdd28e9f..a87f1a5a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -38,11 +38,11 @@ import type { NativeProcessSandboxOptions } from './native-process.js'; import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, - GITHUB_CREDENTIAL_ENV_NAME, GitHubAppCredentialProvider, + gitHubCommandCredentialEnvironment, + gitHubMaskedCredentialVariables, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, - gitHubCredentialEnvironment, normalizeGitHubHost, wrapGitHubCredentialCommand, } from './github.js'; @@ -783,16 +783,11 @@ async function run( ...(github.provider ? { maskedEnvironment: { - variables: [ - { - name: GITHUB_CREDENTIAL_ENV_NAME, - extract: '^(.+)$', - injectHosts: [github.host], - }, - ], + variables: gitHubMaskedCredentialVariables(github.host), async resolve(signal?: AbortSignal) { - return gitHubCredentialEnvironment( + return gitHubCommandCredentialEnvironment( await github.provider!.getCredential(signal), + github.host, ); }, wrapCommand(command: string, platform: NodeJS.Platform) { diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 78d9751d..4b028239 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -17,6 +17,9 @@ import { GitHubAppCredentialProvider, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, + gitHubCliTokenEnvironmentName, + gitHubCommandCredentialEnvironment, + gitHubMaskedCredentialVariables, GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, normalizeGitHubHost, @@ -150,6 +153,65 @@ test('builds process-scoped Git HTTPS authorization without embedding credential assert.ok(!encodedCredential.includes('github_pat_')); }); +test('adds a GitHub CLI token only to the command-sandbox credential bundle', async () => { + const provider = new StaticGitHubCredentialProvider( + 'github_pat_abcdefghijklmnopqrstuvwxyz', + ); + const credential = await provider.getCredential(); + assert.deepEqual(gitHubCommandCredentialEnvironment(credential), { + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'), + GH_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }); + assert.deepEqual( + gitHubCommandCredentialEnvironment(credential, 'github.example.test'), + { + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'), + GH_ENTERPRISE_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }, + ); +}); + +test('selects the GitHub CLI token variable for public and enterprise hosts', () => { + assert.equal(gitHubCliTokenEnvironmentName('github.com'), 'GH_TOKEN'); + assert.equal( + gitHubCliTokenEnvironmentName('github.example.test'), + 'GH_ENTERPRISE_TOKEN', + ); +}); + +test('restricts Git and GitHub CLI credential substitution to their respective hosts', () => { + assert.deepEqual(gitHubMaskedCredentialVariables('github.com'), [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: ['github.com'], + }, + { + name: 'GH_TOKEN', + extract: '^(.+)$', + injectHosts: ['api.github.com'], + }, + ]); + assert.deepEqual(gitHubMaskedCredentialVariables('github.example.test'), [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: ['github.example.test'], + }, + { + name: 'GH_ENTERPRISE_TOKEN', + extract: '^(.+)$', + injectHosts: ['github.example.test'], + }, + ]); +}); + test('composes the masked credential with SRT Git configuration inside the sandbox', () => { const wrapped = wrapGitHubCredentialCommand( 'git push', @@ -160,10 +222,21 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); + assert.doesNotMatch(wrapped, /unset GH_TOKEN/); assert.equal(wrapped.match(/Authorization: Basic/g)?.length, 1); assert.ok(!wrapped.includes('github_pat_')); }); +test('targets GitHub CLI at an enterprise host without exposing its token', () => { + const wrapped = wrapGitHubCredentialCommand( + 'gh pr create', + 'github.example.test', + 'linux', + ); + assert.match(wrapped, /GH_HOST=github\.example\.test/); + assert.doesNotMatch(wrapped, /GH_ENTERPRISE_TOKEN=/); +}); + test('rejects an insecure GitHub App API endpoint before reading the private key', () => { assert.throws( () => diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index b8046897..c727e70d 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -204,6 +204,43 @@ export function gitHubCredentialEnvironment( }; } +export function gitHubCommandCredentialEnvironment( + credential: GitHubCredential, + host = 'github.com', +): Record { + return { + ...gitHubCredentialEnvironment(credential), + [gitHubCliTokenEnvironmentName(host)]: credential.value, + }; +} + +export function gitHubCliTokenEnvironmentName(host: string): string { + return host === 'github.com' ? 'GH_TOKEN' : 'GH_ENTERPRISE_TOKEN'; +} + +export function gitHubApiHost(host: string): string { + return host === 'github.com' ? 'api.github.com' : host; +} + +export function gitHubMaskedCredentialVariables(host: string): Array<{ + name: string; + injectHosts: string[]; + extract: string; +}> { + return [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: [host], + }, + { + name: gitHubCliTokenEnvironmentName(host), + extract: '^(.+)$', + injectHosts: [gitHubApiHost(host)], + }, + ]; +} + export function gitHubAuthenticationPolicyIdentity(options: { mode?: 'app' | 'token'; host: string; @@ -249,10 +286,12 @@ export function wrapGitHubCredentialCommand( platform: NodeJS.Platform = process.platform, ): string { const key = `http.https://${host}/.extraheader`; + const cliHost = host === 'github.com' ? undefined : host; if (platform === 'win32') { return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', + ...(cliHost ? [`set "GH_HOST=${cliHost}"`] : []), `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, command, @@ -260,6 +299,7 @@ export function wrapGitHubCredentialCommand( } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', + ...(cliHost ? [`export GH_HOST=${cliHost}`] : []), `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, command, From 03e2fc11951fa3faf2fb5943afc445346177434e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 13:58:14 -0400 Subject: [PATCH 093/116] feat: Run PTC in Selected BYOM Workspaces (#195) * feat: run PTC in selected BYOM workspaces * fix: harden native workspace PTC replay * fix: preserve replay isolation and bridge limits * test: tolerate hosts without filesystem cloning * test: surface copy-on-write clone faults * fix: harden native workspace PTC admission * fix: close native replay effect and finalization boundaries --- docs/remote-bridge/README.md | 19 +- packages/code/README.md | 38 +- packages/code/src/cli.ts | 14 + packages/code/src/index.ts | 1 + packages/code/src/native-pool.test.ts | 25 + packages/code/src/native-pool.ts | 76 +- packages/code/src/native-process-child.ts | 35 +- packages/code/src/native-process.test.ts | 166 +++- packages/code/src/native-process.ts | 249 +++++- .../code/src/native-programmatic-live.test.ts | 41 + packages/code/src/native-programmatic.test.ts | 608 +++++++++++++ packages/code/src/native-programmatic.ts | 816 ++++++++++++++++++ packages/code/src/native-sandbox.test.ts | 357 ++++++-- packages/code/src/native-sandbox.ts | 448 ++++++++-- packages/code/src/protocol.test.ts | 184 ++++ packages/code/src/protocol.ts | 454 ++++++++-- packages/code/src/worker-slots.test.ts | 48 ++ packages/code/src/worker.ts | 159 +++- packages/code/src/workspace-worker.test.ts | 82 ++ packages/code/src/workspace.ts | 5 + service/src/bridge/router.ts | 1 + service/src/bridge/selection.ts | 1 + service/src/bridge/store.ts | 112 ++- service/src/bridge/workspace-store.test.ts | 117 +++ service/src/egress-gateway.ts | 37 +- service/src/egress-grant.test.ts | 8 + service/src/preamble-bash.test.ts | 226 ++++- service/src/preamble-bash.ts | 125 ++- service/src/preamble.test.ts | 46 +- service/src/preamble.ts | 276 ++++-- service/src/ptc-constants.test.ts | 9 + service/src/ptc-constants.ts | 7 +- .../src/sandbox-backend/remote-bridge.test.ts | 28 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/sandbox-backend/types.ts | 3 + service/src/sandbox-dispatch.test.ts | 67 +- service/src/sandbox-dispatch.ts | 33 +- service/src/sandbox-egress.ts | 29 + service/src/service/programmatic-router.ts | 477 +++++++--- .../src/service/programmatic-state.test.ts | 2 + service/src/service/programmatic-state.ts | 2 + service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 29 +- service/src/workers.ts | 11 + 44 files changed, 4850 insertions(+), 624 deletions(-) create mode 100644 packages/code/src/native-programmatic-live.test.ts create mode 100644 packages/code/src/native-programmatic.test.ts create mode 100644 packages/code/src/native-programmatic.ts create mode 100644 service/src/ptc-constants.test.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index ab7e2f64..99a0dece 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -149,6 +149,17 @@ implementation and an allowlist of workspace IDs, preserves per-workspace operation restrictions, validates bounded results, and treats an unknown command failure as an uncertain mutation. +Selected attached workspaces on macOS, Linux, and WSL2 can also advertise Bash +Programmatic Tool Calling. Native Windows workers do not advertise Bash PTC. +Code API then runs each replay iteration through the same workspace-scoped +native SRT executor. Source code operates in the selected local root, while +replay metadata, injected skills and attachments, and generated artifacts are +staged in an execution-private data directory and removed after settlement. +Only authorized file references and returned artifacts cross the relay; the +repository is never uploaded to Code API. This capability is advertised only +when native SRT commands and a file-relay upstream are both configured, so +older or partially configured workers continue to fail closed. + Native SRT is the MVP and default command backend on a user's chosen laptop or VM. It uses Seatbelt on macOS, bubblewrap/seccomp on Linux, and the SRT restricted-account helper on Windows. It confines writes to the registered @@ -157,9 +168,11 @@ credentials, and denies network egress by default. Startup fails closed when the platform dependencies are unavailable; there is no unsandboxed fallback. Use `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` for an explicit comma-separated egress allowlist. -Linux hosts must provide Bash at `/bin/bash`, `bubblewrap`, `socat`, and -`ripgrep`; macOS uses system facilities. Windows requires SRT's one-time -restricted-account setup. +Linux hosts must provide `bubblewrap`, `socat`, and `ripgrep`; macOS uses +system facilities. Bash Programmatic Tool Calling additionally requires Bash +5.2 or newer and `jq` on `PATH` on macOS, Linux, and WSL2. The worker resolves +the compatible shell from `PATH` rather than assuming `/bin/bash`. Windows +requires SRT's one-time restricted-account setup. The optional `docker-nsjail` adapter enables a stronger container boundary with `--allow-workspace-commands` (or diff --git a/packages/code/README.md b/packages/code/README.md index d06fbf9a..87d9dded 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -143,14 +143,48 @@ policy. This matches the personal-machine SRT trust model; use the Docker/NsJail backend or a dedicated VM boundary when hard teardown of adversarial process trees is required. -Linux hosts need Bash at `/bin/bash`, `bubblewrap`, `socat`, and `ripgrep`; macOS uses system -facilities. Follow SRT's one-time restricted-account setup when using Windows. +Linux hosts need `bubblewrap`, `socat`, and `ripgrep`; macOS uses system +facilities. Bash Programmatic Tool Calling additionally requires Bash 5.2 or +newer and `jq` on `PATH` on macOS, Linux, and WSL2. The worker resolves that +shell explicitly instead of assuming `/bin/bash`, which remains Bash 3.2 on +many macOS hosts. Follow SRT's one-time restricted-account setup when using Windows. An operator may allow explicit egress destinations with the comma-separated `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` setting. Treat that as a security policy: an allowed destination can receive workspace data. The normalized allowlist is included in the worker policy digest. Tool approval hooks remain the user-facing allow/deny boundary for each invocation. +When Code API negotiates `bash` programmatic execution for a selected +workspace, the same native SRT executor also supports replay-mode Programmatic +Tool Calling on macOS, Linux, and WSL2 workers. Native Windows does not +advertise this Bash capability. The repository remains the command working directory. Generated +PTC scripts, replay history, skill files, chat attachments, and returned +artifacts use an owner-only per-execution directory under the worker's private +SRT scratch root, exposed to code as `LIBRECHAT_CODE_DATA_DIR`. That directory +is removed after every iteration and is never placed in the repository. + +Replay probes run against a disposable copy-on-write snapshot with network and +socket access denied, including under `trusted-vm`. External effects must not +repeat while discovering pending tools. Use registered tools for network-dependent +replay control flow; the final commit pass runs once under the configured policy. +Each probe's SRT proxy session is revoked before restoring the commit policy; +per-command network overrides alone do not restrict SRT's session-level proxies. +Probe failures do not quarantine the real workspace. Once the commit pass starts, +its fence remains until result restoration succeeds; uncertain finalization +quarantines only that workspace. + +Reference inputs and artifact outputs travel only through the configured +`LIBRECHAT_CODE_FILE_RELAY_UPSTREAM`, using Code API's execution-scoped opaque +egress grant. The worker rejects redirects and bounds each transfer to 10 MiB, +each execution to 100 files and 100 MiB total, and transfer concurrency to four. +Caller inputs are limited to 98 files, reserving two for the script and replay +history. Code API reserves one third of the job budget for all transfer batches +and negotiates each transfer's deadline before signing the request. +Its parent process keeps a 64-entry/32-MiB LRU input cache keyed by a stable, +Code-API-authorized digest; sandboxed commands cannot read that cache. Requests +against one workspace remain serialized, while negotiated lease slots allow +different registered roots to execute concurrently. + The native sandbox preserves standard `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` names (including lowercase forms), plus Windows process and profile variables on Windows. SRT remains responsible for the final sandbox environment diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index a87f1a5a..289b0dc4 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -408,6 +408,11 @@ async function run( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + const nativeProgrammaticEnabled = + allowWorkspaceCommands && + commandSandboxMode === 'native-srt' && + process.platform !== 'win32' && + (fileRelayUpstream?.length ?? 0) > 0; const commandPolicy = resolveNativeSrtCommandPolicy( option(args, '--command-policy-preset') ?? process.env.LIBRECHAT_CODE_COMMAND_POLICY_PRESET?.trim().toLowerCase() ?? @@ -780,6 +785,9 @@ async function run( github.privateKeyPath, ].filter((path): path is string => path != null), allowedDomains: commandAllowedDomains, + ...(nativeProgrammaticEnabled + ? { programmaticFileUpstream: fileRelayUpstream } + : {}), ...(github.provider ? { maskedEnvironment: { @@ -819,6 +827,9 @@ async function run( workspaceTools = new SandboxWorkspaceTools({ workspaceTools, commandWorkspaces: roots.map((root) => root.id), + ...(nativeProgrammaticEnabled + ? { programmaticLanguages: ['bash'] } + : {}), commandSandbox: nativeCommandSandbox ?? new RuntimeWorkspaceCommandSandbox({ @@ -878,6 +889,9 @@ async function run( runtimeSupervisor, capabilities, workspaceTools, + ...(nativeProgrammaticEnabled && nativeCommandSandbox + ? { workspaceProgrammatic: nativeCommandSandbox } + : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 5363f0c0..6f94b190 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -7,6 +7,7 @@ export * from './workspace.js'; export * from './workspace-runtime.js'; export * from './native-policy.js'; export * from './native-sandbox.js'; +export * from './native-programmatic.js'; export * from './native-process.js'; export * from './github.js'; export * from './worker.js'; diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index 3a24db34..cad30a52 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -13,6 +13,31 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ workspaceId, command: 'fixture', }); + +test('native pool preflights every registered root with bounded concurrency', async () => { + const prepared: string[] = []; + let active = 0; + let peak = 0; + const pool = new NativeWorkspaceCommandPool(roots, 2, (options) => ({ + async prepare() { + active += 1; + peak = Math.max(peak, active); + await new Promise(resolve => setTimeout(resolve, 5)); + prepared.push(options.workspaceRoot); + active -= 1; + }, + async close() {}, + async execute() { + throw new Error('unreachable'); + }, + })); + + await pool.prepare(); + assert.deepEqual(prepared.sort(), ['/fixture/a', '/fixture/b', '/fixture/c']); + assert.equal(peak, 2); + await pool.close(); +}); + test('a known-clean executor failure is retired without replaying the command', async () => { let created = 0; let executed = 0; diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 28f155e2..766409b1 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -2,6 +2,7 @@ import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { + BridgeWorkspaceProgrammaticRequest, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, } from './protocol.js'; @@ -10,7 +11,10 @@ interface Entry { sandbox: Pick< NativeProcessWorkspaceCommandSandbox, 'prepare' | 'execute' | 'close' - >; + > & + Partial< + Pick + >; busy: boolean; } @@ -92,12 +96,25 @@ export class NativeWorkspaceCommandPool { } async prepare(): Promise { - const entry = await this.allocate(this.roots.keys().next().value!); - try { - await entry.sandbox.prepare(); - } finally { - entry.busy = false; - } + const workspaceIds = [...this.roots.keys()]; + let next = 0; + await Promise.all( + Array.from( + { length: Math.min(this.capacity, workspaceIds.length) }, + async () => { + for (;;) { + const index = next++; + if (index >= workspaceIds.length) return; + const entry = await this.allocate(workspaceIds[index]!); + try { + await entry.sandbox.prepare(); + } finally { + entry.busy = false; + } + } + }, + ), + ); } async execute( @@ -136,6 +153,51 @@ export class NativeWorkspaceCommandPool { } } + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const entry = await this.allocate(workspaceId); + let enteredExecutor = false; + try { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Programmatic execution cancelled before dispatch', + 'EXECUTION_ABORTED', + ); + enteredExecutor = true; + if (!entry.sandbox.executeProgrammatic) { + throw new WorkspaceToolError( + 'Native programmatic executor is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + return await entry.sandbox.executeProgrammatic( + workspaceId, + request, + signal, + ); + } catch (error) { + if ( + enteredExecutor && + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted + ) { + try { + await entry.sandbox.close(); + if (this.entries.get(workspaceId) === entry) + this.entries.delete(workspaceId); + } catch { + /* Retain ownership for subsequent cleanup/shutdown. */ + } + } + throw error; + } finally { + entry.busy = false; + } + } + async close(): Promise { this.closing = true; await this.allocation; diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 05ffedfe..2e8beb38 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -1,11 +1,16 @@ import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; import { WorkspaceToolError } from './workspace.js'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; -import type { WorkspaceExecuteCommandRequest } from './protocol.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, +} from './protocol.js'; // This entrypoint is private to a forked trusted executor. No HTTP listener, // argv credentials, bridge token, or persisted pairing material is required. let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; +let programmaticExecutor: NativeWorkspaceProgrammaticExecutor | undefined; let active: { id: string; controller: AbortController } | undefined; let busy = false; let credentials: Record = {}; @@ -44,11 +49,14 @@ process.on('message', async (raw: unknown) => { NativeSrtWorkspaceCommandSandboxOptions, 'maskedEnvironment' > & { + programmaticFileUpstream?: string; variables?: NonNullable< NativeSrtWorkspaceCommandSandboxOptions['maskedEnvironment'] >['variables']; }; request: WorkspaceExecuteCommandRequest; + programmaticRequest?: BridgeWorkspaceProgrammaticRequest; + workspaceId?: string; credentials?: Record; wrappedCommand?: string; }; @@ -62,7 +70,8 @@ process.on('message', async (raw: unknown) => { try { let result: unknown; if (message.type === 'prepare' && !sandbox) { - const { variables, ...options } = message.options; + const { variables, programmaticFileUpstream, ...options } = + message.options; sandbox = new NativeSrtWorkspaceCommandSandbox({ ...options, ...(variables @@ -80,11 +89,33 @@ process.on('message', async (raw: unknown) => { : {}), }); await sandbox.prepare(); + programmaticExecutor = programmaticFileUpstream + ? new NativeWorkspaceProgrammaticExecutor({ + sandbox, + upstreamUrl: programmaticFileUpstream, + }) + : undefined; + await programmaticExecutor?.prepare(); } else if (message.type === 'execute' && sandbox) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; result = await sandbox.execute(message.request, active.controller.signal); + } else if ( + message.type === 'programmatic' && + sandbox && + programmaticExecutor && + message.programmaticRequest && + typeof message.workspaceId === 'string' + ) { + active = { id: message.id, controller: new AbortController() }; + credentials = message.credentials ?? {}; + wrappedCommand = message.wrappedCommand; + result = await programmaticExecutor.execute( + message.programmaticRequest, + message.workspaceId, + active.controller.signal, + ); } else if (message.type === 'close' && sandbox) { await sandbox.close(); } else throw new Error('Invalid executor state'); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 800dcf11..e96f1e65 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -1,13 +1,29 @@ import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import test from 'node:test'; +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import { NativeProcessWorkspaceCommandSandbox, nativeExecutorEnvironment, + trustedProgrammaticExecutable, } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +test('preflight rejects relative and workspace-controlled executables including symlinks', async t => { + const root = await mkdtemp(join(tmpdir(), 'native-ptc-path-')); + const outside = await mkdtemp(join(tmpdir(), 'native-ptc-link-')); + t.after(async () => { await rm(root, { recursive: true, force: true }); await rm(outside, { recursive: true, force: true }); }); + const executable = join(root, 'bash'); + await writeFile(executable, '#!/bin/sh\nexit 0\n', { mode: 0o700 }); + await symlink(executable, join(outside, 'bash')); + await assert.rejects(trustedProgrammaticExecutable('./bash', root), /absolute/); + await assert.rejects(trustedProgrammaticExecutable(executable, root), /outside the workspace/); + await assert.rejects(trustedProgrammaticExecutable(join(outside, 'bash'), root), /outside the workspace/); +}); + const request = { protocolVersion: 1 as const, operation: 'execute_command' as const, @@ -43,13 +59,19 @@ function fixture( queueMicrotask(() => { if (message.type === 'prepare' && prepare) return prepare(child, message); - if (message.type === 'execute' && execute) + if ( + (message.type === 'execute' || message.type === 'programmatic') && + execute + ) return execute(child, message); if (message.type === 'cancel') return; child.emit('message', { id: message.id, ok: true, - ...(message.type === 'execute' ? { result } : {}), + ...(message.type === 'execute' || + message.type === 'programmatic' + ? { result } + : {}), }); }); return true; @@ -171,8 +193,88 @@ test('executor hands credentials over IPC only for the current command', async ( await sandbox.close(); }); +test('programmatic executor resolves and scopes credentials to its command', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + maskedEnvironment: { + variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], + async resolve() { + return { TOKEN: 'per-programmatic-secret' }; + }, + wrapCommand(command) { + return `wrapped ${command}`; + }, + }, + }, + fake.fork, + ); + const programmaticRequest = { + headers: {}, + body: { + language: 'bash' as const, + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'git status' }], + }, + }; + await sandbox.executeProgrammatic('primary', programmaticRequest); + assert.equal( + JSON.stringify(fake.options).includes('per-programmatic-secret'), + false, + ); + const message = fake.messages.find( + candidate => candidate.type === 'programmatic', + )!; + assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); + assert.equal( + message.wrappedCommand, + 'wrapped exec "$LIBRECHAT_CODE_BASH_PATH" "$LIBRECHAT_CODE_DATA_DIR/main.sh"', + ); + await sandbox.close(); +}); + +test('programmatic executor preserves a child-reported pre-dispatch failure', async () => { + const fake = fixture((child, message) => + child.emit('message', { + id: message.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'Programmatic input download failed', + mutation: false, + requiresQuarantine: false, + }), + ); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + await sandbox.close(); +}); + test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { - const fake = fixture((child) => child.emit('exit', 1)); + const fake = fixture(child => child.emit('exit', 1)); const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: '/workspace' }, fake.fork, @@ -180,16 +282,17 @@ test('executor loss after dispatch is an uncertain mutation and is never replaye await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted, ); await assert.rejects(sandbox.execute(request), /unavailable/); - assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'execute').length, 1); await sandbox.close(); }); test('executor cancellation targets the active request and preserves mutation certainty', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -202,7 +305,7 @@ test('executor cancellation targets the active request and preserves mutation ce await dispatch; await assert.rejects(sandbox.execute(request), /unavailable/); controller.abort(); - const command = fake.messages.find((m) => m.type === 'execute')!; + const command = fake.messages.find(m => m.type === 'execute')!; assert.deepEqual(fake.messages.at(-1), { type: 'cancel', id: command.id }); fake.child.emit('message', { id: command.id, @@ -224,7 +327,7 @@ test('executor cancellation targets the active request and preserves mutation ce test('executor ignores a cleanup exemption on non-cancellation failures', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -234,7 +337,7 @@ test('executor ignores a cleanup exemption on non-cancellation failures', async ); const execution = sandbox.execute(request); await dispatch; - const command = fake.messages.find((message) => message.type === 'execute')!; + const command = fake.messages.find(message => message.type === 'execute')!; fake.child.emit('message', { id: command.id, ok: false, @@ -269,7 +372,8 @@ test('executor rejects mismatched results as uncertain and fences subsequent com await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted, ); await assert.rejects(sandbox.execute(request), /unavailable/); await sandbox.close(); @@ -277,7 +381,7 @@ test('executor rejects mismatched results as uncertain and fences subsequent com test('executor close drains an active command before closing IPC', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -288,16 +392,16 @@ test('executor close drains an active command before closing IPC', async () => { const execution = sandbox.execute(request); await dispatch; const closing = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal( - fake.messages.some((m) => m.type === 'close'), + fake.messages.some(m => m.type === 'close'), false, ); - const command = fake.messages.find((m) => m.type === 'execute')!; + const command = fake.messages.find(m => m.type === 'execute')!; fake.child.emit('message', { id: command.id, ok: true, result }); assert.deepEqual(await execution, result); await closing; - assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'close').length, 1); await assert.rejects(sandbox.execute(request), /unavailable/); }); @@ -321,7 +425,7 @@ test('executor close resolves when the child exits during the close handshake', }, }); await sandbox.close(); - assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'close').length, 1); await assert.rejects(sandbox.execute(request), /unavailable/); }); @@ -365,7 +469,7 @@ test('executor close skips the handshake once the child is already lost', async fake.child.emit('exit', 1, null); await sandbox.close(); assert.equal( - fake.messages.some((m) => m.type === 'close'), + fake.messages.some(m => m.type === 'close'), false, ); await assert.rejects(sandbox.execute(request), /unavailable/); @@ -377,17 +481,20 @@ test('executor startup loss is not reported as an applied mutation', async () => { workspaceRoot: '/workspace' }, (path, args, options) => { const child = fake.fork(path, args, options); - queueMicrotask(() => child.emit('error', new Error('startup failed'))); + queueMicrotask(() => + child.emit('error', new Error('startup failed')), + ); return child; }, ); await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted, ); assert.equal( - fake.messages.some((m) => m.type === 'execute'), + fake.messages.some(m => m.type === 'execute'), false, ); await sandbox.close(); @@ -409,7 +516,7 @@ test('executor shutdown receipt fences reuse before the OS exit event', async () ); await assert.rejects(sandbox.execute(request)); await assert.rejects(sandbox.execute(request), /unavailable/); - assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'execute').length, 1); await sandbox.close(); }); @@ -431,7 +538,8 @@ test('executor preserves bounded startup diagnostics and conventional host setti ok: false, mutation: false, code: 'COMMAND_UNAVAILABLE', - errorMessage: 'Native sandbox dependencies are unavailable: bubblewrap', + errorMessage: + 'Native sandbox dependencies are unavailable: bubblewrap', }), ); const sandbox = new NativeProcessWorkspaceCommandSandbox( @@ -465,7 +573,10 @@ test('executor matches POSIX names exactly and folds names only on Windows', () https_proxy: 'http://proxy:8080', }); assert.deepEqual( - nativeExecutorEnvironment({ Path: 'C:\\bin', Temp: 'C:\\temp' }, 'win32'), + nativeExecutorEnvironment( + { Path: 'C:\\bin', Temp: 'C:\\temp' }, + 'win32', + ), { Path: 'C:\\bin', Temp: 'C:\\temp' }, ); }); @@ -486,7 +597,8 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', return {}; }, wrapCommand(command) { - if (failure === 'wrapper') throw new Error('wrapper failed'); + if (failure === 'wrapper') + throw new Error('wrapper failed'); return command; }, }, @@ -503,10 +615,12 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted && error.code === - (failure === 'abort' ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE'), + (failure === 'abort' + ? 'EXECUTION_ABORTED' + : 'COMMAND_UNAVAILABLE'), ); assert.equal( - fake.messages.some((m) => m.type === 'execute'), + fake.messages.some(m => m.type === 'execute'), false, ); await sandbox.close(); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index e1a02498..d2ea0d72 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -1,11 +1,21 @@ -import { fork } from 'node:child_process'; +import { execFile, fork } from 'node:child_process'; import { randomUUID } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, realpath } from 'node:fs/promises'; +import { isAbsolute, join, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; -import { isWorkspaceToolRequest, isWorkspaceToolResult } from './protocol.js'; +import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + isWorkspaceToolRequest, + isWorkspaceToolResult, +} from './protocol.js'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; import type { WorkspaceCommandSandbox } from './workspace.js'; import type { + BridgeWorkspaceProgrammaticRequest, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, } from './protocol.js'; @@ -13,7 +23,84 @@ import type { export type NativeProcessSandboxOptions = Omit< NativeSrtWorkspaceCommandSandboxOptions, 'manager' | 'spawnCommand' | 'platform' ->; +> & { + /** Hardened Code API egress gateway used for execution-scoped files. */ + programmaticFileUpstream?: string; +}; + +const execFileAsync = promisify(execFile); + +async function systemProgrammaticExecutable( + name: string, + workspaceRoot: string, +): Promise { + // Preflight runs outside SRT. Never execute a workspace-controlled PATH + // entry (including cwd, node_modules/.bin, or a symlink to another root). + for (const directory of ['/opt/homebrew/bin', '/usr/local/bin', '/usr/bin', '/bin', '/home/linuxbrew/.linuxbrew/bin']) { + const candidate = join(directory, name); + try { + const canonical = await trustedProgrammaticExecutable(candidate, workspaceRoot); + if (!['/opt/homebrew/', '/usr/local/', '/usr/bin/', '/bin/', '/home/linuxbrew/.linuxbrew/'].some(root => canonical.startsWith(root))) continue; + return canonical; + } catch { + // Continue through the bounded PATH entries. + } + } +} + +export async function trustedProgrammaticExecutable(candidate: string, workspaceRoot: string): Promise { + if (!isAbsolute(candidate)) throw new Error('Programmatic executable must be absolute'); + const [canonical, root] = await Promise.all([realpath(candidate), realpath(workspaceRoot)]); + const path = relative(root, canonical); + if (path === '' || (!isAbsolute(path) && path !== '..' && !path.startsWith(`..${sep}`))) { + throw new Error('Programmatic executable must be outside the workspace'); + } + await access(canonical, fsConstants.X_OK); + return canonical; +} + +async function resolveProgrammaticShell( + options: NativeProcessSandboxOptions, +): Promise { + const environment = options.environment ?? process.env; + const shellPath = + options.shellPath != null + ? await trustedProgrammaticExecutable(options.shellPath, options.workspaceRoot) + : await systemProgrammaticExecutable('bash', options.workspaceRoot); + const jqPath = await systemProgrammaticExecutable('jq', options.workspaceRoot); + if (!shellPath || !jqPath) { + throw new WorkspaceToolError( + 'Native programmatic execution requires trusted host installations of Bash 5.2 or newer and jq', + 'COMMAND_UNAVAILABLE', + ); + } + try { + const [{ stdout: bashVersion }] = await Promise.all([ + execFileAsync(shellPath, ['--version'], { + env: nativeExecutorEnvironment(environment), + timeout: 5_000, + }), + execFileAsync(jqPath, ['--version'], { + env: nativeExecutorEnvironment(environment), + timeout: 5_000, + }), + ]); + const match = /version\s+(\d+)\.(\d+)/i.exec(bashVersion); + if ( + !match || + Number(match[1]) < 5 || + (Number(match[1]) === 5 && Number(match[2]) < 2) + ) { + throw new Error('unsupported Bash version'); + } + } catch { + throw new WorkspaceToolError( + 'Native programmatic execution requires trusted host installations of Bash 5.2 or newer and jq', + 'COMMAND_UNAVAILABLE', + ); + } + return shellPath; +} /** Only OS discovery and conventional proxy settings cross into the executor. * In particular, never inherit NODE_OPTIONS, bridge identity, or app secrets. */ @@ -77,20 +164,22 @@ export function nativeExecutorEnvironment( * deadline, as opposed to a failure the executor reported explicitly. */ class NativeExecutorUnavailableError extends WorkspaceToolError { constructor(mutation: boolean) { - super('Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation); + super( + 'Native executor is unavailable', + 'COMMAND_UNAVAILABLE', + mutation, + ); this.name = 'NativeExecutorUnavailableError'; } } /** One persistent, process-isolated SRT manager per workspace. No automatic * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ -export class NativeProcessWorkspaceCommandSandbox - implements WorkspaceCommandSandbox -{ +export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private child?: ChildProcess; private ready?: Promise; - private active?: Promise; + private active?: Promise; private closing?: Promise; private failed = false; private terminationTimer?: ReturnType; @@ -122,12 +211,17 @@ export class NativeProcessWorkspaceCommandSandbox } private async start(): Promise { + const programmaticShellPath = this.options.programmaticFileUpstream + ? await resolveProgrammaticShell(this.options) + : this.options.shellPath; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], { execArgv: [], - env: nativeExecutorEnvironment(this.options.environment ?? process.env), + env: nativeExecutorEnvironment( + this.options.environment ?? process.env, + ), stdio: ['ignore', 'ignore', 'ignore', 'ipc'], serialization: 'json', }, @@ -165,6 +259,8 @@ export class NativeProcessWorkspaceCommandSandbox const processTerminationConfirmed = code === 'EXECUTION_ABORTED' && message.requiresQuarantine === false; + const mutationMayHaveCommitted = + pending.mutation && message.mutation !== false; pending.reject( new WorkspaceToolError( typeof message.errorMessage === 'string' && @@ -172,8 +268,8 @@ export class NativeProcessWorkspaceCommandSandbox ? message.errorMessage : 'Native executor request failed', code, - pending.mutation && message.mutation !== false, - pending.mutation && !processTerminationConfirmed, + mutationMayHaveCommitted, + mutationMayHaveCommitted && !processTerminationConfirmed, ), ); } @@ -192,6 +288,7 @@ export class NativeProcessWorkspaceCommandSandbox allowedDomains, homeDirectory, shellPath, + programmaticFileUpstream, } = this.options; await this.rpc( 'prepare', @@ -202,13 +299,14 @@ export class NativeProcessWorkspaceCommandSandbox protectedPaths, allowedDomains, homeDirectory, - shellPath, + shellPath: programmaticShellPath ?? shellPath, + programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, }, 30_000, false, - ).catch((error) => { + ).catch(error => { this.failed = true; this.terminate(); throw error; @@ -223,7 +321,10 @@ export class NativeProcessWorkspaceCommandSandbox !isWorkspaceToolRequest(request) || request.operation !== 'execute_command' ) { - throw new WorkspaceToolError('Invalid native command', 'INVALID_REQUEST'); + throw new WorkspaceToolError( + 'Invalid native command', + 'INVALID_REQUEST', + ); } if (this.active || this.closing || this.failed) throw this.unavailable(false); @@ -236,12 +337,114 @@ export class NativeProcessWorkspaceCommandSandbox } } + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + if (this.active || this.closing || this.failed) + throw this.unavailable(false); + if (!this.options.programmaticFileUpstream) { + throw new WorkspaceToolError( + 'Native programmatic file transport is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const active = this.executeProgrammaticOnce( + request, + workspaceId, + signal, + ); + this.active = active; + try { + return await active; + } finally { + this.active = undefined; + } + } + + private async executeProgrammaticOnce( + request: BridgeWorkspaceProgrammaticRequest, + workspaceId: string, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + let credentials: Record | undefined; + let wrappedCommand: string | undefined; + try { + await this.prepare(); + if (signal?.aborted) throw new Error('aborted'); + credentials = await this.options.maskedEnvironment?.resolve(signal); + if (signal?.aborted) throw new Error('aborted'); + wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( + NATIVE_PROGRAMMATIC_COMMAND, + process.platform, + ); + if (signal?.aborted) throw new Error('aborted'); + } catch (error) { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError(error.message, error.code, false) + : new WorkspaceToolError( + 'Native programmatic executor setup failed before dispatch', + 'COMMAND_UNAVAILABLE', + ); + } + const result = await this.rpc( + 'programmatic', + { + programmaticRequest: request, + workspaceId, + credentials, + wrappedCommand, + }, + (request.body.run_timeout ?? 30_000) * + ((request.body.replay_tool_count ?? 0) > 0 ? 2 : 1) + + (Math.ceil( + request.body.files.filter(file => 'id' in file).length / 4, + ) + + Math.ceil( + (request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, + )) * + (request.body.transfer_timeout_ms ?? 30_000) + + 5_000, + true, + signal, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + true, + ); + } + if (typeof result !== 'object' || result === null) { + this.failed = true; + this.terminate(); + throw this.unavailable(true); + } + return result; + } + private async executeOnce( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, ): Promise { if (signal?.aborted) - throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + ); let credentials: Record | undefined; let wrappedCommand: string | undefined; try { @@ -258,7 +461,10 @@ export class NativeProcessWorkspaceCommandSandbox // No execute RPC has been sent: setup, token refresh and wrapping cannot // have mutated the workspace. Do not quarantine it for setup failures. if (signal?.aborted) - throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + ); throw error instanceof WorkspaceToolError ? new WorkspaceToolError(error.message, error.code, false) : new WorkspaceToolError( @@ -324,7 +530,7 @@ export class NativeProcessWorkspaceCommandSandbox reject(this.unavailable(mutation)); }; try { - child.send({ type, id, ...payload }, (error) => { + child.send({ type, id, ...payload }, error => { if (error) sendFailed(); }); } catch { @@ -353,9 +559,12 @@ export class NativeProcessWorkspaceCommandSandbox await this.ready?.catch(() => undefined); try { if (this.child?.connected && !this.failed) - await this.rpc('close', {}, 10_000, false).catch((error: unknown) => { - if (!(error instanceof NativeExecutorUnavailableError)) throw error; - }); + await this.rpc('close', {}, 10_000, false).catch( + (error: unknown) => { + if (!(error instanceof NativeExecutorUnavailableError)) + throw error; + }, + ); } finally { this.failed = true; this.terminate(); diff --git a/packages/code/src/native-programmatic-live.test.ts b/packages/code/src/native-programmatic-live.test.ts new file mode 100644 index 00000000..2bc22356 --- /dev/null +++ b/packages/code/src/native-programmatic-live.test.ts @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { AddressInfo } from 'node:net'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +test('real SRT prevents speculative network effects under trusted-vm', { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, +}, async () => { + const root = await mkdtemp(join(tmpdir(), 'native-ptc-effects-')); + let effects = 0; + const server = createServer((_req, res) => { effects += 1; res.end('ok'); }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as AddressInfo).port; + const executor = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + programmaticFileUpstream: `http://127.0.0.1:${port}`, + }); + try { + await executor.prepare(); + const result = await executor.executeProgrammatic('primary', { headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'isolated-canary', replay_tool_count: 1, + run_timeout: 5000, + files: [{ name: 'main.sh', content: `curl --noproxy '*' --connect-timeout 1 --max-time 2 -s -X POST http://127.0.0.1:${port}/effect >/dev/null\nprintf once >> commit.txt\n` }], + } }) as { run: { code: number } }; + assert.equal(result.run.code, 0); + assert.equal(effects, 1, 'probe must not emit a network effect'); + assert.equal(await readFile(join(root, 'commit.txt'), 'utf8'), 'once'); + } finally { + try { await executor.close(); } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + } + } +}); diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts new file mode 100644 index 00000000..82650f95 --- /dev/null +++ b/packages/code/src/native-programmatic.test.ts @@ -0,0 +1,608 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { AddressInfo } from 'node:net'; +import type { BridgeWorkspaceProgrammaticRequest } from './protocol.js'; + +test('stages skill files privately and returns generated artifacts', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-test-')); + const uploads = new Map(); + let downloadCount = 0; + const server = createServer(async (req, res) => { + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant'); + if (req.method === 'GET') { + downloadCount += 1; + assert.match( + req.url ?? '', + /\/sessions\/input-session\/objects\/skill-file$/, + ); + res.end('skill-value'); + return; + } + assert.equal(req.method, 'PUT'); + assert.equal(req.headers['content-type'], 'text/plain'); + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + uploads.set( + decodeURIComponent(req.headers['x-original-filename'] as string), + Buffer.concat(chunks), + ); + res.statusCode = 200; + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + let observedDataDirectory = ''; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + observedDataDirectory = dataDirectory; + assert.equal( + await readFile( + join(dataDirectory, 'skills/example/reference.txt'), + 'utf8', + ), + 'skill-value', + ); + await writeFile(join(dataDirectory, 'result.txt'), 'artifact'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const request: BridgeWorkspaceProgrammaticRequest = { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'execution-one', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'printf done' }, + { name: '_ptc_history.json', content: '{}' }, + { + name: 'skills/example/reference.txt', + id: 'skill-file', + storage_session_id: 'input-session', + input_cache_key: createHash('sha256') + .update('stable-authorized-input-identity') + .digest('hex'), + }, + ], + }, + }; + try { + const result = await executor.execute(request, 'primary'); + const replay = await executor.execute(request, 'primary'); + assert.equal(result.run.stdout, 'done\n'); + assert.equal(replay.run.stdout, 'done\n'); + assert.equal(result.session_id, 'output-session'); + assert.equal(downloadCount, 1); + await executor.execute( + { + ...request, + body: { ...request.body, execution_id: 'execution-two' }, + }, + 'primary', + ); + assert.equal(downloadCount, 2); + assert.equal(result.files.length, 1); + assert.equal(result.files[0]?.name, 'result.txt'); + assert.equal(uploads.get('result.txt')?.toString(), 'artifact'); + assert.deepEqual( + await readdir(observedDataDirectory).catch(() => []), + [], + ); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('reports unsupported and rejected artifacts without invalidating a completed command', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-test-')); + const uploads = new Map(); + const server = createServer(async (req, res) => { + const name = decodeURIComponent(req.headers['x-original-filename'] as string); + uploads.set(name, req.headers['content-type']); + for await (const _chunk of req) { + // Drain the bounded request body before responding. + } + res.statusCode = name === 'image.png' ? 503 : 200; + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(_request, dataDirectory) { + await writeFile(join(dataDirectory, '_ptc_report.csv'), 'a,b\n1,2\n'); + await writeFile(join(dataDirectory, 'image.png'), 'not-a-real-png'); + await writeFile(join(dataDirectory, 'model.bin'), 'unsupported'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ); + assert.equal(result.run.code, 0); + assert.deepEqual(result.files.map(file => file.name), ['_ptc_report.csv']); + assert.deepEqual(result.artifact_delivery, { + code: 'artifact_delivery_failed', + status: 'partial', + attempted: 3, + delivered: 1, + failed: 2, + }); + assert.equal(uploads.get('_ptc_report.csv'), 'text/csv'); + assert.equal(uploads.get('image.png'), 'image/png'); + assert.equal(uploads.has('model.bin'), false); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('reports artifact transport failure without quarantining a completed command', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-transport-test-')); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + fetchImpl: async () => { + throw new TypeError('transport unavailable'); + }, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(_request, dataDirectory) { + await writeFile(join(dataDirectory, 'result.txt'), 'artifact'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ); + assert.equal(result.run.code, 0); + assert.deepEqual(result.files, []); + assert.deepEqual(result.artifact_delivery, { + code: 'artifact_delivery_failed', + status: 'failed', + attempted: 1, + delivered: 0, + failed: 1, + }); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('preflights copy-on-write isolation and removes its private snapshot', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-preflight-test-')); + let executionDirectory = ''; + let probes = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + executionDirectory = await mkdtemp(join(scratch, 'execution-')); + return executionDirectory; + }, + async createProgrammaticProbeWorkspace(directory) { + probes += 1; + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + try { + await executor.prepare(); + assert.equal(probes, 1); + assert.deepEqual(await readdir(executionDirectory).catch(() => []), []); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('keeps replay probes read-only and commits the script exactly once', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-probe-test-')); + const phases: boolean[] = []; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async createProgrammaticProbeWorkspace(executionDirectory) { + const workspace = join(executionDirectory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic( + _request, + dataDirectory, + _signal, + options, + ) { + phases.push(options?.probe === true); + if (options?.probe === true) { + assert.match(options.workspaceRoot ?? '', /\/workspace$/); + } + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: options?.probe ? 1 : 0, + stdout: options?.probe ? 'probe\n' : 'commit\n', + stderr: options?.probe ? 'expected probe denial\n' : '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'probe-then-commit', + replay_tool_count: 1, + session_id: 'execution-session', + files: [ + { name: 'main.sh', content: 'printf done' }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }, + 'primary', + ); + assert.deepEqual(phases, [true, false]); + assert.equal(result.run.stdout, 'commit\n'); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('returns pending calls from the private control file even when stdout truncates', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-control-test-')); + let phases = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async createProgrammaticProbeWorkspace(executionDirectory) { + const workspace = join(executionDirectory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic( + _request, + dataDirectory, + _signal, + options, + ) { + assert.equal(options?.probe, true); + phases += 1; + await writeFile( + join(dataDirectory, '_ptc_pending_result.json'), + JSON.stringify({ + pending: [ + { + call_id: 'call_001', + tool_name: 'lookup', + input: {}, + }, + ], + }), + ); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'x'.repeat(256 * 1024), + stderr: '', + truncated: true, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'truncated-control', + replay_tool_count: 1, + session_id: 'execution-session', + files: [ + { name: 'main.sh', content: 'lookup "{}"' }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }, + 'primary', + ); + assert.equal(phases, 1); + assert.equal(result.run.stdout, ''); + assert.equal(result.run.stderr, ''); + assert.deepEqual(JSON.parse(result.pending_tool_calls_payload ?? ''), { + pending: [{ call_id: 'call_001', tool_name: 'lookup', input: {} }], + }); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('rejects traversal before creating execution state', async () => { + let allocated = false; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + allocated = true; + return '/unused'; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + await assert.rejects( + executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + files: [{ name: '../main.sh', content: 'echo unsafe' }], + }, + }, + 'primary', + ), + /Invalid selected-workspace programmatic request/, + ); + assert.equal(allocated, false); +}); + +test('rejects artifacts above the negotiated byte ceiling before upload', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-output-limit-')); + let uploads = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + fetchImpl: async () => { + uploads += 1; + return new Response(); + }, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await writeFile(join(dataDirectory, 'artifact.txt'), 'too large'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + await assert.rejects( + executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + max_output_file_bytes: 4, + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ), + /exceeds the file limit/, + ); + assert.equal(uploads, 0); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +for (const failure of ['truncated', 'process-error']) test(`a failed speculative probe does not quarantine the real workspace (${failure})`, async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-failed-probe-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async createProgrammaticProbeWorkspace(directory) { const root = join(directory, 'workspace'); await mkdir(root); return root; }, + async executeProgrammatic(request, _directory, _signal, options) { + assert.equal(options?.probe, true); + if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true); + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + exitCode: 0, stdout: '', stderr: '', truncated: true, timedOut: false }; + }, + }, + }); + await assert.rejects(executor.execute({ headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'session', replay_tool_count: 1, + files: [{ name: 'main.sh', content: 'true' }], + } }, 'primary'), (error: unknown) => { + assert.match(String(error), /probe output exceeded/); + assert.equal((error as { requiresQuarantine: boolean }).requiresQuarantine, false); + assert.equal((error as { mutationMayHaveCommitted: boolean }).mutationMayHaveCommitted, false); + return true; + }); +}); + +test('unchanged inputs do not consume the negotiated output budget', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-unchanged-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async executeProgrammatic(request) { + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + exitCode: 0, stdout: '', stderr: '', truncated: false, timedOut: false }; + }, + }, + }); + const result = await executor.execute({ headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'session', max_output_file_bytes: 1, + files: [{ name: 'main.sh', content: 'true' }, { name: 'input.txt', content: 'unchanged input' }], + } }, 'primary'); + assert.deepEqual(result.files, []); +}); + +test('stops admitting downloads and drains in-flight transfers before cleanup', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-transfer-test-')); + let executionDirectory = ''; + let requestCount = 0; + const server = createServer((req, res) => { + requestCount += 1; + if (requestCount === 1) { + res.statusCode = 503; + res.end(); + return; + } + setTimeout(() => res.end('in-flight'), 25); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + executionDirectory = await mkdtemp(join(scratch, 'execution-')); + return executionDirectory; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + const request: BridgeWorkspaceProgrammaticRequest = { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [ + ...Array.from({ length: 8 }, (_, index) => ({ + name: `inputs/${index}.txt`, + id: `input-${index}`, + storage_session_id: 'input-session', + })), + { name: 'main.sh', content: 'printf done' }, + ], + }, + }; + try { + const startedAt = performance.now(); + await assert.rejects( + executor.execute(request, 'primary'), + /Programmatic input download failed with HTTP 503/, + ); + assert.ok(performance.now() - startedAt >= 20); + assert.ok(requestCount <= 4); + assert.deepEqual(await readdir(executionDirectory).catch(() => []), []); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts new file mode 100644 index 00000000..42974b79 --- /dev/null +++ b/packages/code/src/native-programmatic.ts @@ -0,0 +1,816 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; +import { cp, mkdir, open, readdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, relative, sep } from 'node:path'; + +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES, + bridgeArtifactMediaType, + isBridgeWorkspaceProgrammaticRequest, + isSafePortableRelativePath, + isSupportedBridgeArtifactName, +} from './protocol.js'; +import { validateFileRelayUpstream } from './relay.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { + BridgeProgrammaticPayloadFile, + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; +import type { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; + +const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +const EXECUTION_MAIN_FILE = 'main.sh'; +const EXECUTION_HISTORY_FILE = '_ptc_history.json'; +const EXECUTION_CONTROL_FILE = '_ptc_pending_result.json'; +export const NATIVE_PROGRAMMATIC_COMMAND = + 'exec "$LIBRECHAT_CODE_BASH_PATH" "$LIBRECHAT_CODE_DATA_DIR/main.sh"'; +const TRANSFER_TIMEOUT_MS = 30_000; +const TRANSFER_CONCURRENCY = 4; +const MAX_WALK_ENTRIES = 2_000; +const INPUT_CACHE_MAX_ENTRIES = 64; +const INPUT_CACHE_MAX_BYTES = 32 * 1024 * 1024; +const CONTROL_PAYLOAD_MAX_BYTES = 512 * 1024; + +type ProgrammaticFileResult = { + id: string; + name: string; + storage_session_id: string; + modified_from?: { id: string; storage_session_id: string }; +}; + +type ProgrammaticResult = { + language: 'bash'; + version: string; + session_id: string; + files: ProgrammaticFileResult[]; + artifact_delivery?: { + code: 'artifact_delivery_failed'; + status: 'partial' | 'failed'; + attempted: number; + delivered: number; + failed: number; + }; + pending_tool_calls_payload?: string; + run: { + stdout: string; + stderr: string; + code: number | null; + signal: string | null; + output: string; + memory: null; + message: string | null; + status: string | null; + cpu_time: null; + wall_time: number; + }; +}; + +type InputBaseline = { + sha256: string; + source?: { id: string; storage_session_id: string }; +}; + +function sha256(value: Uint8Array): string { + return createHash('sha256').update(value).digest('hex'); +} + +function outputFileId(): string { + return randomBytes(18).toString('base64url').slice(0, 21); +} + +function localPath(root: string, name: string): string { + if (!isSafePortableRelativePath(name)) { + throw new WorkspaceToolError( + 'Invalid programmatic file path', + 'INVALID_PATH', + ); + } + const path = join(root, ...name.split('/')); + const child = relative(root, path); + if (child === '' || child === '..' || child.startsWith(`..${sep}`)) { + throw new WorkspaceToolError( + 'Invalid programmatic file path', + 'INVALID_PATH', + ); + } + return path; +} + +async function readBoundedResponse( + response: Response, + signal: AbortSignal, +): Promise { + const declaredLength = response.headers.get('content-length'); + if ( + declaredLength != null && + (!/^\d+$/.test(declaredLength) || + Number(declaredLength) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) + ) { + await response.body?.cancel(); + throw new WorkspaceToolError( + 'Programmatic input exceeds the file limit', + 'READ_LIMIT_EXCEEDED', + ); + } + if (!response.body) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let bytes = 0; + try { + for (;;) { + if (signal.aborted) throw signal.reason; + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) { + throw new WorkspaceToolError( + 'Programmatic input exceeds the file limit', + 'READ_LIMIT_EXCEEDED', + ); + } + chunks.push(Buffer.from(value)); + } + } finally { + await reader.cancel().catch(() => undefined); + } + return Buffer.concat(chunks, bytes); +} + +async function mapConcurrent( + values: readonly T[], + concurrency: number, + action: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let next = 0; + let failed = false; + let failure: unknown; + await Promise.all( + Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + for (;;) { + if (failed) return; + const index = next++; + if (index >= values.length) return; + try { + results[index] = await action(values[index]!); + } catch (error) { + if (!failed) { + failed = true; + failure = error; + } + return; + } + } + }, + ), + ); + if (failed) throw failure; + return results; +} + +async function listRegularFiles(root: string): Promise { + const files: string[] = []; + const pending = ['']; + let entries = 0; + while (pending.length > 0) { + const directory = pending.pop()!; + for (const entry of await readdir(join(root, directory), { + withFileTypes: true, + })) { + if (++entries > MAX_WALK_ENTRIES) { + throw new WorkspaceToolError( + 'Programmatic output contains too many entries', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const name = directory ? `${directory}/${entry.name}` : entry.name; + if (!isSafePortableRelativePath(name)) continue; + if (entry.isSymbolicLink()) continue; + if (entry.isDirectory()) pending.push(name); + else if (entry.isFile()) files.push(name); + } + } + return files.sort(); +} + +export interface NativeWorkspaceProgrammaticOptions { + sandbox: Pick< + NativeSrtWorkspaceCommandSandbox, + 'createExecutionDirectory' | 'executeProgrammatic' + > & + Partial< + Pick + >; + upstreamUrl: string; + fetchImpl?: typeof fetch; +} + +/** + * Executes one replay-mode Bash PTC iteration in an attached workspace. + * Program code and injected files live in a private SRT scratch directory; + * the selected repository remains the command cwd and is never used as a + * transport cache. + */ +export class NativeWorkspaceProgrammaticExecutor { + private readonly upstream: URL; + private readonly fetchImpl: typeof fetch; + /** Parent-process cache: sandboxed children cannot inspect this memory. */ + private readonly inputCache = new Map< + string, + { bytes: Buffer; lastUsed: number } + >(); + private inputCacheBytes = 0; + + constructor(private readonly options: NativeWorkspaceProgrammaticOptions) { + this.upstream = validateFileRelayUpstream(options.upstreamUrl); + this.fetchImpl = options.fetchImpl ?? fetch; + } + + /** + * Prove copy-on-write isolation before the worker advertises Bash PTC. + * The probe uses the exact registered root and private scratch path that a + * real replay will use, then removes the snapshot before registration. + */ + async prepare(signal?: AbortSignal): Promise { + const createProbeWorkspace = + this.options.sandbox.createProgrammaticProbeWorkspace; + if (createProbeWorkspace == null) { + throw new WorkspaceToolError( + 'Selected-workspace PTC probe isolation is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const executionDirectory = + await this.options.sandbox.createExecutionDirectory(); + try { + await createProbeWorkspace.call( + this.options.sandbox, + executionDirectory, + signal, + ); + } finally { + await rm(executionDirectory, { recursive: true, force: true }); + } + } + + private cacheKey( + executionId: string | undefined, + file: Extract, + ): string | undefined { + return executionId && file.input_cache_key + ? `${executionId}:${file.input_cache_key}` + : undefined; + } + + private cachedInput(key: string): Buffer | undefined { + const cached = this.inputCache.get(key); + if (!cached) return undefined; + cached.lastUsed = Date.now(); + return cached.bytes; + } + + private cacheInput(key: string, bytes: Buffer): void { + if (bytes.byteLength > INPUT_CACHE_MAX_BYTES) return; + const existing = this.inputCache.get(key); + if (existing) this.inputCacheBytes -= existing.bytes.byteLength; + while ( + this.inputCache.size >= INPUT_CACHE_MAX_ENTRIES || + this.inputCacheBytes + bytes.byteLength > INPUT_CACHE_MAX_BYTES + ) { + let oldestKey: string | undefined; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [candidate, value] of this.inputCache) { + if (value.lastUsed < oldestAt) { + oldestAt = value.lastUsed; + oldestKey = candidate; + } + } + if (!oldestKey) break; + this.inputCacheBytes -= + this.inputCache.get(oldestKey)!.bytes.byteLength; + this.inputCache.delete(oldestKey); + } + this.inputCache.set(key, { bytes, lastUsed: Date.now() }); + this.inputCacheBytes += bytes.byteLength; + } + + private async downloadInput( + file: Extract, + grant: string, + executionId: string | undefined, + signal?: AbortSignal, + transferTimeoutMs = TRANSFER_TIMEOUT_MS, + ): Promise { + const key = this.cacheKey(executionId, file); + const cached = key ? this.cachedInput(key) : undefined; + if (cached) return cached; + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout(() => controller.abort(), transferTimeoutMs); + try { + const response = await this.fetchImpl( + new URL( + `sessions/${encodeURIComponent(file.storage_session_id)}/objects/${encodeURIComponent(file.id)}`, + `${this.upstream.toString().replace(/\/+$/, '')}/`, + ), + { + headers: { [EGRESS_GRANT_HEADER]: grant }, + redirect: 'error', + signal: controller.signal, + }, + ); + if (!response.ok) { + await response.body?.cancel(); + throw new WorkspaceToolError( + `Programmatic input download failed with HTTP ${response.status}`, + 'COMMAND_UNAVAILABLE', + ); + } + const bytes = await readBoundedResponse( + response, + controller.signal, + ); + if (key) this.cacheInput(key, bytes); + return bytes; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } + } + + async execute( + request: BridgeWorkspaceProgrammaticRequest, + workspaceId: string, + signal?: AbortSignal, + ): Promise { + if (!isBridgeWorkspaceProgrammaticRequest(request)) { + throw new WorkspaceToolError( + 'Invalid selected-workspace programmatic request', + 'INVALID_REQUEST', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + const grant = request.body.egress_grant; + const refFiles = request.body.files.filter( + ( + file, + ): file is Extract => + 'id' in file, + ); + if (refFiles.length > 0 && !grant) { + throw new WorkspaceToolError( + 'Programmatic input grant is unavailable', + 'INVALID_REQUEST', + ); + } + const executionDirectory = + await this.options.sandbox.createExecutionDirectory(); + const inputDirectory = join(executionDirectory, 'inputs'); + let dataDirectory = join(executionDirectory, 'final'); + const baselines = new Map(); + let totalInputBytes = 0; + const startedAt = performance.now(); + let commandDispatched = false; + try { + await mkdir(inputDirectory, { mode: 0o700 }); + await mapConcurrent( + request.body.files, + TRANSFER_CONCURRENCY, + async (file): Promise => { + const bytes = + 'content' in file + ? Buffer.from(file.content) + : await this.downloadInput( + file, + grant!, + request.body.execution_id, + signal, + request.body.transfer_timeout_ms, + ); + totalInputBytes += bytes.byteLength; + if ( + totalInputBytes > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ) { + throw new WorkspaceToolError( + 'Programmatic inputs exceed the total byte limit', + 'READ_LIMIT_EXCEEDED', + ); + } + const path = localPath(inputDirectory, file.name); + await mkdir(dirname(path), { + recursive: true, + mode: 0o700, + }); + await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); + baselines.set(file.name, { + sha256: sha256(bytes), + ...('id' in file + ? { + source: { + id: file.id, + storage_session_id: + file.storage_session_id, + }, + } + : {}), + }); + }, + ); + + const run = async ( + directory: string, + probe: boolean, + workspaceRoot?: string, + ): Promise => { + await cp(inputDirectory, directory, { + recursive: true, + force: false, + errorOnExist: true, + mode: constants.COPYFILE_FICLONE, + }); + if (!probe) commandDispatched = true; + return await this.options.sandbox.executeProgrammatic( + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId, + command: NATIVE_PROGRAMMATIC_COMMAND, + timeoutMs: Math.min( + request.body.run_timeout ?? + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ), + maxOutputBytes: + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + }, + directory, + signal, + { probe, workspaceRoot }, + ); + }; + + const readPending = async ( + directory: string, + ): Promise => { + try { + const path = join(directory, EXECUTION_CONTROL_FILE); + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if ( + !metadata.isFile() || + metadata.size === 0 || + metadata.size > CONTROL_PAYLOAD_MAX_BYTES + ) { + throw new WorkspaceToolError( + 'Native programmatic control frame is invalid', + 'COMMAND_UNAVAILABLE', + ); + } + return await handle.readFile('utf8'); + } finally { + await handle.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') + return; + throw error; + } + }; + + if ((request.body.replay_tool_count ?? 0) > 0) { + const probeDirectory = join(executionDirectory, 'probe'); + const createProbeWorkspace = + this.options.sandbox.createProgrammaticProbeWorkspace; + if (createProbeWorkspace == null) { + throw new WorkspaceToolError( + 'Selected-workspace PTC probe isolation is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const probeWorkspace = await createProbeWorkspace.call( + this.options.sandbox, + executionDirectory, + signal, + ); + const probeResult = await run( + probeDirectory, + true, + probeWorkspace, + ); + const pending = await readPending(probeDirectory); + if (pending) { + return this.result( + request, + { + ...probeResult, + /** Probe output is speculative and the script will + * run once under its real policy after tool + * resolution. Never duplicate it or expose + * expected read-only policy denials to callers. */ + stdout: '', + stderr: '', + }, + [], + performance.now() - startedAt, + pending, + ); + } + if (probeResult.truncated) { + throw new WorkspaceToolError( + 'Native programmatic probe output exceeded its limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + if ( + probeResult.timedOut || + probeResult.signal + ) { + return this.result( + request, + probeResult, + [], + performance.now() - startedAt, + ); + } + /** A read-only probe commonly exits non-zero after it reaches + * an intentional workspace write denial. With no pending call, + * run the script once under its real policy so ordinary writes + * and their resulting exit status are evaluated exactly once. */ + } + + const commandResult = await run(dataDirectory, false); + if (await readPending(dataDirectory)) { + throw new WorkspaceToolError( + 'Native programmatic commit pass requested an unexpected replay tool', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + } + if (commandResult.truncated) { + throw new WorkspaceToolError( + 'Native programmatic output exceeded its limit', + 'WRITE_LIMIT_EXCEEDED', + true, + true, + ); + } + + const outputSessionId = request.body.output_session_id; + const outputNames = (await listRegularFiles(dataDirectory)).filter( + name => + name !== EXECUTION_MAIN_FILE && + name !== EXECUTION_HISTORY_FILE && + name !== EXECUTION_CONTROL_FILE && + !name.startsWith('skills/'), + ); + const changed: Array<{ + name: string; + bytes: Buffer; + source?: { id: string; storage_session_id: string }; + }> = []; + let totalOutputBytes = 0; + for (const name of outputNames) { + const path = localPath(dataDirectory, name); + const baseline = baselines.get(name); + const maxOutputFileBytes = Math.min( + request.body.max_output_file_bytes ?? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + ); + let bytes: Buffer; + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if (!metadata.isFile()) continue; + // An unchanged input is not an output. It may legitimately + // exceed the negotiated output ceiling, but never the + // protocol's bounded input limit. + if (metadata.size > (baseline ? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES : maxOutputFileBytes)) { + throw new WorkspaceToolError( + 'Programmatic output exceeds the file limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + bytes = await handle.readFile(); + } finally { + await handle.close(); + } + if (baseline?.sha256 === sha256(bytes)) continue; + if (bytes.byteLength > maxOutputFileBytes) { + throw new WorkspaceToolError( + 'Programmatic output exceeds the file limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + totalOutputBytes += bytes.byteLength; + if ( + totalOutputBytes > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ) { + throw new WorkspaceToolError( + 'Programmatic outputs exceed the total byte limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + changed.push({ name, bytes, source: baseline?.source }); + } + const maxOutputFiles = Math.min( + request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + ); + if (changed.length > maxOutputFiles) { + throw new WorkspaceToolError( + 'Programmatic output contains too many files', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const uploadable = changed.filter(({ name }) => + isSupportedBridgeArtifactName(name), + ); + if (uploadable.length > 0 && (!grant || !outputSessionId)) { + throw new WorkspaceToolError( + 'Programmatic output grant is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const uploadResults = await mapConcurrent( + uploadable, + TRANSFER_CONCURRENCY, + async ({ name, bytes, source }): Promise => { + const id = outputFileId(); + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout( + () => controller.abort(), + request.body.transfer_timeout_ms ?? TRANSFER_TIMEOUT_MS, + ); + try { + let response: Response; + try { + response = await this.fetchImpl( + new URL( + `sessions/${encodeURIComponent(outputSessionId!)}/objects/${id}`, + `${this.upstream.toString().replace(/\/+$/, '')}/`, + ), + { + method: 'PUT', + headers: { + [EGRESS_GRANT_HEADER]: grant!, + 'Content-Type': bridgeArtifactMediaType(name), + 'Content-Length': String(bytes.byteLength), + 'X-Original-Filename': encodeURIComponent(name), + }, + body: new Uint8Array(bytes), + redirect: 'error', + signal: controller.signal, + }, + ); + } catch (error) { + if (signal?.aborted) throw error; + return undefined; + } + await response.body?.cancel(); + if (!response.ok) { + return undefined; + } + return { + id, + name, + storage_session_id: outputSessionId!, + ...(source ? { modified_from: source } : {}), + }; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } + }, + ); + const files = uploadResults.filter( + (file): file is ProgrammaticFileResult => file != null, + ); + const artifactDelivery = + files.length < changed.length + ? { + code: 'artifact_delivery_failed' as const, + status: files.length > 0 ? ('partial' as const) : ('failed' as const), + attempted: changed.length, + delivered: files.length, + failed: changed.length - files.length, + } + : undefined; + return this.result( + request, + commandResult, + files, + performance.now() - startedAt, + undefined, + artifactDelivery, + ); + } catch (error) { + if (!commandDispatched) { + // The low-level command runner classifies any launched process as a + // possible mutation. A probe can only mutate its disposable snapshot, + // so translate that classification at this ownership boundary. + throw new WorkspaceToolError( + error instanceof Error ? error.message : 'Programmatic preparation failed', + error instanceof WorkspaceToolError ? error.code : 'COMMAND_UNAVAILABLE', + false, + false, + ); + } + if (error instanceof WorkspaceToolError) { + if ( + error.mutationMayHaveCommitted || + error.requiresQuarantine + ) { + throw error; + } + throw new WorkspaceToolError( + error.message, + error.code, + true, + true, + ); + } + throw new WorkspaceToolError( + 'Native programmatic execution failed after dispatch', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + } finally { + try { + await rm(executionDirectory, { recursive: true, force: true }); + } catch { + throw new WorkspaceToolError( + 'Native programmatic execution cleanup failed', + 'COMMAND_UNAVAILABLE', + commandDispatched, + commandDispatched, + ); + } + } + } + + private result( + request: BridgeWorkspaceProgrammaticRequest, + command: WorkspaceExecuteCommandResult, + files: ProgrammaticFileResult[], + elapsedMs: number, + pendingToolCallsPayload?: string, + artifactDelivery?: ProgrammaticResult['artifact_delivery'], + ): ProgrammaticResult { + return { + language: 'bash', + version: request.body.version, + // Code API masks the execution session separately from the writable + // output bucket. Sandbox results must identify the output bucket so the + // gateway can restore it to the caller-owned session after upload. + session_id: + request.body.output_session_id ?? request.body.session_id, + files, + ...(artifactDelivery ? { artifact_delivery: artifactDelivery } : {}), + ...(pendingToolCallsPayload + ? { pending_tool_calls_payload: pendingToolCallsPayload } + : {}), + run: { + stdout: command.stdout, + stderr: command.stderr, + code: command.exitCode, + signal: command.signal ?? null, + output: `${command.stdout}${command.stderr}`, + memory: null, + message: command.timedOut ? 'Execution timed out' : null, + status: command.timedOut ? 'timeout' : null, + cpu_time: null, + wall_time: elapsedMs / 1000, + }, + }; + } +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index cf790965..3252ee50 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -7,6 +7,7 @@ import { mkdtemp, mkdir, open, + readFile, realpath, rename, rm, @@ -25,7 +26,10 @@ import type { } from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; -import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { + CopyOnWriteCloneUnavailableError, + NativeSrtWorkspaceCommandSandbox, +} from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; @@ -54,6 +58,8 @@ function fakeManager( let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; let scratchSelectorSeenDuringWrap: string | undefined; + let networkSeenDuringWrap: SandboxRuntimeConfig['network'] | undefined; + let customConfigSeenDuringWrap: Partial | undefined; const manager = { isSupportedPlatform: () => true, async checkDependenciesAsync() { @@ -67,19 +73,30 @@ function fakeManager( askCallback = callback; if (options.initializeError) throw options.initializeError; }, - async wrapWithSandboxArgv(command: string) { + updateConfig(value: SandboxRuntimeConfig) { config = value; }, + async wrapWithSandboxArgv( + command: string, + _binShell?: string, + customConfig?: Partial, + ) { await options.beforeWrap?.(); - credentialSeenDuringWrap = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + networkSeenDuringWrap = config?.network; + customConfigSeenDuringWrap = customConfig; + credentialSeenDuringWrap = + process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; gitLfsRequiredSeenDuringWrap = process.env.GIT_CONFIG_VALUE_3; scratchSelectorSeenDuringWrap = process.env.CLAUDE_CODE_TMPDIR; const ambientGitEnvironment = Object.fromEntries( Object.entries(process.env).filter( - ([name, value]) => name.startsWith('GIT_CONFIG_') && value != null, + ([name, value]) => + name.startsWith('GIT_CONFIG_') && value != null, ), ); let gitEnvironment = ambientGitEnvironment; if (options.appendGitSafeDirectory) { - const index = Number(ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0'); + const index = Number( + ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0', + ); gitEnvironment = { ...(options.inheritedGitEnvironment ?? {}), GIT_CONFIG_COUNT: String(index + 1), @@ -130,10 +147,118 @@ function fakeManager( get scratchSelectorSeenDuringWrap() { return scratchSelectorSeenDuringWrap; }, + get networkSeenDuringWrap() { return networkSeenDuringWrap; }, + get customConfigSeenDuringWrap() { + return customConfigSeenDuringWrap; + }, }; } -test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async (t) => { +for (const trustedVm of [false, true]) test(`programmatic probe denies real-workspace writes and external effects (trusted=${trustedVm})`, async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + allowedDomains: ['api.example.com'], + ...(trustedVm ? { commandPolicy: { version: 1 as const, preset: 'trusted-vm' as const, + network: { outbound: 'unrestricted' as const, allowLocalBinding: true, allowAllUnixSockets: true }, + } } : {}), + }); + const dataDirectory = await sandbox.createExecutionDirectory(); + await sandbox.executeProgrammatic(request, dataDirectory, undefined, { + probe: true, + }); + assert.deepEqual(fake.customConfigSeenDuringWrap?.network, { + allowedDomains: [], + deniedDomains: [], + strictAllowlist: true, + allowUnixSockets: [], + allowAllUnixSockets: false, + allowLocalBinding: false, + }); + assert.deepEqual(fake.networkSeenDuringWrap, fake.customConfigSeenDuringWrap?.network); + assert.equal(fake.config?.network.strictAllowlist, !trustedVm); + assert.equal(fake.reset, true, 'probe proxy session must be revoked before restoring policy'); + assert.deepEqual(fake.customConfigSeenDuringWrap?.filesystem?.allowWrite, [ + await realpath(dataDirectory), + ]); + assert.equal( + fake.scratchSelectorSeenDuringWrap, + await realpath(dataDirectory), + ); + assert.ok( + fake.customConfigSeenDuringWrap?.filesystem?.denyWrite?.includes( + await realpath(root), + ), + ); + await sandbox.close(); +}); + +test('probe network cleanup failure fences executor reuse', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-probe-cleanup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: root, manager: fake.manager }); + const directory = await sandbox.createExecutionDirectory(); + const reset = fake.manager.reset; + fake.manager.reset = async () => { throw new Error('proxy shutdown failed'); }; + await assert.rejects(sandbox.executeProgrammatic(request, directory, undefined, { probe: true }), /probe network cleanup failed/); + await assert.rejects(sandbox.execute(request)); + fake.manager.reset = reset; + await sandbox.close(); +}); + +test('programmatic probes use a copy-on-write workspace without mutating the project', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'state.txt'), 'original'); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + let snapshot: string; + try { + snapshot = await sandbox.createProgrammaticProbeWorkspace(executionDirectory); + } catch (error) { + if (error instanceof CopyOnWriteCloneUnavailableError) { + t.skip('host filesystem does not support copy-on-write cloning'); + return; + } + throw error; + } + await writeFile(join(snapshot, 'state.txt'), 'probe-only'); + assert.equal(await readFile(join(root, 'state.txt'), 'utf8'), 'original'); + assert.equal(await readFile(join(snapshot, 'state.txt'), 'utf8'), 'probe-only'); +}); + +test('programmatic probes do not hide clone implementation failures as unsupported filesystems', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + spawnCommand() { + throw Object.assign(new Error('spawn /bin/cp ENOENT'), { code: 'ENOENT' }); + }, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + + await assert.rejects( + sandbox.createProgrammaticProbeWorkspace(executionDirectory), + (error: unknown) => + error instanceof WorkspaceToolError && + !(error instanceof CopyOnWriteCloneUnavailableError) && + error.message === 'Copy-on-write workspace clone failed unexpectedly', + ); +}); + +test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -164,15 +289,15 @@ test('exclusive lifecycle rejects a second workspace sharing an SRT manager', as assert.equal((await second.execute(request)).stdout, 'hello'); }); -test('exclusive lifecycle rejects overlapping commands and waits before resetting', async (t) => { +test('exclusive lifecycle rejects overlapping commands and waits before resetting', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); let entered!: () => void; - const wrapping = new Promise((resolve) => { + const wrapping = new Promise(resolve => { entered = resolve; }); let release!: () => void; - const gate = new Promise((resolve) => { + const gate = new Promise(resolve => { release = resolve; }); const fake = fakeManager({ @@ -191,7 +316,7 @@ test('exclusive lifecycle rejects overlapping commands and waits before resettin await assert.rejects(sandbox.execute(request), /active command/); const closing = sandbox.close(); const secondClose = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(fake.reset, false); await assert.rejects(sandbox.prepare(), /closing/); release(); @@ -200,7 +325,7 @@ test('exclusive lifecycle rejects overlapping commands and waits before resettin assert.equal(fake.reset, true); }); -test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async (t) => { +test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -226,16 +351,16 @@ test('exclusive lifecycle retains ownership after a failed reset until cleanup s await second.close(); }); -test('exclusive lifecycle waits for initialization before resetting the manager', async (t) => { +test('exclusive lifecycle waits for initialization before resetting the manager', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); let entered!: () => void; - const initializing = new Promise((resolve) => { + const initializing = new Promise(resolve => { entered = resolve; }); let release!: () => void; - const gate = new Promise((resolve) => { + const gate = new Promise(resolve => { release = resolve; }); fake.manager.initialize = async () => { @@ -249,7 +374,7 @@ test('exclusive lifecycle waits for initialization before resetting the manager' const preparing = sandbox.prepare(); await initializing; const closing = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(fake.reset, false); release(); await preparing; @@ -257,7 +382,7 @@ test('exclusive lifecycle waits for initialization before resetting the manager' assert.equal(fake.reset, true); }); -test('initializes SRT with a default-deny network and scrubbed worker credentials', async (t) => { +test('initializes SRT with a default-deny network and scrubbed worker credentials', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const identity = join(tmpdir(), 'librechat-code-identity.json'); t.after(() => rm(root, { recursive: true, force: true })); @@ -301,7 +426,7 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.ok(fake.config?.filesystem.denyRead.includes(canonicalHome)); assert.ok(fake.config?.filesystem.denyWrite.includes(canonicalIdentity)); assert.ok( - fake.config?.filesystem.denyWrite.some((path) => + fake.config?.filesystem.denyWrite.some(path => path.endsWith('/tmp/claude'), ), ); @@ -318,7 +443,7 @@ test('initializes SRT with a default-deny network and scrubbed worker credential await assert.rejects(access(scratchDirectory!)); }); -test('trusted-vm permits unmatched egress and local development sockets', async (t) => { +test('trusted-vm permits unmatched egress and local development sockets', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -351,7 +476,7 @@ test('trusted-vm permits unmatched egress and local development sockets', async ]); }); -test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { +test('provides an isolated scratch directory to commands and restores the host environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const originalTmpdir = process.env.TMPDIR; @@ -380,7 +505,7 @@ test('provides an isolated scratch directory to commands and restores the host e await assert.rejects(access(result.stdout)); }); -test('removes scratch storage when SRT initialization fails', async (t) => { +test('removes scratch storage when SRT initialization fails', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ initializeError: new Error('init failed') }); @@ -396,7 +521,7 @@ test('removes scratch storage when SRT initialization fails', async (t) => { assert.equal(fake.reset, true); }); -test('rejects workspaces nested inside SRT shared scratch storage', async (t) => { +test('rejects workspaces nested inside SRT shared scratch storage', async t => { if (process.platform === 'win32') return; const sharedRoot = '/tmp/claude'; await mkdir(sharedRoot, { recursive: true }); @@ -431,17 +556,17 @@ test('rejects a workspace that contains worker scratch storage', async () => { await sandbox.close(); }); -test('keeps concurrent sandbox scratch directories independent', async (t) => { +test('keeps concurrent sandbox scratch directories independent', async t => { const firstRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const secondRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(firstRoot, { recursive: true, force: true })); t.after(() => rm(secondRoot, { recursive: true, force: true })); let releaseWrap!: () => void; let wrapStarted!: () => void; - const wrapStartedPromise = new Promise((resolve) => { + const wrapStartedPromise = new Promise(resolve => { wrapStarted = resolve; }); - const holdWrap = new Promise((resolve) => { + const holdWrap = new Promise(resolve => { releaseWrap = resolve; }); const firstFake = fakeManager({ @@ -478,7 +603,7 @@ test('keeps concurrent sandbox scratch directories independent', async (t) => { await secondSandbox.close(); }); -test('removes scratch storage after a command revokes traversal permissions', async (t) => { +test('removes scratch storage after a command revokes traversal permissions', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -498,7 +623,7 @@ test('removes scratch storage after a command revokes traversal permissions', as await assert.rejects(access(result.stdout)); }); -test('scratch traversal never follows a descendant replaced after inspection', async (t) => { +test('scratch traversal never follows a descendant replaced after inspection', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-race-')); const outside = await mkdtemp(join(tmpdir(), 'librechat-code-outside-')); @@ -529,7 +654,7 @@ test('scratch traversal never follows a descendant replaced after inspection', a assert.equal((await stat(outsideChild)).mode & 0o777, 0o711); }); -test('scratch traversal removes command-created Darwin ACLs', async (t) => { +test('scratch traversal removes command-created Darwin ACLs', async t => { if (process.platform !== 'darwin') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -548,7 +673,7 @@ test('scratch traversal removes command-created Darwin ACLs', async (t) => { await assert.rejects(access(result.stdout)); }); -test('scratch traversal bounds descriptors and work across a deep tree', async (t) => { +test('scratch traversal bounds descriptors and work across a deep tree', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -565,12 +690,17 @@ test('scratch traversal bounds descriptors and work across a deep tree', async ( await restoreScratchTraversal(rootHandle); - assert.equal((await stat(directories[directories.length - 1])).mode & 0o777, 0o700); + assert.equal( + (await stat(directories[directories.length - 1])).mode & 0o777, + 0o700, + ); }); -test('scratch traversal rejects trees beyond its recovery depth limit', async (t) => { +test('scratch traversal rejects trees beyond its recovery depth limit', async t => { if (process.platform === 'win32') return; - const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-limit-')); + const root = await mkdtemp( + join(tmpdir(), 'librechat-code-scratch-depth-limit-'), + ); t.after(() => rm(root, { recursive: true, force: true })); let directory = root; for (let depth = 0; depth < 129; depth += 1) { @@ -586,7 +716,7 @@ test('scratch traversal rejects trees beyond its recovery depth limit', async (t ); }); -test('does not replace scratch state while cleanup remains pending', async (t) => { +test('does not replace scratch state while cleanup remains pending', async t => { if (process.platform === 'win32') return; const workspace = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const retained = await mkdtemp(join(tmpdir(), 'librechat-code-retained-')); @@ -642,7 +772,7 @@ const windowsEnvironment = { }; for (const platform of ['darwin', 'linux', 'win32'] as const) { - test(`preserves required ${platform} environment names without allowing credentials`, async (t) => { + test(`preserves required ${platform} environment names without allowing credentials`, async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -658,23 +788,43 @@ for (const platform of ['darwin', 'linux', 'win32'] as const) { LD_PRELOAD: '/host/private.so', }; const sandbox = new NativeSrtWorkspaceCommandSandbox({ - workspaceRoot: root, platform, allowedDomains: ['github.com'], + workspaceRoot: root, + platform, + allowedDomains: ['github.com'], environment: { - ...proxyEnvironment, ...windowsEnvironment, ...credentials, + ...proxyEnvironment, + ...windowsEnvironment, + ...credentials, HtTp_PrOxY: 'http://mixed-case.invalid:8080', - PATH: '/usr/bin', LC_ALL: 'C.UTF-8', + PATH: '/usr/bin', + LC_ALL: 'C.UTF-8', }, manager: fake.manager, }); t.after(() => sandbox.close()); await sandbox.prepare(); - const denied = new Set(fake.config?.credentials?.envVars - ?.filter(({ mode }) => mode === 'deny').map(({ name }) => name)); - for (const name of [...Object.keys(proxyEnvironment), 'PATH', 'LC_ALL']) { - assert.equal(denied.has(name), false, `${name} must remain available`); + const denied = new Set( + fake.config?.credentials?.envVars + ?.filter(({ mode }) => mode === 'deny') + .map(({ name }) => name), + ); + for (const name of [ + ...Object.keys(proxyEnvironment), + 'PATH', + 'LC_ALL', + ]) { + assert.equal( + denied.has(name), + false, + `${name} must remain available`, + ); } for (const name of Object.keys(windowsEnvironment)) { - assert.equal(denied.has(name), platform !== 'win32', `${name} must be platform-specific`); + assert.equal( + denied.has(name), + platform !== 'win32', + `${name} must be platform-specific`, + ); } assert.equal(denied.has('HtTp_PrOxY'), platform !== 'win32'); for (const name of Object.keys(credentials)) { @@ -685,12 +835,14 @@ for (const platform of ['darwin', 'linux', 'win32'] as const) { }); } -test('uses SRT proxy values without restoring inherited proxies or credentials', async (t) => { +test('uses SRT proxy values without restoring inherited proxies or credentials', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const wrappedEnvironment = { - HTTP_PROXY: 'http://localhost:3128', HTTPS_PROXY: 'http://localhost:3128', - ALL_PROXY: 'http://localhost:3128', NO_PROXY: 'localhost', + HTTP_PROXY: 'http://localhost:3128', + HTTPS_PROXY: 'http://localhost:3128', + ALL_PROXY: 'http://localhost:3128', + NO_PROXY: 'localhost', }; const fake = fakeManager({ wrappedEnvironment }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -700,14 +852,19 @@ test('uses SRT proxy values without restoring inherited proxies or credentials', }); t.after(() => sandbox.close()); const result = await sandbox.execute({ - ...request, maxOutputBytes: 256, - command: 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', + ...request, + maxOutputBytes: 256, + command: + 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', }); assert.equal(result.exitCode, 0); - assert.equal(result.stdout, `${Object.values(wrappedEnvironment).join('|')}|unset`); + assert.equal( + result.stdout, + `${Object.values(wrappedEnvironment).join('|')}|unset`, + ); }); -test('masks a host credential for only its injection host and restores the parent environment', async (t) => { +test('masks a host credential for only its injection host and restores the parent environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -731,7 +888,8 @@ test('masks a host credential for only its injection host and restores the paren ], async resolve() { return { - LIBRECHAT_CODE_TEST_CREDENTIAL: 'Authorization: Bearer real-secret', + LIBRECHAT_CODE_TEST_CREDENTIAL: + 'Authorization: Bearer real-secret', }; }, }, @@ -759,7 +917,7 @@ test('masks a host credential for only its injection host and restores the paren }); }); -test('serializes credential handoff across concurrent sandbox instances', async (t) => { +test('serializes credential handoff across concurrent sandbox instances', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const original = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; @@ -770,11 +928,11 @@ test('serializes credential handoff across concurrent sandbox instances', async else process.env.LIBRECHAT_CODE_TEST_CREDENTIAL = original; }); let firstEntered!: () => void; - const firstEnteredPromise = new Promise((resolve) => { + const firstEnteredPromise = new Promise(resolve => { firstEntered = resolve; }); let releaseFirst!: () => void; - const firstGate = new Promise((resolve) => { + const firstGate = new Promise(resolve => { releaseFirst = resolve; }); let secondEntered = false; @@ -817,7 +975,7 @@ test('serializes credential handoff across concurrent sandbox instances', async const secondExecution = sandbox(second.manager, 'second-secret').execute( request, ); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(secondEntered, false); releaseFirst(); await firstExecution; @@ -828,7 +986,7 @@ test('serializes credential handoff across concurrent sandbox instances', async assert.equal(process.env.LIBRECHAT_CODE_TEST_CREDENTIAL, undefined); }); -test('isolates Git from host-level global and system configuration', async (t) => { +test('isolates Git from host-level global and system configuration', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -844,7 +1002,7 @@ test('isolates Git from host-level global and system configuration', async (t) = assert.equal(result.stdout, '/dev/null|1'); }); -test('restores trusted Git LFS filters without reading host Git configuration', async (t) => { +test('restores trusted Git LFS filters without reading host Git configuration', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ @@ -884,7 +1042,7 @@ test('restores trusted Git LFS filters without reading host Git configuration', assert.ok(!denied?.includes('GIT_CONFIG_VALUE_0')); }); -test('filters environment names case-insensitively only on Windows', async (t) => { +test('filters environment names case-insensitively only on Windows', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -923,7 +1081,7 @@ test('filters environment names case-insensitively only on Windows', async (t) = assert.ok(!denied?.includes('git_config_count')); }); -test('fails closed when the configured POSIX shell is unavailable', async (t) => { +test('fails closed when the configured POSIX shell is unavailable', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -942,7 +1100,7 @@ test('fails closed when the configured POSIX shell is unavailable', async (t) => ); }); -test('fails closed when SRT dependencies are unavailable', async (t) => { +test('fails closed when SRT dependencies are unavailable', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ dependencyErrors: ['bubblewrap missing'] }); @@ -960,7 +1118,7 @@ test('fails closed when SRT dependencies are unavailable', async (t) => { ); }); -test('refuses workspace roots that expose worker home or control files', async (t) => { +test('refuses workspace roots that expose worker home or control files', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const controlDirectory = join(root, '.control'); await mkdir(controlDirectory); @@ -985,7 +1143,7 @@ test('refuses workspace roots that expose worker home or control files', async ( ); }); -test('executes in the canonical workspace and bounds aggregate output', async (t) => { +test('executes in the canonical workspace and bounds aggregate output', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); await mkdir(join(root, 'src')); t.after(() => rm(root, { recursive: true, force: true })); @@ -1015,7 +1173,7 @@ test('executes in the canonical workspace and bounds aggregate output', async (t ); }); -test('rejects an escaping or unavailable command working directory', async (t) => { +test('rejects an escaping or unavailable command working directory', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1026,11 +1184,12 @@ test('rejects an escaping or unavailable command working directory', async (t) = await assert.rejects( sandbox.execute({ ...request, cwd: '..' }), (error: unknown) => - error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + error instanceof WorkspaceToolError && + error.code === 'INVALID_REQUEST', ); }); -test('terminates detached command descendants before returning', async (t) => { +test('terminates detached command descendants before returning', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1043,15 +1202,15 @@ test('terminates detached command descendants before returning', async (t) => { command: '(sleep 0.2; printf late > late.txt) >/dev/null 2>&1 &', }); assert.equal(result.exitCode, 0); - await new Promise((resolve) => setTimeout(resolve, 350)); + await new Promise(resolve => setTimeout(resolve, 350)); await assert.rejects(access(join(root, 'late.txt'))); }); -test('reports cancellation after command start as a potentially committed mutation', async (t) => { +test('reports cancellation after command start as a potentially committed mutation', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); let commandStarted!: () => void; - const commandStartedPromise = new Promise((resolve) => { + const commandStartedPromise = new Promise(resolve => { commandStarted = resolve; }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1081,7 +1240,7 @@ test('reports cancellation after command start as a potentially committed mutati ); }); -test('closes stdin immediately when the command protocol provides no input', async (t) => { +test('closes stdin immediately when the command protocol provides no input', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1098,7 +1257,7 @@ test('closes stdin immediately when the command protocol provides no input', asy assert.equal(result.timedOut, false); }); -test('maps platform-native exit statuses into the bridge protocol range', async (t) => { +test('maps platform-native exit statuses into the bridge protocol range', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const spawnCommand = () => { @@ -1123,7 +1282,7 @@ test('maps platform-native exit statuses into the bridge protocol range', async assert.equal(result.exitCode, 1); }); -test('cleans allocated command state exactly once on every execution exit', async (t) => { +test('cleans allocated command state exactly once on every execution exit', async t => { for (const outcome of [ 'abort-before-spawn', 'spawn-throw', @@ -1134,8 +1293,12 @@ test('cleans allocated command state exactly once on every execution exit', asyn 'wrap-throw', ] as const) { for (const cleanupThrows of [false, true]) { - await t.test(`${outcome}, cleanup throws: ${cleanupThrows}`, async (t) => { - const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + await t.test( + `${outcome}, cleanup throws: ${cleanupThrows}`, + async t => { + const root = await mkdtemp( + join(tmpdir(), 'librechat-code-native-'), + ); t.after(() => rm(root, { recursive: true, force: true })); const controller = new AbortController(); let cleanupCalls = 0; @@ -1143,9 +1306,11 @@ test('cleans allocated command state exactly once on every execution exit', asyn let allocated = false; const fake = fakeManager({ async beforeWrap() { - if (outcome === 'wrap-throw') throw new Error('wrap failed'); + if (outcome === 'wrap-throw') + throw new Error('wrap failed'); allocated = true; - if (outcome === 'abort-before-spawn') controller.abort(); + if (outcome === 'abort-before-spawn') + controller.abort(); }, }); fake.manager.cleanupAfterCommand = () => { @@ -1160,13 +1325,17 @@ test('cleans allocated command state exactly once on every execution exit', asyn spawnCommand() { spawnCalls += 1; assert.equal(allocated, true); - if (outcome === 'spawn-throw') throw new Error('spawn failed'); - const child = new EventEmitter() as ChildProcessWithoutNullStreams; + if (outcome === 'spawn-throw') + throw new Error('spawn failed'); + const child = + new EventEmitter() as ChildProcessWithoutNullStreams; let closeQueued = false; const close = () => { if (!closeQueued) { closeQueued = true; - queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + queueMicrotask(() => + child.emit('close', null, 'SIGKILL'), + ); } return true; }; @@ -1180,7 +1349,10 @@ test('cleans allocated command state exactly once on every execution exit', asyn queueMicrotask(() => { assert.equal(cleanupCalls, 0); if (outcome === 'error') { - child.emit('error', new Error('spawn failed')); + child.emit( + 'error', + new Error('spawn failed'), + ); } else if (outcome === 'abort-after-spawn') { controller.abort(); } else if (outcome === 'close') { @@ -1196,28 +1368,43 @@ test('cleans allocated command state exactly once on every execution exit', asyn ); if (outcome === 'close' || outcome === 'timeout') { const result = await execution; - assert.equal(result.exitCode, outcome === 'close' ? 0 : null); + assert.equal( + result.exitCode, + outcome === 'close' ? 0 : null, + ); assert.equal(result.timedOut, outcome === 'timeout'); } else { - await assert.rejects(execution, (error: unknown) => + await assert.rejects( + execution, + (error: unknown) => error instanceof WorkspaceToolError && - error.code === (outcome.startsWith('abort') + error.code === + (outcome.startsWith('abort') ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE') && - error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn') && + error.mutationMayHaveCommitted === + (outcome === 'abort-after-spawn') && error.requiresQuarantine === - (outcome === 'abort-after-spawn' && process.platform === 'win32'), + (outcome === 'abort-after-spawn' && + process.platform === 'win32'), ); } assert.equal( spawnCalls, - outcome === 'abort-before-spawn' || outcome === 'wrap-throw' ? 0 : 1, + outcome === 'abort-before-spawn' || + outcome === 'wrap-throw' + ? 0 + : 1, + ); + assert.equal( + cleanupCalls, + outcome === 'wrap-throw' ? 0 : 1, ); - assert.equal(cleanupCalls, outcome === 'wrap-throw' ? 0 : 1); assert.equal(allocated, false); await sandbox.close(); assert.equal(fake.reset, true); - }); + }, + ); } } }); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 171ea663..8d150d9f 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -11,14 +11,7 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { - access, - mkdtemp, - open, - realpath, - rm, - stat, -} from 'node:fs/promises'; +import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -141,8 +134,12 @@ interface NativeSandboxManager { cwd?: string, options?: { commandId?: string; commandText?: string }, ): Promise<{ argv: string[]; env: NodeJS.ProcessEnv }>; - annotateStderrWithSandboxFailures(commandId: string, stderr: string): string; + annotateStderrWithSandboxFailures( + commandId: string, + stderr: string, + ): string; cleanupAfterCommand(): void; + updateConfig?(config: SandboxRuntimeConfig): void; reset(): Promise; } @@ -224,13 +221,16 @@ function deniedEnvironmentNames( platform: NodeJS.Platform, ): string[] { return Object.keys(environment) - .filter((name) => { + .filter(name => { const normalized = platform === 'win32' ? name.toUpperCase() : name; return ( normalized.startsWith('LIBRECHAT_CODE_') || (!SAFE_CHILD_ENV_NAMES.has(normalized) && !PROXY_CHILD_ENV_NAMES.has(normalized) && - !(platform === 'win32' && WINDOWS_CHILD_ENV_NAMES.has(normalized)) && + !( + platform === 'win32' && + WINDOWS_CHILD_ENV_NAMES.has(normalized) + ) && !normalized.startsWith('LC_')) ); }) @@ -244,6 +244,36 @@ function normalizedEnvironmentName( return platform === 'win32' ? name.toUpperCase() : name; } +/** Distinguishes an unsupported host filesystem from an implementation fault. */ +export class CopyOnWriteCloneUnavailableError extends WorkspaceToolError { + constructor() { + super( + 'Selected-workspace PTC requires copy-on-write filesystem cloning', + 'COMMAND_UNAVAILABLE', + ); + this.name = 'CopyOnWriteCloneUnavailableError'; + } +} + +function isCopyOnWriteUnsupported( + error: unknown, + platform: NodeJS.Platform, +): boolean { + if (platform === 'win32') return true; + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error.code === 'ENOTSUP' || error.code === 'EOPNOTSUPP') + ) { + return true; + } + return ( + error instanceof Error && + error.message.toLowerCase().includes('operation not supported') + ); +} + export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private readonly manager: NativeSandboxManager; @@ -252,6 +282,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + private runtimeConfig?: SandboxRuntimeConfig; + private denyReadPaths: string[] = []; + private denyWritePaths: string[] = []; private scratchDirectory?: string; private scratchHandle?: FileHandle; private execution?: Promise; @@ -288,7 +321,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } managerOwners.set(this.manager, this); - this.initialized = this.initializeOnce().catch(async (error) => { + this.initialized = this.initializeOnce().catch(async error => { await this.manager.reset().catch(() => { this.resetFailed = true; }); @@ -314,7 +347,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'COMMAND_UNAVAILABLE', ); } - const home = await canonicalPath(this.options.homeDirectory ?? homedir()); + const home = await canonicalPath( + this.options.homeDirectory ?? homedir(), + ); if (isWithin(root, home)) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain the worker home directory', @@ -324,7 +359,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const protectedPaths = await Promise.all( (this.options.protectedPaths ?? []).map(canonicalPath), ); - if (protectedPaths.some((path) => isWithin(root, path))) { + if (protectedPaths.some(path => isWithin(root, path))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain worker control files', 'REGISTRATION_INVALID', @@ -338,13 +373,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const inheritedWritablePaths = [ ...sharedScratchPaths, ...(await Promise.all( - [join(home, '.npm', '_logs'), join(home, '.claude', 'debug')].map( - canonicalPath, - ), + [ + join(home, '.npm', '_logs'), + join(home, '.claude', 'debug'), + ].map(canonicalPath), )), ]; - const deniedInheritedWritablePaths = [...new Set(inheritedWritablePaths)]; - if (deniedInheritedWritablePaths.some((path) => isWithin(path, root))) { + const deniedInheritedWritablePaths = [ + ...new Set(inheritedWritablePaths), + ]; + if (deniedInheritedWritablePaths.some(path => isWithin(path, root))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot be inside an inherited writable path', 'REGISTRATION_INVALID', @@ -359,7 +397,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } if (this.platform !== 'win32') { try { - await access(this.options.shellPath ?? '/bin/bash', fsConstants.X_OK); + await access( + this.options.shellPath ?? '/bin/bash', + fsConstants.X_OK, + ); } catch { throw new WorkspaceToolError( `Native sandbox shell is unavailable: ${this.options.shellPath ?? '/bin/bash'}`, @@ -383,35 +424,40 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); const unrestrictedNetwork = commandPolicy.network.outbound === 'unrestricted'; - const config: SandboxRuntimeConfig = { - network: { + const network: SandboxRuntimeConfig['network'] = { allowedDomains: [...(this.options.allowedDomains ?? [])], deniedDomains: [], strictAllowlist: !unrestrictedNetwork, allowAllUnixSockets: commandPolicy.network.allowAllUnixSockets, allowLocalBinding: commandPolicy.network.allowLocalBinding, ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), - }, + }; + const config: SandboxRuntimeConfig = { + network, filesystem: { denyRead: [ home, - ...sharedScratchPaths.filter((path) => + ...sharedScratchPaths.filter(path => deniedInheritedWritablePaths.includes(path), ), ], allowRead: [ root, - ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ...(canonicalScratchDirectory + ? [canonicalScratchDirectory] + : []), ], allowWrite: [ root, - ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ...(canonicalScratchDirectory + ? [canonicalScratchDirectory] + : []), ], denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], allowGitConfig: false, }, credentials: { - files: protectedPaths.map((path) => ({ + files: protectedPaths.map(path => ({ path, mode: 'deny' as const, })), @@ -424,15 +470,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox }, this.platform, ) - .filter((name) => { + .filter(name => { const normalized = normalizedEnvironmentName( name, this.platform, ); return ( - !Object.hasOwn(TRUSTED_GIT_ENVIRONMENT, normalized) && + !Object.hasOwn( + TRUSTED_GIT_ENVIRONMENT, + normalized, + ) && !this.options.maskedEnvironment?.variables.some( - (variable) => + variable => normalizedEnvironmentName( variable.name, this.platform, @@ -440,12 +489,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ) ); }) - .map((name) => ({ name, mode: 'deny' as const })), - ...(this.options.maskedEnvironment?.variables.map((variable) => ({ + .map(name => ({ name, mode: 'deny' as const })), + ...(this.options.maskedEnvironment?.variables.map( + variable => ({ ...variable, mode: 'mask' as const, - ...(variable.extract ? { onExtractNoMatch: 'error' as const } : {}), - })) ?? []), + ...(variable.extract + ? { onExtractNoMatch: 'error' as const } + : {}), + }), + ) ?? []), ], }, allowAppleEvents: false, @@ -458,6 +511,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox unrestrictedNetwork ? async () => true : undefined, ); this.canonicalRoot = root; + this.runtimeConfig = config; + this.denyReadPaths = [ + home, + ...sharedScratchPaths.filter(path => + deniedInheritedWritablePaths.includes(path), + ), + ]; + this.denyWritePaths = [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ]; } async execute( @@ -479,9 +543,248 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } + /** + * Allocate an owner-only execution directory that is already inside this + * sandbox's allowlist. The caller must remove the returned directory after + * the execution settles. It is intentionally unavailable on native Windows + * until the restricted-account TEMP directory can be opened and verified by + * the trusted parent process. + */ + async createExecutionDirectory(): Promise { + await this.initialize(); + if (!this.scratchDirectory || this.platform === 'win32') { + throw new WorkspaceToolError( + 'Native programmatic execution storage is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + return await mkdtemp(join(this.scratchDirectory, 'execution-')); + } + + /** + * Clone the current workspace into private scratch for a side-effect- + * equivalent replay probe. Platform clone flags are intentionally strict: + * silently falling back to a byte copy would make every tool-bearing run + * consume time and disk proportional to the repository size. + */ + async createProgrammaticProbeWorkspace( + executionDirectory: string, + signal?: AbortSignal, + ): Promise { + await this.initialize(); + const scratchDirectory = this.scratchDirectory; + const root = this.canonicalRoot; + let parent: string; + try { + parent = await realpath(executionDirectory); + if ( + !scratchDirectory || + !root || + !isWithin(scratchDirectory, parent) || + !(await stat(parent)).isDirectory() + ) { + throw new Error('invalid execution directory'); + } + } catch { + throw new WorkspaceToolError( + 'Programmatic execution directory is unavailable', + 'INVALID_PATH', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + const destination = join(parent, 'workspace'); + try { + if (this.platform === 'win32') { + throw new Error('copy-on-write cloning is unavailable on Windows'); + } + const args = + this.platform === 'darwin' + ? ['-cR', root, destination] + : ['--archive', '--reflink=always', root, destination]; + await new Promise((resolveCopy, rejectCopy) => { + const child = this.spawnCommand('/bin/cp', args, { + env: { + PATH: this.environment.PATH, + LANG: this.environment.LANG, + LC_ALL: this.environment.LC_ALL, + }, + signal, + }); + let stderr = Buffer.alloc(0); + child.stderr.on('data', (chunk: Buffer) => { + if (stderr.byteLength < 4_096) { + stderr = Buffer.concat([stderr, chunk]).subarray(0, 4_096); + } + }); + child.once('error', rejectCopy); + child.once('close', code => { + if (code === 0) resolveCopy(); + else { + rejectCopy( + new Error( + `copy-on-write clone failed (${code ?? 'signal'}): ${boundedUtf8(stderr, 4_096)}`, + ), + ); + } + }); + }); + return await realpath(destination); + } catch (error) { + await rm(destination, { recursive: true, force: true }).catch( + () => undefined, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (isCopyOnWriteUnsupported(error, this.platform)) { + throw new CopyOnWriteCloneUnavailableError(); + } + throw new WorkspaceToolError( + 'Copy-on-write workspace clone failed unexpectedly', + 'COMMAND_UNAVAILABLE', + ); + } + } + + /** Run a generated program from a verified private execution directory. */ + async executeProgrammatic( + request: WorkspaceExecuteCommandRequest, + dataDirectory: string, + signal?: AbortSignal, + options?: { probe?: boolean; workspaceRoot?: string }, + ): Promise { + if (this.execution || this.closing) { + throw new WorkspaceToolError( + 'Native sandbox already has an active command or is closing', + 'COMMAND_UNAVAILABLE', + ); + } + await this.initialize(); + const scratchDirectory = this.scratchDirectory; + let canonicalDataDirectory: string; + let canonicalWorkspaceRoot: string | undefined; + try { + canonicalDataDirectory = await realpath(dataDirectory); + canonicalWorkspaceRoot = options?.workspaceRoot + ? await realpath(options.workspaceRoot) + : undefined; + if ( + !scratchDirectory || + !isWithin(scratchDirectory, canonicalDataDirectory) || + !(await stat(canonicalDataDirectory)).isDirectory() || + (canonicalWorkspaceRoot != null && + (!isWithin(scratchDirectory, canonicalWorkspaceRoot) || + !(await stat(canonicalWorkspaceRoot)).isDirectory())) + ) { + throw new Error('invalid execution directory'); + } + } catch { + throw new WorkspaceToolError( + 'Programmatic execution directory is unavailable', + 'INVALID_PATH', + ); + } + const execute = () => this.executeExclusive( + request, + signal, + { + LIBRECHAT_CODE_DATA_DIR: canonicalDataDirectory, + LIBRECHAT_CODE_CONTROL_PATH: join( + canonicalDataDirectory, + '_ptc_pending_result.json', + ), + LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', + PTC_HISTORY_PATH: join( + canonicalDataDirectory, + '_ptc_history.json', + ), + TMPDIR: canonicalDataDirectory, + }, + options?.probe + ? { + filesystem: { + allowRead: [ + canonicalWorkspaceRoot ?? this.canonicalRoot!, + canonicalDataDirectory, + ], + allowWrite: [ + ...(canonicalWorkspaceRoot != null + ? [canonicalWorkspaceRoot] + : []), + canonicalDataDirectory, + ], + denyRead: this.denyReadPaths, + denyWrite: [ + this.canonicalRoot!, + ...this.denyWritePaths, + ], + }, + network: { + // A probe is speculative, even on a trusted VM. + // Copy-on-write protects files, not remote mutations. + allowedDomains: [], + deniedDomains: [], + strictAllowlist: true, + allowUnixSockets: [], + allowAllUnixSockets: false, + allowLocalBinding: false, + }, + } + : undefined, + canonicalDataDirectory, + canonicalWorkspaceRoot, + ); + const execution = options?.probe ? this.withProbeNetwork(execute) : execute(); + this.execution = execution; + try { + return await execution; + } finally { + this.execution = undefined; + } + } + + private async withProbeNetwork(execute: () => Promise): Promise { + const config = this.runtimeConfig; + if (!config || !this.manager.updateConfig) { + throw new WorkspaceToolError('Native probe network isolation is unavailable', 'COMMAND_UNAVAILABLE'); + } + // SRT's proxies and Unix/local socket rules read session configuration, + // not wrapWithSandboxArgv's per-command override. + this.manager.updateConfig({ ...config, network: { + allowedDomains: [], deniedDomains: [], strictAllowlist: true, + allowUnixSockets: [], allowAllUnixSockets: false, allowLocalBinding: false, + } }); + try { + return await execute(); + } finally { + try { + // Revoke the probe's proxy endpoints and credentials before restoring + // network access. A lingering probe must never inherit the commit's + // permissive proxy session through a live updateConfig. + await this.manager.reset(); + await this.manager.initialize(config, config.network.strictAllowlist ? undefined : async () => true); + } catch { + this.resetFailed = true; + throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE'); + } + } + } + private async executeExclusive( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, + customConfig?: Partial, + sandboxScratchDirectory?: string, + workspaceRoot?: string, ): Promise { if ( !isWorkspaceToolRequest(request) || @@ -499,7 +802,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } await this.initialize(); - const root = this.canonicalRoot!; + const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { cwd = await realpath(resolve(root, request.cwd ?? '.')); @@ -528,7 +831,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { ...TRUSTED_GIT_ENVIRONMENT, ...(credentialEnvironment ?? {}), - ...this.scratchSelectorEnvironment(), + ...this.scratchSelectorEnvironment(sandboxScratchDirectory), }, () => this.manager.wrapWithSandboxArgv( @@ -536,7 +839,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'win32' ? undefined : (this.options.shellPath ?? '/bin/bash'), - undefined, + customConfig, signal, cwd, { commandId, commandText: request.command }, @@ -561,7 +864,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'EXECUTION_ABORTED', ); } - return await this.runWrapped(request, wrapped, cwd, commandId, signal); + return await this.runWrapped( + request, + wrapped, + cwd, + commandId, + signal, + trustedEnvironment, + ); } finally { // A successful wrap owns command state even when no child is spawned. try { @@ -578,7 +888,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ): Promise { const previousMutation = hostEnvironmentMutationQueue; let releaseMutation!: () => void; - hostEnvironmentMutationQueue = new Promise((resolve) => { + hostEnvironmentMutationQueue = new Promise(resolve => { releaseMutation = resolve; }); await previousMutation; @@ -604,31 +914,41 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd: string, commandId: string, signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, ): Promise { const outputLimit = - request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + request.maxOutputBytes ?? + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; const timeoutMs = request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS; return await new Promise( (resolvePromise, reject) => { let child: ChildProcessWithoutNullStreams; try { - child = this.spawnCommand(wrapped.argv[0], wrapped.argv.slice(1), { + child = this.spawnCommand( + wrapped.argv[0], + wrapped.argv.slice(1), + { cwd, env: { ...wrapped.env, ...this.scratchEnvironment(), + ...trustedEnvironment, ...TRUSTED_GIT_CONFIG_ENTRIES, GIT_CONFIG_COUNT: - wrapped.env.GIT_CONFIG_COUNT ?? TRUSTED_GIT_CONFIG_COUNT, + wrapped.env.GIT_CONFIG_COUNT ?? + TRUSTED_GIT_CONFIG_COUNT, GIT_CONFIG_GLOBAL: - this.platform === 'win32' ? 'NUL' : '/dev/null', + this.platform === 'win32' + ? 'NUL' + : '/dev/null', GIT_CONFIG_NOSYSTEM: '1', }, detached: this.platform !== 'win32', shell: false, windowsHide: true, - }); + }, + ); child.stdin.end(); } catch { reject( @@ -654,10 +974,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const accepted = chunk.subarray(0, remaining); target.push(accepted); outputBytes += accepted.byteLength; - if (accepted.byteLength !== chunk.byteLength) truncated = true; + if (accepted.byteLength !== chunk.byteLength) + truncated = true; }; - child.stdout.on('data', (chunk: Buffer) => append(stdout, chunk)); - child.stderr.on('data', (chunk: Buffer) => append(stderr, chunk)); + child.stdout.on('data', (chunk: Buffer) => + append(stdout, chunk), + ); + child.stderr.on('data', (chunk: Buffer) => + append(stderr, chunk), + ); const abort = (): void => { if (settled) return; this.killCommandTree(child); @@ -706,7 +1031,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); return; } - const stdoutValue = boundedUtf8(Buffer.concat(stdout), outputLimit); + const stdoutValue = boundedUtf8( + Buffer.concat(stdout), + outputLimit, + ); const stderrBudget = Math.max( 0, outputLimit - Buffer.byteLength(stdoutValue), @@ -714,7 +1042,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const rawStderr = Buffer.concat(stderr).toString('utf8'); let annotatedStderr = rawStderr; try { - annotatedStderr = this.manager.annotateStderrWithSandboxFailures( + annotatedStderr = + this.manager.annotateStderrWithSandboxFailures( commandId, rawStderr, ); @@ -730,12 +1059,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox operation: 'execute_command', workspaceId: request.workspaceId, exitCode: - timedOut || childSignal ? null : this.protocolExitCode(code), + timedOut || childSignal + ? null + : this.protocolExitCode(code), ...(childSignal ? { signal: childSignal } : {}), stdout: stdoutValue, stderr: stderrValue, truncated: - truncated || Buffer.byteLength(annotatedStderr) > stderrBudget, + truncated || + Buffer.byteLength(annotatedStderr) > stderrBudget, timedOut, }); }); @@ -775,7 +1107,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const canonicalTemporaryRoot = await canonicalPath(HOST_TEMPORARY_ROOT); - const sharedScratchRoot = sharedScratchPaths.find((path) => + const sharedScratchRoot = sharedScratchPaths.find(path => isWithin(path, canonicalTemporaryRoot), ); const scratchDirectory = await mkdtemp( @@ -798,7 +1130,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox true, ); if (((await scratchHandle.stat()).mode & 0o777) !== 0o700) { - throw new Error('Native sandbox scratch directory is not private'); + throw new Error( + 'Native sandbox scratch directory is not private', + ); } this.scratchHandle = scratchHandle; } catch (error) { @@ -829,11 +1163,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : { TMPDIR: scratchDirectory }; } - private scratchSelectorEnvironment(): NodeJS.ProcessEnv { - const scratchDirectory = this.scratchDirectory; + private scratchSelectorEnvironment( + selectedDirectory = this.scratchDirectory, + ): NodeJS.ProcessEnv { + const scratchDirectory = selectedDirectory; if (!scratchDirectory) return {}; return Object.fromEntries( - SRT_SCRATCH_SELECTOR_NAMES.map((name) => [name, scratchDirectory]), + SRT_SCRATCH_SELECTOR_NAMES.map(name => [name, scratchDirectory]), ); } diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index fd426783..08cba1c7 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -1,8 +1,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + bridgeArtifactMediaType, bridgeWorkerPath, comparePortableRelativePaths, + isBridgeWorkspaceProgrammaticRequest, + isSupportedBridgeArtifactName, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolRequest, @@ -13,6 +16,23 @@ import type { WorkspacePreviewEditRequest, } from './protocol.js'; +test('accepts gateway directory markers as artifacts', () => { + assert.equal(isSupportedBridgeArtifactName('.dirkeep'), true); + assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep'), true); + assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep.exe'), false); +}); + +test('rejects caller-supplied programmatic control payloads', () => { + for (const name of ['_ptc_pending_result.json', '_PTC_PENDING_RESULT.JSON', 'nested/_ptc_pending_result.json']) { + assert.equal(isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { language: 'bash', version: '5.2.0', session_id: 'session', files: [ + { name: 'main.sh', content: 'true' }, { name, content: '{}' }, + ] }, + }), false); + } +}); + const validSingleEditRequest: WorkspaceEditFileRequest = { protocolVersion: 1, operation: 'edit_file', @@ -592,3 +612,167 @@ test('workspace capabilities allow per-workspace operation restrictions', () => false, ); }); + +test('workspace programmatic capability is closed to Bash command roots', () => { + const workspaceTools = { + protocolVersion: 1, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'project-a' }], + }; + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { ...workspaceTools, operations: ['read_file'] }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { ...workspaceTools, programmaticLanguages: ['python'] }, + }), + false, + ); +}); + +test('workspace programmatic requests accept only stable input cache identities', () => { + const request = { + headers: {}, + body: { + language: 'bash', + version: '5.2', + execution_id: 'execution_1', + replay_tool_count: 2, + max_output_files: 50, + max_output_file_bytes: 10_000_000, + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { + name: 'skills/example.txt', + id: 'file-1', + storage_session_id: 'storage-1', + input_cache_key: 'a'.repeat(64), + }, + ], + }, + }; + assert.equal(isBridgeWorkspaceProgrammaticRequest(request), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + ...request, + body: { + ...request.body, + files: [request.body.files[0], { ...request.body.files[1], input_cache_key: '../cache' }], + }, + }), + false, + ); + for (const body of [ + { ...request.body, execution_id: '../execution' }, + { ...request.body, replay_tool_count: -1 }, + { ...request.body, replay_tool_count: 257 }, + { ...request.body, max_output_files: -1 }, + { ...request.body, max_output_files: 101 }, + { ...request.body, max_output_file_bytes: 0 }, + { ...request.body, max_output_file_bytes: 10 * 1024 * 1024 + 1 }, + ]) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ ...request, body }), + false, + ); + } +}); + +test('workspace programmatic history can use the bounded replay aggregate budget', () => { + const history = 'h'.repeat(10 * 1024 * 1024 + 1); + const body = { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { name: '_ptc_history.json', content: history }, + ], + }; + assert.equal(isBridgeWorkspaceProgrammaticRequest({ headers: {}, body }), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + ...body, + files: [ + { name: 'main.sh', content: history }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }), + false, + ); +}); + +test('workspace programmatic requests reject non-canonical file paths', () => { + for (const name of ['./main.sh', 'scripts//main.sh', 'scripts/./main.sh', '.']) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { name, content: 'data' }, + ], + }, + }), + false, + name, + ); + } +}); + +test('workspace programmatic requests reject ancestor-descendant input conflicts', () => { + for (const names of [ + ['main.sh', 'main.sh/data.txt'], + ['main.sh', 'assets', 'assets/logo.png'], + ['main.sh', 'deep/path/file.txt', 'deep'], + ]) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: names.map(name => ({ name, content: 'data' })), + }, + }), + false, + names.join(', '), + ); + } +}); + +test('bridge artifact policy and media types match the hardened gateway contract', () => { + assert.equal(isSupportedBridgeArtifactName('reports/result.json'), true); + assert.equal(isSupportedBridgeArtifactName('preview.png'), true); + assert.equal(isSupportedBridgeArtifactName('model.bin'), false); + assert.equal(bridgeArtifactMediaType('preview.png'), 'image/png'); + assert.equal(bridgeArtifactMediaType('reports/result.json'), 'application/json'); + assert.equal(bridgeArtifactMediaType('Dockerfile'), 'application/octet-stream'); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c1dad949..b92d21ac 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -20,9 +20,144 @@ export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES = 100; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY = 4; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; + +/** Reserve a bounded share for all input/output batches, not per-file grants. */ +export function programmaticTransferReserveMs(jobTimeoutMs: number): number { + return Math.max(1, Math.floor(jobTimeoutMs / 3)); +} +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES = 10 * 1024 * 1024; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES = 40_000_000; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES = 100 * 1024 * 1024; /** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; +/** + * Artifact names accepted by the hardened egress gateway. Keep this policy in + * the bridge protocol package so a remote worker can reject unsupported output + * locally instead of discovering the mismatch only after mutating a workspace. + */ +const BRIDGE_ARTIFACT_EXTENSIONS = new Set([ + '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', + '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', + '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', + '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', + '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', + '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', + '.xml', '.yaml', '.yml', + '.ics', '.ical', '.ifb', '.icalendar', + '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', + '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', + '.odt', '.ods', '.odp', '.rtf', + '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', + '.tif', '.tiff', '.webp', + '.eot', '.ttf', '.woff', '.woff2', + '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', + '.tf', '.tfvars', '.tfstate', '.hcl', + '.dockerfile', '.Dockerfile', '.dockerignore', + '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', + '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', + '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', +]); + +function portableBasename(name: string): string { + return name.slice(name.lastIndexOf('/') + 1); +} + +/** Apply the gateway's extension allowlist without importing service code. */ +export function isSupportedBridgeArtifactName(name: string): boolean { + const basename = portableBasename(name); + if (basename === '.dirkeep') return true; + const dot = basename.lastIndexOf('.'); + const extension = dot > 0 ? basename.slice(dot).toLowerCase() : ''; + const dottedBasename = `.${basename}`; + return ( + (extension !== '' && BRIDGE_ARTIFACT_EXTENSIONS.has(extension)) || + BRIDGE_ARTIFACT_EXTENSIONS.has(basename) || + BRIDGE_ARTIFACT_EXTENSIONS.has(basename.toLowerCase()) || + (extension === '' && + (BRIDGE_ARTIFACT_EXTENSIONS.has(dottedBasename) || + BRIDGE_ARTIFACT_EXTENSIONS.has(dottedBasename.toLowerCase()))) + ); +} + +const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { + '.avif': 'image/avif', + '.bmp': 'image/bmp', + '.bz2': 'application/x-bzip2', + '.c': 'text/x-c', + '.conf': 'text/plain', + '.cpp': 'text/x-c++src', + '.css': 'text/css', + '.csv': 'text/csv', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.gif': 'image/gif', + '.gz': 'application/gzip', + '.gzip': 'application/gzip', + '.htm': 'text/html', + '.html': 'text/html', + '.ico': 'image/x-icon', + '.ics': 'text/calendar', + '.ifb': 'text/calendar', + '.ical': 'text/calendar', + '.icalendar': 'text/calendar', + '.ini': 'text/plain', + '.java': 'text/x-java-source', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.js': 'text/javascript', + '.json': 'application/json', + '.json5': 'application/json5', + '.jsonl': 'application/x-ndjson', + '.jsx': 'text/jsx', + '.log': 'text/plain', + '.md': 'text/markdown', + '.odt': 'application/vnd.oasis.opendocument.text', + '.ods': 'application/vnd.oasis.opendocument.spreadsheet', + '.odp': 'application/vnd.oasis.opendocument.presentation', + '.parquet': 'application/vnd.apache.parquet', + '.pdf': 'application/pdf', + '.png': 'image/png', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.py': 'text/x-python', + '.rst': 'text/x-rst', + '.rtf': 'application/rtf', + '.sh': 'application/x-sh', + '.sql': 'application/sql', + '.svg': 'image/svg+xml', + '.tar': 'application/x-tar', + '.tex': 'application/x-tex', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.toml': 'application/toml', + '.ts': 'text/typescript', + '.tsx': 'text/tsx', + '.tsv': 'text/tab-separated-values', + '.txt': 'text/plain', + '.webp': 'image/webp', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xml': 'application/xml', + '.yaml': 'application/yaml', + '.yml': 'application/yaml', + '.zip': 'application/zip', +}; + +/** Infer a safe response media type from an already-validated artifact name. */ +export function bridgeArtifactMediaType(name: string): string { + const basename = portableBasename(name).toLowerCase(); + const dot = basename.lastIndexOf('.'); + const extension = dot > 0 ? basename.slice(dot) : basename; + return BRIDGE_ARTIFACT_MEDIA_TYPES[extension] ?? 'application/octet-stream'; +} + export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; export type BridgeWorkspaceToolOperation = @@ -38,6 +173,7 @@ export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; export type WorkspaceEditFileFeature = 'expected_base_sha256'; export type WorkspaceListFileFeature = 'after_path'; +export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { id: string; @@ -58,6 +194,8 @@ export interface BridgeWorkspaceToolCapabilities { editFileFeatures?: WorkspaceEditFileFeature[]; /** Omitted by workers that cannot continue a bounded file listing. */ listFileFeatures?: WorkspaceListFileFeature[]; + /** Languages that can execute PTC replay inside a selected workspace. */ + programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } export interface WorkspaceReadFileRequest { @@ -153,8 +291,7 @@ interface WorkspaceEditFileRequestBase { expectedBaseSha256?: string; } -export interface WorkspaceSingleEditFileRequest - extends WorkspaceEditFileRequestBase { +export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequestBase { /** Legacy single-edit form. */ oldText: string; /** Legacy single-edit form. */ @@ -162,8 +299,7 @@ export interface WorkspaceSingleEditFileRequest edits?: never; } -export interface WorkspaceBatchEditFileRequest - extends WorkspaceEditFileRequestBase { +export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestBase { /** Ordered exact replacements applied atomically as one file mutation. */ edits: WorkspaceTextEdit[]; oldText?: never; @@ -171,8 +307,7 @@ export interface WorkspaceBatchEditFileRequest } export type WorkspaceEditFileRequest = - | WorkspaceSingleEditFileRequest - | WorkspaceBatchEditFileRequest; + WorkspaceSingleEditFileRequest | WorkspaceBatchEditFileRequest; export interface WorkspaceTextEdit { oldText: string; @@ -195,23 +330,20 @@ interface WorkspacePreviewEditRequestBase { path: string; } -export interface WorkspaceSinglePreviewEditRequest - extends WorkspacePreviewEditRequestBase { +export interface WorkspaceSinglePreviewEditRequest extends WorkspacePreviewEditRequestBase { oldText: string; newText: string; edits?: never; } -export interface WorkspaceBatchPreviewEditRequest - extends WorkspacePreviewEditRequestBase { +export interface WorkspaceBatchPreviewEditRequest extends WorkspacePreviewEditRequestBase { edits: WorkspaceTextEdit[]; oldText?: never; newText?: never; } export type WorkspacePreviewEditRequest = - | WorkspaceSinglePreviewEditRequest - | WorkspaceBatchPreviewEditRequest; + WorkspaceSinglePreviewEditRequest | WorkspaceBatchPreviewEditRequest; export interface WorkspacePreviewEditResult { protocolVersion: BridgeProtocolVersion; @@ -392,12 +524,7 @@ const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'truncated', 'timedOut', ]); -const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ - 'path', - 'line', - 'column', - 'text', -]); +const WORKSPACE_SEARCH_MATCH_KEYS = new Set(['path', 'line', 'column', 'text']); export interface BridgeWorkerCapabilities { /** Opt-in protocol: maximum concurrently leased independent workspace roots. */ @@ -437,6 +564,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceEditFileFeatures?: WorkspaceEditFileFeature[]; /** Listing features this Code API can safely route to a capability-aware worker. */ supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; + /** PTC languages this Code API can safely route into a selected workspace. */ + supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -469,6 +598,37 @@ export interface BridgeSandboxRequest { headers: Record; } +export type BridgeProgrammaticPayloadFile = + | { name: string; content: string } + | { + name: string; + id: string; + storage_session_id: string; + input_cache_key?: string; + }; + +export interface BridgeWorkspaceProgrammaticBody { + language: 'bash'; + version: string; + /** Stable identity shared by every replay iteration of one execution. */ + execution_id?: string; + /** Declared replay tools; zero allows the worker to skip the probe pass. */ + replay_tool_count?: number; + run_timeout?: number; + transfer_timeout_ms?: number; + /** Manifest-bound upload ceiling negotiated by Code API. */ + max_output_files?: number; + /** Effective per-file upload ceiling negotiated by Code API. */ + max_output_file_bytes?: number; + files: BridgeProgrammaticPayloadFile[]; + session_id: string; + output_session_id?: string; + egress_grant?: string; +} + +export type BridgeWorkspaceProgrammaticRequest = + BridgeSandboxRequest; + export interface BridgeAssignment { workspaceLeaseSlot?: number; protocolVersion: BridgeProtocolVersion; @@ -481,7 +641,9 @@ export interface BridgeAssignment { /** Server-calculated execution budget at lease time; avoids VM clock skew. */ remainingMs?: number; runtimeSessionId?: string; - executionKind?: 'sandbox' | 'workspace_tool'; + executionKind?: 'sandbox' | 'workspace_tool' | 'workspace_programmatic'; + /** Selected workspace for workspace-scoped programmatic execution. */ + workspaceId?: string; request: BridgeSandboxRequest | WorkspaceToolRequest; } @@ -589,6 +751,140 @@ export function isValidBridgeWorkerId(workerId: string): boolean { return BRIDGE_WORKER_ID_PATTERN.test(workerId); } +export function isBridgeWorkspaceProgrammaticRequest( + value: unknown, +): value is BridgeWorkspaceProgrammaticRequest { + if (typeof value !== 'object' || value === null) return false; + const request = value as Record; + if ( + typeof request.headers !== 'object' || + request.headers === null || + !Object.values(request.headers).every( + entry => typeof entry === 'string', + ) || + typeof request.body !== 'object' || + request.body === null + ) { + return false; + } + const body = request.body as Record; + if ( + body.language !== 'bash' || + typeof body.version !== 'string' || + body.version.length === 0 || + body.version.length > BRIDGE_RUNTIME_MAX_LENGTH || + (body.execution_id !== undefined && + (typeof body.execution_id !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || + (body.replay_tool_count !== undefined && + (!Number.isSafeInteger(body.replay_tool_count) || + Number(body.replay_tool_count) < 0 || + Number(body.replay_tool_count) > 256)) || + (body.max_output_files !== undefined && + (!Number.isSafeInteger(body.max_output_files) || + Number(body.max_output_files) < 0 || + Number(body.max_output_files) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES)) || + (body.max_output_file_bytes !== undefined && + (!Number.isSafeInteger(body.max_output_file_bytes) || + Number(body.max_output_file_bytes) < 1 || + Number(body.max_output_file_bytes) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES)) || + typeof body.session_id !== 'string' || + body.session_id.length === 0 || + body.session_id.length > 32_768 || + /[\0\r\n]/.test(body.session_id) || + (body.output_session_id !== undefined && + (typeof body.output_session_id !== 'string' || + body.output_session_id.length === 0 || + body.output_session_id.length > 32_768 || + /[\0\r\n]/.test(body.output_session_id))) || + (body.egress_grant !== undefined && + (typeof body.egress_grant !== 'string' || + body.egress_grant.length === 0 || + body.egress_grant.length > 256 * 1024)) || + (body.transfer_timeout_ms !== undefined && + (!Number.isSafeInteger(body.transfer_timeout_ms) || + Number(body.transfer_timeout_ms) < 1 || + Number(body.transfer_timeout_ms) > BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || + (body.run_timeout !== undefined && + (!Number.isSafeInteger(body.run_timeout) || + Number(body.run_timeout) < 1 || + Number(body.run_timeout) > + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS)) || + !Array.isArray(body.files) || + body.files.length < 1 || + body.files.length > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES + ) { + return false; + } + let inlineBytes = 0; + const names = new Set(); + for (const rawFile of body.files) { + if (typeof rawFile !== 'object' || rawFile === null) return false; + const file = rawFile as Record; + if ( + !isSafePortableRelativePath(file.name) || + file.name === '.' || + portableBasename(file.name).toLowerCase() === '_ptc_pending_result.json' || + normalizePortableRelativePath(file.name) !== file.name || + names.has(file.name) + ) { + return false; + } + names.add(file.name); + if (typeof file.content === 'string') { + inlineBytes += Buffer.byteLength(file.content); + if ( + Object.keys(file).some( + key => key !== 'name' && key !== 'content', + ) || + Buffer.byteLength(file.content) > + (file.name === '_ptc_history.json' + ? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES + : BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) + ) { + return false; + } + continue; + } + if ( + typeof file.id !== 'string' || + file.id.length === 0 || + file.id.length > 32_768 || + /[\0\r\n]/.test(file.id) || + typeof file.storage_session_id !== 'string' || + file.storage_session_id.length === 0 || + file.storage_session_id.length > 32_768 || + /[\0\r\n]/.test(file.storage_session_id) || + Object.keys(file).some( + key => + key !== 'name' && + key !== 'id' && + key !== 'storage_session_id' && + key !== 'input_cache_key', + ) || + (file.input_cache_key !== undefined && + (typeof file.input_cache_key !== 'string' || + !/^[a-f0-9]{64}$/.test(file.input_cache_key))) + ) { + return false; + } + } + for (const name of names) { + const segments = name.split('/'); + let ancestor = ''; + for (let index = 0; index < segments.length - 1; index += 1) { + ancestor = ancestor ? `${ancestor}/${segments[index]}` : segments[index]!; + if (names.has(ancestor)) return false; + } + } + return ( + names.has('main.sh') && + inlineBytes <= BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ); +} + export function isSafePortableRelativePath(value: unknown): value is string { if ( typeof value !== 'string' || @@ -602,20 +898,23 @@ export function isSafePortableRelativePath(value: unknown): value is string { ) { return false; } - return value.split('/').every((segment) => segment !== '..'); + return value.split('/').every(segment => segment !== '..'); } function normalizePortableRelativePath(value: string): string { return ( value .split('/') - .filter((segment) => segment.length > 0 && segment !== '.') + .filter(segment => segment.length > 0 && segment !== '.') .join('/') || '.' ); } /** Compare path segments in ripgrep's sorted, depth-first traversal order. */ -export function comparePortableRelativePaths(left: string, right: string): number { +export function comparePortableRelativePaths( + left: string, + right: string, +): number { const encoder = new TextEncoder(); const leftSegments = left.split('/'); const rightSegments = right.split('/'); @@ -645,9 +944,14 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } -function isValidWorkspaceEditRequest(request: Record): boolean { +function isValidWorkspaceEditRequest( + request: Record, +): boolean { const hasBatch = request.edits !== undefined; - if (hasBatch && (request.oldText !== undefined || request.newText !== undefined)) { + if ( + hasBatch && + (request.oldText !== undefined || request.newText !== undefined) + ) { return false; } const edits = hasBatch @@ -665,7 +969,10 @@ function isValidWorkspaceEditRequest(request: Record): boolean if ( typeof edit !== 'object' || edit === null || - !hasOnlyKeys(edit as Record, WORKSPACE_TEXT_EDIT_KEYS) + !hasOnlyKeys( + edit as Record, + WORKSPACE_TEXT_EDIT_KEYS, + ) ) { return false; } @@ -673,9 +980,11 @@ function isValidWorkspaceEditRequest(request: Record): boolean if ( typeof candidate.oldText !== 'string' || candidate.oldText.length === 0 || - Buffer.from(candidate.oldText).toString('utf8') !== candidate.oldText || + Buffer.from(candidate.oldText).toString('utf8') !== + candidate.oldText || typeof candidate.newText !== 'string' || - Buffer.from(candidate.newText).toString('utf8') !== candidate.newText + Buffer.from(candidate.newText).toString('utf8') !== + candidate.newText ) { return false; } @@ -698,7 +1007,7 @@ function hasOnlyKeys( value: Record, allowed: ReadonlySet, ): boolean { - return Object.keys(value).every((key) => allowed.has(key)); + return Object.keys(value).every(key => allowed.has(key)); } export function isWorkspaceToolRequest( @@ -723,7 +1032,8 @@ export function isWorkspaceToolRequest( (request.maxLines === undefined || (Number.isSafeInteger(request.maxLines) && Number(request.maxLines) >= 1 && - Number(request.maxLines) <= BRIDGE_WORKSPACE_READ_MAX_LINES)) + Number(request.maxLines) <= + BRIDGE_WORKSPACE_READ_MAX_LINES)) ); } if (request.operation === 'search_text') { @@ -743,7 +1053,8 @@ export function isWorkspaceToolRequest( (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && - Number(request.maxResults) <= BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) + Number(request.maxResults) <= + BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) ); } if (request.operation === 'list_files') { @@ -753,12 +1064,14 @@ export function isWorkspaceToolRequest( isSafePortableRelativePath(request.path)) && (request.afterPath === undefined || (isSafePortableRelativePath(request.afterPath) && - normalizePortableRelativePath(request.afterPath) === request.afterPath && + normalizePortableRelativePath(request.afterPath) === + request.afterPath && isWithinRequestedPath(request.afterPath, request.path))) && (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && - Number(request.maxResults) <= BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) + Number(request.maxResults) <= + BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) ); } if (request.operation === 'write_file') { @@ -799,7 +1112,8 @@ export function isWorkspaceToolRequest( !request.command.includes('\0') && new TextEncoder().encode(request.command).byteLength <= BRIDGE_WORKSPACE_COMMAND_MAX_BYTES && - (request.cwd === undefined || isSafePortableRelativePath(request.cwd)) && + (request.cwd === undefined || + isSafePortableRelativePath(request.cwd)) && (request.timeoutMs === undefined || (Number.isSafeInteger(request.timeoutMs) && Number(request.timeoutMs) >= 1 && @@ -838,13 +1152,19 @@ export function isWorkspaceToolResult( if (request.operation === 'read_file') { const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; - const content = typeof result.content === 'string' ? result.content : null; + const content = + typeof result.content === 'string' ? result.content : null; const reportedLineCount = - Number.isSafeInteger(result.endLine) && Number(result.endLine) >= startLine - 1 + Number.isSafeInteger(result.endLine) && + Number(result.endLine) >= startLine - 1 ? Number(result.endLine) - startLine + 1 : -1; const actualLineCount = - content === null ? -1 : content.length === 0 ? reportedLineCount : content.split('\n').length; + content === null + ? -1 + : content.length === 0 + ? reportedLineCount + : content.split('\n').length; return ( hasOnlyKeys(result, WORKSPACE_READ_RESULT_KEYS) && result.path === request.path && @@ -899,7 +1219,10 @@ export function isWorkspaceToolResult( (enforcesPaginationContract && (normalizedPath !== path || (previousPath !== undefined && - comparePortableRelativePaths(normalizedPath, previousPath) <= 0))) + comparePortableRelativePaths( + normalizedPath, + previousPath, + ) <= 0))) ) { return false; } @@ -909,7 +1232,8 @@ export function isWorkspaceToolResult( if (!enforcesPaginationContract) { return result.nextAfterPath === undefined; } - if (result.truncated !== true) return result.nextAfterPath === undefined; + if (result.truncated !== true) + return result.nextAfterPath === undefined; return ( result.paths.length > 0 && result.nextAfterPath === result.paths[result.paths.length - 1] @@ -942,7 +1266,8 @@ export function isWorkspaceToolResult( if (request.operation === 'preview_edit') { const replacements = request.edits?.length ?? 1; - const content = typeof result.content === 'string' ? result.content : null; + const content = + typeof result.content === 'string' ? result.content : null; return ( hasOnlyKeys(result, WORKSPACE_PREVIEW_EDIT_RESULT_KEYS) && result.path === request.path && @@ -964,7 +1289,8 @@ export function isWorkspaceToolResult( const stdout = typeof result.stdout === 'string' ? result.stdout : null; const stderr = typeof result.stderr === 'string' ? result.stderr : null; const outputLimit = - request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + request.maxOutputBytes ?? + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; return ( hasOnlyKeys(result, WORKSPACE_COMMAND_RESULT_KEYS) && stdout !== null && @@ -980,7 +1306,8 @@ export function isWorkspaceToolResult( Number(result.exitCode) <= 255)) && (result.signal === undefined || (typeof result.signal === 'string' && - result.signal.length <= BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && + result.signal.length <= + BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && /^SIG[A-Z0-9]+$/.test(result.signal))) && typeof result.truncated === 'boolean' && typeof result.timedOut === 'boolean' && @@ -995,7 +1322,7 @@ export function isWorkspaceToolResult( return ( hasOnlyKeys(result, WORKSPACE_SEARCH_RESULT_KEYS) && result.matches.length <= maxResults && - result.matches.every((match) => { + result.matches.every(match => { if (typeof match !== 'object' || match === null) return false; const candidate = match as Record; return ( @@ -1007,7 +1334,8 @@ export function isWorkspaceToolResult( Number.isSafeInteger(candidate.column) && Number(candidate.column) >= 1 && typeof candidate.text === 'string' && - candidate.text.length <= BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && + candidate.text.length <= + BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && candidate.text.includes(request.query) ); }) @@ -1025,7 +1353,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.operations.length < 1 || capabilities.operations.length > 7 || !capabilities.operations.every( - (operation) => + operation => operation === 'read_file' || operation === 'search_text' || operation === 'list_files' || @@ -1034,7 +1362,8 @@ export function isValidBridgeWorkspaceToolCapabilities( operation === 'edit_file' || operation === 'execute_command', ) || - new Set(capabilities.operations).size !== capabilities.operations.length || + new Set(capabilities.operations).size !== + capabilities.operations.length || !Array.isArray(capabilities.workspaces) || capabilities.workspaces.length < 1 || capabilities.workspaces.length > BRIDGE_WORKSPACE_MAX_COUNT @@ -1049,7 +1378,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.writeFileModes.length > 2 || !capabilities.operations.includes('write_file') || !capabilities.writeFileModes.every( - (mode) => mode === 'replace' || mode === 'create', + mode => mode === 'replace' || mode === 'create', ) || new Set(capabilities.writeFileModes).size !== capabilities.writeFileModes.length) @@ -1065,7 +1394,7 @@ export function isValidBridgeWorkspaceToolCapabilities( (!capabilities.operations.includes('edit_file') && !capabilities.operations.includes('preview_edit')) || !capabilities.editFileModes.every( - (mode) => mode === 'single' || mode === 'batch', + mode => mode === 'single' || mode === 'batch', ) || new Set(capabilities.editFileModes).size !== capabilities.editFileModes.length) @@ -1093,13 +1422,23 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.programmaticLanguages !== undefined && + (!Array.isArray(capabilities.programmaticLanguages) || + capabilities.programmaticLanguages.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.programmaticLanguages[0] !== 'bash') + ) { + return false; + } + const workspaceIds = new Set(); - return capabilities.workspaces.every((workspace) => { + return capabilities.workspaces.every(workspace => { if (typeof workspace !== 'object' || workspace === null) return false; const descriptor = workspace as Record; if ( Object.keys(descriptor).some( - (key) => key !== 'id' && key !== 'name' && key !== 'operations', + key => key !== 'id' && key !== 'name' && key !== 'operations', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || @@ -1107,17 +1446,21 @@ export function isValidBridgeWorkspaceToolCapabilities( (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || - descriptor.name.length > BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || + descriptor.name.length > + BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || (descriptor.operations !== undefined && (!Array.isArray(descriptor.operations) || descriptor.operations.length < 1 || descriptor.operations.length > (capabilities.operations as unknown[]).length || descriptor.operations.some( - (operation) => - !(capabilities.operations as unknown[]).includes(operation), + operation => + !(capabilities.operations as unknown[]).includes( + operation, + ), ) || - new Set(descriptor.operations).size !== descriptor.operations.length)) + new Set(descriptor.operations).size !== + descriptor.operations.length)) ) { return false; } @@ -1139,11 +1482,12 @@ export function isValidBridgeWorkerCapabilities( typeof capabilities.statefulWorkspace === 'boolean' && typeof capabilities.sandboxProfile === 'string' && capabilities.sandboxProfile.trim().length > 0 && - capabilities.sandboxProfile.length <= BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && + capabilities.sandboxProfile.length <= + BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && Array.isArray(capabilities.runtimes) && capabilities.runtimes.length <= BRIDGE_RUNTIME_MAX_COUNT && capabilities.runtimes.every( - (runtime) => + runtime => typeof runtime === 'string' && runtime.length > 0 && runtime.length <= BRIDGE_RUNTIME_MAX_LENGTH, diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index ba65a3cf..6e8cc8b1 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -312,3 +312,51 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b assert.equal(executed, true); assert.equal(internals.activeWorkspaceAssignments.size, 0); }); + +test('programmatic work on an independent workspace bypasses another root cleanup', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map< + string, + { id: string; done: Promise } + >; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + internals.activeWorkspaceAssignments.set('a', { + id: 'previous', + done: new Promise(() => {}), + }); + let executed = false; + internals.executeOwned = async () => { + executed = true; + assert.equal(internals.activeWorkspaceAssignments.get('b')?.id, 'next'); + }; + await worker.executeAndSettle({ + assignmentId: 'next', + executionKind: 'workspace_programmatic', + workspaceId: 'b', + remainingMs: 1_000, + request: { + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }, + } as BridgeAssignment); + assert.equal(executed, true); + assert.equal(internals.activeWorkspaceAssignments.has('b'), false); + assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 19f219da..a0517bd9 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -5,6 +5,7 @@ import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, + isBridgeWorkspaceProgrammaticRequest, isWorkspaceToolResult, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; @@ -21,6 +22,7 @@ import type { BridgeWorkerCredentialResponse, BridgeWorkerRegistrationResponse, BridgeWorkspaceToolOperation, + BridgeWorkspaceProgrammaticRequest, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -35,6 +37,13 @@ export interface BridgeWorkerOptions { runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; + workspaceProgrammatic?: { + executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise; + }; workspaceMutationQuarantine?: WorkspaceMutationQuarantine; /** Required per-root durable guards when opting into concurrent workspace leases. */ workspaceQuarantines?: ReadonlyMap; @@ -164,6 +173,11 @@ function workspaceCapabilitiesMatch( (advertised.listFileFeatures?.every( (feature, index) => feature === executor.listFileFeatures?.[index], ) ?? executor.listFileFeatures == null) && + advertised.programmaticLanguages?.length === + executor.programmaticLanguages?.length && + (advertised.programmaticLanguages?.every( + (language, index) => language === executor.programmaticLanguages?.[index], + ) ?? executor.programmaticLanguages == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -223,6 +237,7 @@ function registrationCompatibleCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; return { @@ -302,11 +317,16 @@ function supportedWorkspaceCapabilities( const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), ); + const programmaticLanguages = desired.programmaticLanguages?.filter( + (language) => + registration.supportedWorkspaceProgrammaticLanguages?.includes(language), + ); const { writeFileModes: _writeFileModes, editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; return { @@ -327,6 +347,10 @@ function supportedWorkspaceCapabilities( ...(operations.includes('list_files') && listFileFeatures?.length ? { listFileFeatures } : {}), + ...(operations.includes('execute_command') && + programmaticLanguages?.length + ? { programmaticLanguages } + : {}), }, }; } @@ -406,6 +430,16 @@ export class BridgeWorker { 'Workspace tool capabilities require a matching executor', ); } + if ( + (options.workspaceProgrammatic != null) !== + (options.capabilities.workspaceTools?.programmaticLanguages?.includes( + 'bash', + ) === true) + ) { + throw new BridgeProtocolError( + 'Workspace programmatic capability requires a matching executor', + ); + } if ( options.capabilities.workspaceTools?.operations.some( (operation) => @@ -1130,11 +1164,7 @@ export class BridgeWorker { assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const root = - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ? assignment.request.workspaceId - : undefined; + const root = this.assignmentWorkspaceId(assignment); const waitingAt = Date.now(); while (root != null && this.activeWorkspaceAssignments.has(root)) { const active = this.activeWorkspaceAssignments.get(root)!; @@ -1212,14 +1242,31 @@ export class BridgeWorker { private workspaceGuard( assignment: BridgeAssignment, ): WorkspaceMutationQuarantine | undefined { - return assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ? (this.options.workspaceQuarantines?.get( - assignment.request.workspaceId, - ) ?? this.options.workspaceMutationQuarantine) + const workspaceId = this.assignmentWorkspaceId(assignment); + return workspaceId != null + ? (this.options.workspaceQuarantines?.get(workspaceId) ?? + this.options.workspaceMutationQuarantine) : this.options.workspaceMutationQuarantine; } + private assignmentWorkspaceId( + assignment: BridgeAssignment, + ): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.workspaceId; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + typeof assignment.workspaceId === 'string' + ) { + return assignment.workspaceId; + } + return undefined; + } + private async executeOwned( assignment: BridgeAssignment, signal?: AbortSignal, @@ -1472,6 +1519,76 @@ export class BridgeWorker { 'Bridge assignment expired during workspace execution', ); } + } else if (assignment.executionKind === 'workspace_programmatic') { + const workspaceId = assignment.workspaceId; + if ( + workspaceId == null || + this.options.workspaceProgrammatic == null || + !isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + throw new BridgeProtocolError( + 'Worker does not provide valid selected-workspace programmatic execution', + ); + } + try { + if (this.quarantinedWorkspaces.has(workspaceId)) { + throw new Error('Workspace requires an explicit quarantine reset'); + } + if (this.options.workspaceQuarantines != null) + await guard?.assertAvailable(); + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Workspace is quarantined', + error, + ); + } + const advertised = this.activeCapabilities.workspaceTools; + const workspace = advertised?.workspaces.find( + (candidate) => candidate.id === workspaceId, + ); + if ( + workspace == null || + !advertised?.operations.includes('execute_command') || + (workspace.operations != null && + !workspace.operations.includes('execute_command')) || + !advertised.programmaticLanguages?.includes('bash') + ) { + throw new BridgeProtocolError( + 'Selected-workspace programmatic execution is not advertised', + ); + } + this.mutationGuardArmed = true; + try { + this.armedWorkspaces.add(workspaceId); + await guard!.arm( + 'Workspace programmatic execution is pending settlement', + assignment.assignmentId, + ); + workspaceMutationArmed = true; + } catch (error) { + this.mutationGuardArmed = false; + throw new BridgeWorkspaceQuarantinedError( + 'Workspace mutation quarantine could not be armed before execution', + error, + ); + } + payload = await this.options.workspaceProgrammatic.executeProgrammatic( + workspaceId, + assignment.request, + executionController.signal, + ); + workspaceMutationApplied = true; + if (executionController.signal.aborted) { + throw ( + executionController.signal.reason ?? + new DOMException('aborted', 'AbortError') + ); + } + if (Date.now() >= localDeadlineAtMs) { + throw new BridgeProtocolError( + 'Bridge assignment expired during programmatic execution', + ); + } } else { runtimeLease = await this.runtimeSupervisor.acquire( assignment, @@ -1581,7 +1698,8 @@ export class BridgeWorker { leaseToken: assignment.leaseToken, incarnationId: this.incarnationId, status: 'rejected', - ...(assignment.executionKind === 'workspace_tool' && + ...((assignment.executionKind === 'workspace_tool' || + assignment.executionKind === 'workspace_programmatic') && error instanceof WorkspaceToolError ? { errorCode: error.code } : {}), @@ -1693,12 +1811,8 @@ export class BridgeWorker { clearTimeout(timer); } } - if ( - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ) { - this.armedWorkspaces.delete(assignment.request.workspaceId); - } + const workspaceId = this.assignmentWorkspaceId(assignment); + if (workspaceId != null) this.armedWorkspaces.delete(workspaceId); this.mutationGuardArmed = false; } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1957,11 +2071,12 @@ export class BridgeWorker { const fulfilledWorkspaceMutation = workspaceMutationApplied && settlement.status === 'fulfilled' && - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) && - (assignment.request.operation === 'write_file' || - assignment.request.operation === 'edit_file' || - assignment.request.operation === 'execute_command'); + (assignment.executionKind === 'workspace_programmatic' || + (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + (assignment.request.operation === 'write_file' || + assignment.request.operation === 'edit_file' || + assignment.request.operation === 'execute_command'))); if (signal?.aborted === true) { if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 8f9fcd45..fa08a84c 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -950,6 +950,88 @@ test('worker executes a workspace tool assignment locally without acquiring a sa }); }); +test('worker executes programmatic Bash in the selected workspace and preserves its fence', async () => { + const programmaticRequests: object[] = []; + const quarantineEvents: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + async executeProgrammatic(workspaceId, request) { + programmaticRequests.push({ workspaceId, request }); + return { + session_id: 'session-1', + language: 'bash', + version: '5.2', + files: [], + run: { stdout: 'ready\n', stderr: '', code: 0, signal: null }, + }; + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + (reason) => quarantineEvents.push(`quarantine:${reason}`), + (reason) => quarantineEvents.push(`arm:${reason}`), + () => quarantineEvents.push('clear'), + ), + ], + ]), + fetchImpl: async () => Response.json({ protocolVersion: 1, accepted: true }), + }); + const request = { + body: { + language: 'bash' as const, + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }; + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-1', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request, + }); + + assert.deepEqual(programmaticRequests, [{ workspaceId: 'primary', request }]); + assert.deepEqual(quarantineEvents, [ + 'arm:Workspace programmatic execution is pending settlement', + 'clear', + ]); +}); + test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { let quarantinedReason: string | undefined; let armed = 0; diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 5bfc07fe..dfda49fc 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -112,6 +112,8 @@ export interface SandboxWorkspaceToolsOptions { commandSandbox: WorkspaceCommandSandbox; /** Workspace IDs whose sandbox is configured and may run commands. */ commandWorkspaces: string[]; + /** Optional execution-scoped languages supplied by the same command sandbox. */ + programmaticLanguages?: BridgeWorkspaceToolCapabilities['programmaticLanguages']; } const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; @@ -1657,6 +1659,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { ...(base.listFileFeatures != null ? { listFileFeatures: base.listFileFeatures } : {}), + ...(options.programmaticLanguages?.length + ? { programmaticLanguages: [...options.programmaticLanguages] } + : {}), workspaces: base.workspaces.map((workspace) => ({ ...workspace, operations: [ diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 25b0bdaf..369b306c 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -414,6 +414,7 @@ router.post( supportedWorkspaceEditFileModes: ['single', 'batch'], supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], supportedWorkspaceListFileFeatures: ['after_path'], + supportedWorkspaceProgrammaticLanguages: ['bash'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/selection.ts b/service/src/bridge/selection.ts index 0959279f..f3926e8a 100644 --- a/service/src/bridge/selection.ts +++ b/service/src/bridge/selection.ts @@ -1,4 +1,5 @@ export const CODEAPI_BRIDGE_WORKER_HEADER = 'X-LibreChat-Code-Worker-ID'; +export const CODEAPI_BRIDGE_WORKSPACE_HEADER = 'X-LibreChat-Code-Workspace-ID'; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; export class BridgeWorkerSelectionError extends Error { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index b09d3792..d6b91469 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -151,6 +151,26 @@ function supportsWorkspaceTool( return true; } +function supportsWorkspaceProgrammatic( + registration: RegisteredBridgeWorker, + workspaceId: string, + language: string, +): boolean { + const capabilities = registration.capabilities.workspaceTools; + const workspace = capabilities?.workspaces.find( + (candidate) => candidate.id === workspaceId, + ); + return ( + workspace != null && + capabilities?.operations.includes('execute_command') === true && + (workspace.operations == null || + workspace.operations.includes('execute_command')) && + capabilities.programmaticLanguages?.includes( + language as 'bash', + ) === true + ); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -700,6 +720,7 @@ export class RedisBridgeStore { body: t.PayloadBody; headers: Record; workspaceRequest?: WorkspaceToolRequest; + workspaceId?: string; runtimeSessionId?: string; deadlineAtMs: number; executionTimeoutMs?: number; @@ -709,6 +730,12 @@ export class RedisBridgeStore { registration: RegisteredBridgeWorker, ) => Promise; }): Promise { + if (args.workspaceRequest != null && args.workspaceId != null) { + throw new BridgeStoreError( + 'ASSIGNMENT_INVALID', + 'A bridge assignment cannot be both a workspace tool and programmatic execution', + ); + } if ( args.executionTimeoutMs !== undefined && (args.workspaceRequest == null || @@ -773,6 +800,19 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not advertise the requested workspace tool`, ); } + if ( + args.workspaceId != null && + !supportsWorkspaceProgrammatic( + registration, + args.workspaceId, + args.body.language, + ) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} does not advertise programmatic execution for the selected workspace`, + ); + } if ( args.runtimeSessionId !== undefined && (await this.dispatchCommand( @@ -799,14 +839,16 @@ export class RedisBridgeStore { const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let workspaceLeaseSlot: number | undefined; + const selectedWorkspaceId = + args.workspaceRequest?.workspaceId ?? args.workspaceId; const workspaceSlots = - args.workspaceRequest != null && + selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 ? new BridgeWorkspaceSlots(this.redis) : undefined; let resultCommitted = false; const admission = - args.workspaceRequest == null + selectedWorkspaceId == null ? undefined : new BridgeAdmissionQueue(this.redis); try { @@ -820,7 +862,7 @@ export class RedisBridgeStore { args.deadlineAtMs, workspaceSlots == null ? undefined - : args.workspaceRequest?.workspaceId, + : selectedWorkspaceId, ), args, 'Bridge admission enqueue', @@ -855,7 +897,7 @@ export class RedisBridgeStore { workerId: args.workerId, incarnationId: lockIncarnationId, assignmentId, - workspaceId: args.workspaceRequest!.workspaceId, + workspaceId: selectedWorkspaceId!, capacity: registration.capabilities.workspaceLeaseSlots!, expiresAtMs: Date.now() + ttlSeconds * 1000, }), @@ -910,7 +952,14 @@ export class RedisBridgeStore { ); } if ( - !supportsWorkspaceTool(current.registration, args.workspaceRequest!) + (args.workspaceRequest != null && + !supportsWorkspaceTool(current.registration, args.workspaceRequest)) || + (args.workspaceId != null && + !supportsWorkspaceProgrammatic( + current.registration, + args.workspaceId, + args.body.language, + )) ) { throw new BridgeStoreError( 'WORKER_MISMATCH', @@ -935,11 +984,14 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), + ...(selectedWorkspaceId == null ? {} : { + workspaceFence: `native-workspace:${selectedWorkspaceId}`, + }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${args.workspaceRequest!.workspaceId}`, + workspaceFence: `native-workspace:${selectedWorkspaceId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -951,6 +1003,15 @@ export class RedisBridgeStore { executionKind: 'workspace_tool' as const, request: args.workspaceRequest, } + : args.workspaceId != null + ? { + executionKind: 'workspace_programmatic' as const, + workspaceId: args.workspaceId, + request: { + body: args.body, + headers: args.headers, + }, + } : { request: { body: args.body, @@ -1011,6 +1072,19 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} no longer advertises the requested workspace tool`, ); } + if ( + args.workspaceId != null && + !supportsWorkspaceProgrammatic( + replacement.registration, + args.workspaceId, + args.body.language, + ) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} no longer advertises programmatic execution for the selected workspace`, + ); + } registration = replacement.registration; readyToken = replacement.readyToken; } @@ -1034,11 +1108,24 @@ export class RedisBridgeStore { resultCommitted = true; return result; } catch (error) { - if (args.runtimeSessionId !== undefined) { + if (assignment.workspaceFence != null) { + // Native roots retain their own fence through result restoration. + // Do not quarantine unrelated roots or invalidate the worker lease. + await boundedCommand(this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", + "redis.call('SET', KEYS[1], 'quarantined:' .. ARGV[1])", + 'return 1', + ].join('\n'), + 1, + workspaceQuarantineKey(args.workerId, assignment.workspaceFence), + assignment.assignmentId, + ), this.redisCommandTimeoutMs, 'Bridge native workspace finalization quarantine'); + } else if (assignmentWorkspace(assignment) !== undefined) { await this.quarantine( args.workerId, assignment.incarnationId, - args.runtimeSessionId, + assignmentWorkspace(assignment)!, ); } throw error; @@ -1904,10 +1991,11 @@ export class RedisBridgeStore { : undefined; const cancelledMutation = signal.aborted && - workspaceRequest != null && - (workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'); + (assignment.executionKind === 'workspace_programmatic' || + (workspaceRequest != null && + (workspaceRequest.operation === 'write_file' || + workspaceRequest.operation === 'edit_file' || + workspaceRequest.operation === 'execute_command'))); if (cancelledMutation) { try { // Keep the acknowledged assignment available long enough for the diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index 5b0186d0..f248aa40 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -77,6 +77,123 @@ test('dispatches a workspace tool only to a worker advertising its workspace and }); }); +for (const finalizationFails of [false, true]) test(`single-slot programmatic finalization retains the workspace fence (failure=${finalizationFails})`, async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const body = { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }; + const completion = store.dispatch({ + workerId: 'workspace-worker', + body, + headers: {}, + workspaceId: 'primary', + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + finalize: async settlement => { + if (finalizationFails) throw new Error('artifact restoration failed'); + return settlement; + }, + }); + + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { body }, + }); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + session_id: 'session-1', + language: 'bash', + version: '5.2', + files: [], + run: { + stdout: 'ready\n', + stderr: '', + code: 0, + signal: null, + output: 'ready\n', + memory: null, + message: null, + status: null, + cpu_time: null, + wall_time: 0.01, + }, + }, + }); + + if (finalizationFails) { + await expect(completion).rejects.toThrow('artifact restoration failed'); + await expect(store.dispatch({ workerId: 'workspace-worker', body, headers: {}, + workspaceId: 'primary', deadlineAtMs: Date.now() + 1000, + signal: new AbortController().signal, + })).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + return; + } + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { session_id: 'session-1' }, + }); +}); + +test('rejects programmatic execution without the workspace capability', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatch({ + workerId: 'workspace-worker', + body: { + language: 'bash', + version: '5.2', + session_id: 'session-2', + files: [{ name: 'main.sh', content: 'echo denied' }], + }, + headers: {}, + workspaceId: 'primary', + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + test('drains an acknowledged workspace mutation cancellation before releasing it', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index f86ce656..873db173 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -45,6 +45,7 @@ import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; import { isOpaqueObjectContentDisposition } from './file-metadata'; import { mapObjectDetails } from './file-object-resolver'; +import { isSupportedBridgeArtifactName } from '../../packages/code/src/protocol'; export const app: Express = express(); app.disable('x-powered-by'); @@ -52,29 +53,6 @@ validateEgressGatewayHardenedConfig(); app.use(traceHttpRequest('codeapi.egress_gateway.request')); app.use(httpMetricsMiddleware); -const SUPPORTED_OUTPUT_EXTENSIONS = new Set([ - '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', - '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', - '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', - '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', - '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', - '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', - '.xml', '.yaml', '.yml', - '.ics', '.ical', '.ifb', '.icalendar', - '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', - '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', - '.odt', '.ods', '.odp', '.rtf', - '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', - '.tif', '.tiff', '.webp', - '.eot', '.ttf', '.woff', '.woff2', - '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', - '.tf', '.tfvars', '.tfstate', '.hcl', - '.dockerfile', '.Dockerfile', '.dockerignore', - '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', - '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', - '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', -]); - type EgressAuditFields = { execHash?: string; requestExecHash?: string; @@ -248,18 +226,7 @@ function assertOutputFilenameAllowed(name: string): void { throw new EgressGrantError('malformed', 'Output filename must be canonical'); } if (!isDirkeepName(name)) { - const basename = path.posix.basename(name); - const ext = path.posix.extname(basename).toLowerCase(); - const dottedBasename = `.${basename}`; - const allowed = - (ext !== '' && SUPPORTED_OUTPUT_EXTENSIONS.has(ext)) || - SUPPORTED_OUTPUT_EXTENSIONS.has(basename) || - SUPPORTED_OUTPUT_EXTENSIONS.has(basename.toLowerCase()) || - (ext === '' && ( - SUPPORTED_OUTPUT_EXTENSIONS.has(dottedBasename) || - SUPPORTED_OUTPUT_EXTENSIONS.has(dottedBasename.toLowerCase()) - )); - if (!allowed) { + if (!isSupportedBridgeArtifactName(name)) { throw new EgressGrantError('scope_mismatch', 'Output filename extension is not supported'); } } diff --git a/service/src/egress-grant.test.ts b/service/src/egress-grant.test.ts index 32ef2988..cd26fcd7 100644 --- a/service/src/egress-grant.test.ts +++ b/service/src/egress-grant.test.ts @@ -5,6 +5,7 @@ import { env } from './config'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, + normalizeSelectedWorkspaceProgrammaticTimeoutMs, prepareSandboxJobSecurity, refreshEgressGrantClaims, timeoutMsToGrantSeconds, @@ -478,6 +479,13 @@ describe('egress encrypted grants and handles', () => { expect(() => normalizeProgrammaticTimeoutMs(0, 300000)).toThrow('timeout must be a positive number'); }); + test('budgets both selected-workspace replay passes inside the worker deadline', () => { + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(undefined, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(120_000, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(300_000, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(10_000, 20_000)).toBe(2_167); + }); + test('normalizes the gateway callback URL for sandbox-originated PTC', () => { expect(normalizeEgressGatewayUrl(' http://egress-gateway:3190/// ')).toBe('http://egress-gateway:3190'); expect(() => normalizeEgressGatewayUrl(' ')).toThrow('EGRESS_GATEWAY_URL is required'); diff --git a/service/src/preamble-bash.test.ts b/service/src/preamble-bash.test.ts index 815040d3..e35a77f5 100644 --- a/service/src/preamble-bash.test.ts +++ b/service/src/preamble-bash.test.ts @@ -1,10 +1,19 @@ import { execFileSync } from 'child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; import { describe, expect, test } from 'bun:test'; import { extractPendingFromStdout, type LCTool } from './preamble'; -import { generateBashReplayPostamble, generateBashReplayPreamble } from './preamble-bash'; +import { + generateBashReplayPostamble, + generateBashReplayPreamble, +} from './preamble-bash'; interface BashRunResult { stdout: string; @@ -60,9 +69,13 @@ function assemble(userCode: string, toolSet: LCTool[] = tools): string { ].join('\n'); } -function runBash(script: string, options: number | BashRunOptions = {}): BashRunResult { - const timeoutMs = typeof options === 'number' ? options : options.timeoutMs ?? 3000; - const history = typeof options === 'number' ? {} : options.history ?? {}; +function runBash( + script: string, + options: number | BashRunOptions = {}, +): BashRunResult { + const timeoutMs = + typeof options === 'number' ? options : (options.timeoutMs ?? 3000); + const history = typeof options === 'number' ? {} : (options.history ?? {}); const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-unit-')); const file = join(dir, 'main.sh'); const historyPath = join(dir, 'history.json'); @@ -99,46 +112,136 @@ function pendingNames(stdout: string): string[] { return (parsed.pending ?? []).map(call => call.tool_name).sort(); } +describe('generateBashReplayPreamble - private runtime directory', () => { + test('creates every replay tempfile beneath TMPDIR', () => { + const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-private-tmp-')); + const dataDir = join(dir, 'data'); + const runtimeDir = join(dir, 'runtime'); + const file = join(dir, 'main.sh'); + const historyPath = join(dataDir, 'history.json'); + mkdirSync(dataDir, { recursive: true }); + mkdirSync(runtimeDir, { recursive: true }); + writeFileSync(historyPath, '{}'); + writeFileSync( + file, + assemble(` +printf '%s\\n' "$_PTC_PENDING_FILE" "$_PTC_ERROR_FILE" "$_PTC_COUNTER_FILE" +`), + { mode: 0o755 }, + ); + + try { + const stdout = execFileSync('bash', [file], { + env: { + ...process.env, + PTC_HISTORY_PATH: historyPath, + TMPDIR: runtimeDir, + }, + encoding: 'utf8', + }); + const paths = stdout.trim().split('\n'); + expect(paths).toHaveLength(3); + expect( + paths.every(value => value.startsWith(`${runtimeDir}/`)), + ).toBe(true); + expect( + generateBashReplayPreamble({ executionId, tools }), + ).not.toContain('mktemp -t'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('persists pending calls through the private native control path', () => { + const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-control-')); + const file = join(dir, 'main.sh'); + const historyPath = join(dir, 'history.json'); + const controlPath = join(dir, 'control.json'); + writeFileSync(file, assemble(`get_weather '{"city":"Paris"}'`), { + mode: 0o755, + }); + writeFileSync(historyPath, '{}'); + try { + execFileSync('bash', [file], { + env: { + ...process.env, + PTC_HISTORY_PATH: historyPath, + LIBRECHAT_CODE_CONTROL_PATH: controlPath, + TMPDIR: dir, + }, + encoding: 'utf8', + }); + expect(JSON.parse(readFileSync(controlPath, 'utf8'))).toMatchObject( + { + pending: [ + { + call_id: 'call_001', + tool_name: 'get_weather', + input: { city: 'Paris' }, + }, + ], + }, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('generateBashReplayPreamble - command substitution pending emission', () => { test('emits ClickHouse-style object input with SQL quotes from double-quoted JSON', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` SVC="45886e06-932b-4cff-bb49-3f7281d80717" result=$(run_select_query_mcp_ClickHouse "{\\"serviceId\\":\\"$SVC\\",\\"query\\":\\"SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999\\"}") echo "AFTER: $result" -`, [clickHouseTool])); +`, + [clickHouseTool], + ), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); expect(parsed.pending).toHaveLength(1); - expect(parsed.pending?.[0]?.tool_name).toBe('run_select_query_mcp_ClickHouse'); + expect(parsed.pending?.[0]?.tool_name).toBe( + 'run_select_query_mcp_ClickHouse', + ); expect(parsed.pending?.[0]?.input).toEqual({ serviceId: '45886e06-932b-4cff-bb49-3f7281d80717', - query: - "SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999", + query: "SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999", }); expect(parsed.stdout).not.toContain('AFTER'); }); test('emits ClickHouse-style object input with shell-escaped SQL quotes', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` result=$(run_select_query_mcp_ClickHouse '{"serviceId":"45886e06-932b-4cff-bb49-3f7281d80717","query":"SELECT name, type FROM system.columns WHERE database='"'"'default'"'"' AND table='"'"'uk_prices_3'"'"' ORDER BY position"}') echo "AFTER: $result" -`, [clickHouseTool])); +`, + [clickHouseTool], + ), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); expect(parsed.pending).toHaveLength(1); - expect(parsed.pending?.[0]?.tool_name).toBe('run_select_query_mcp_ClickHouse'); + expect(parsed.pending?.[0]?.tool_name).toBe( + 'run_select_query_mcp_ClickHouse', + ); expect(parsed.pending?.[0]?.input).toEqual({ serviceId: '45886e06-932b-4cff-bb49-3f7281d80717', - query: - "SELECT name, type FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", + query: "SELECT name, type FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", }); expect(parsed.stdout).not.toContain('AFTER'); }); test('batches parallel ClickHouse-style command substitutions into one pending block', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` SVC="45886e06-932b-4cff-bb49-3f7281d80717" { @@ -158,7 +261,11 @@ SVC="45886e06-932b-4cff-bb49-3f7281d80717" wait echo "AFTER" -`, [clickHouseTool]), 3000); +`, + [clickHouseTool], + ), + 3000, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -168,7 +275,11 @@ echo "AFTER" 'run_select_query_mcp_ClickHouse', 'run_select_query_mcp_ClickHouse', ]); - expect(parsed.pending?.map(call => (call.input as { query: string }).query).sort()).toEqual([ + expect( + parsed.pending + ?.map(call => (call.input as { query: string }).query) + .sort(), + ).toEqual([ "SELECT name, engine, total_rows, formatReadableSize(total_bytes) AS size, sorting_key, partition_key FROM system.tables WHERE database='default' AND name IN ('uk_prices_3','weather_noaa_mt')", "SELECT name, type, comment FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", "SELECT name, type, comment FROM system.columns WHERE database='default' AND table='weather_noaa_mt' ORDER BY position", @@ -177,12 +288,14 @@ echo "AFTER" }); test('emits a command-substitution tool call before later user code while another job is running', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` sleep 0.2 & result=$(get_weather '{"city":"Madrid"}') echo "AFTER: $result" wait -`)); +`), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -194,12 +307,15 @@ wait }); test('batches background and command-substitution tool calls before command-substitution side effects', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Oslo"}' & result=$(calculate '{"expression":"2+3"}') echo "SIDE_EFFECT: $result" wait -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -210,10 +326,13 @@ wait }); test('waits for background compound commands that invoke tools later', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` (sleep 0.2; get_weather '{"city":"Paris"}') & echo "AFTER LAUNCH" -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -225,10 +344,13 @@ echo "AFTER LAUNCH" }); test('does not wait for unrelated background commands with tool names as arguments', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` bash -c 'sleep 2' get_weather & echo "DONE" -`), 700); +`), + 700, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -238,14 +360,17 @@ echo "DONE" }); test('does not treat arithmetic expansion as command substitution while batching background tools', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Oslo"}' & sleep 0.1 x=$((1+1)) calculate '{"expression":"2+3"}' & wait echo "DONE $x" -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -256,12 +381,15 @@ echo "DONE $x" }); test('handles backtick command substitution without waiting for unrelated background jobs', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` sleep 5 & result=\`get_weather '{"city":"Porto"}'\` echo "AFTER: $result" wait -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -281,7 +409,10 @@ wait echo "DONE" `; const firstRun = runBash(assemble(userCode)); - const firstParsed = extractPendingFromStdout(firstRun.stdout, executionId); + const firstParsed = extractPendingFromStdout( + firstRun.stdout, + executionId, + ); expect(firstRun.exitCode).toBe(0); expect(firstParsed.pending).toHaveLength(2); @@ -302,7 +433,10 @@ echo "DONE" }), ); const replayRun = runBash(assemble(userCode), { history }); - const replayParsed = extractPendingFromStdout(replayRun.stdout, executionId); + const replayParsed = extractPendingFromStdout( + replayRun.stdout, + executionId, + ); expect(replayRun.exitCode).toBe(0); expect(replayParsed.pending).toBeNull(); expect(replayParsed.stdout).toContain('"slot":"first"'); @@ -326,12 +460,15 @@ echo "DONE" }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' get_weather '{"city":"Paris"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -352,12 +489,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -385,12 +525,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -418,12 +561,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index 4ec3fc05..c442b495 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -1,8 +1,5 @@ import type { LCTool } from './preamble'; -import { - buildScopedSentinel, - PTC_HISTORY_SANDBOX_PATH, -} from './ptc-constants'; +import { buildScopedSentinel, PTC_HISTORY_SANDBOX_PATH } from './ptc-constants'; export interface BashReplayPreambleConfig { executionId: string; @@ -42,12 +39,55 @@ export class BashToolNameCollisionError extends Error { } const BASH_RESERVED = new Set([ - 'if', 'then', 'else', 'elif', 'fi', 'case', 'esac', 'for', 'select', - 'while', 'until', 'do', 'done', 'in', 'function', 'time', 'coproc', - 'return', 'exit', 'break', 'continue', 'shift', 'export', 'readonly', - 'local', 'declare', 'typeset', 'unset', 'alias', 'unalias', 'source', - 'echo', 'printf', 'read', 'cd', 'pwd', 'kill', 'trap', 'wait', 'eval', - 'exec', 'jobs', 'bg', 'fg', 'set', 'let', 'test', 'true', 'false', + 'if', + 'then', + 'else', + 'elif', + 'fi', + 'case', + 'esac', + 'for', + 'select', + 'while', + 'until', + 'do', + 'done', + 'in', + 'function', + 'time', + 'coproc', + 'return', + 'exit', + 'break', + 'continue', + 'shift', + 'export', + 'readonly', + 'local', + 'declare', + 'typeset', + 'unset', + 'alias', + 'unalias', + 'source', + 'echo', + 'printf', + 'read', + 'cd', + 'pwd', + 'kill', + 'trap', + 'wait', + 'eval', + 'exec', + 'jobs', + 'bg', + 'fg', + 'set', + 'let', + 'test', + 'true', + 'false', ]); function normalizeBashFunctionName(name: string): string { @@ -60,10 +100,7 @@ function normalizeBashFunctionName(name: string): string { * the end-of-preamble `readonly -f` lockdown runs. Compared case- * insensitively because the `_PTC_` prefix is used for variables and * `_ptc_` for functions, and both live in the same identifier space. */ - if ( - BASH_RESERVED.has(normalized) || - /^_ptc_/i.test(normalized) - ) { + if (BASH_RESERVED.has(normalized) || /^_ptc_/i.test(normalized)) { normalized = normalized + '_tool'; } if (normalized === '') normalized = 'tool'; @@ -95,9 +132,12 @@ function escapeForBashEre(s: string): string { * Users capture results via command substitution; input is passed as a single * JSON object string argument (validated by jq). */ -export function generateBashReplayPreamble(config: BashReplayPreambleConfig): string { +export function generateBashReplayPreamble( + config: BashReplayPreambleConfig, +): string { const { executionId, tools } = config; - const { start: scopedStart, end: scopedEnd } = buildScopedSentinel(executionId); + const { start: scopedStart, end: scopedEnd } = + buildScopedSentinel(executionId); let preamble = `#!/bin/bash # ============================================================================ @@ -109,20 +149,25 @@ _PTC_EXECUTION_ID="${executionId}" _PTC_SENTINEL_START="${scopedStart}" _PTC_SENTINEL_END="${scopedEnd}" _PTC_HISTORY_PATH="\${PTC_HISTORY_PATH:-${PTC_HISTORY_SANDBOX_PATH}}" -_PTC_PENDING_FILE="$(mktemp -t _ptc_pending.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pending.XXXXXX)" -_PTC_ERROR_FILE="$(mktemp -t _ptc_error.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_error.XXXXXX)" -_PTC_CONSUMED_FILE="$(mktemp -t _ptc_consumed.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_consumed.XXXXXX)" -_PTC_SAW_BARE_TOOL_FILE="$(mktemp -t _ptc_saw_tool.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_saw_tool.XXXXXX)" -_PTC_PRE_TOOL_JOBS_FILE="$(mktemp -t _ptc_pre_tool_jobs.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pre_tool_jobs.XXXXXX)" -_PTC_PRE_TOOL_JOBS_READY_FILE="$(mktemp -t _ptc_pre_tool_jobs_ready.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pre_tool_jobs_ready.XXXXXX)" -_PTC_TOOL_JOBS_FILE="$(mktemp -t _ptc_tool_jobs.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_tool_jobs.XXXXXX)" -_PTC_WAIT_RAN_FILE="$(mktemp -t _ptc_wait_ran.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_wait_ran.XXXXXX)" -_PTC_SUPPRESS_SUBSHELL_TOOL_FILE="$(mktemp -t _ptc_suppress_subshell_tool.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_suppress_subshell_tool.XXXXXX)" -_PTC_SUPPRESS_SUBSHELL_TOOL_CLEAR_FILE="$(mktemp -t _ptc_suppress_subshell_tool_clear.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_suppress_subshell_tool_clear.XXXXXX)" +_PTC_CONTROL_PATH="\${LIBRECHAT_CODE_CONTROL_PATH:-}" +_PTC_RUNTIME_DIR="\${TMPDIR:-/tmp}" +_ptc_mktemp() { + mktemp "\${_PTC_RUNTIME_DIR%/}/$1.XXXXXX" +} +_PTC_PENDING_FILE="$(_ptc_mktemp _ptc_pending)" +_PTC_ERROR_FILE="$(_ptc_mktemp _ptc_error)" +_PTC_CONSUMED_FILE="$(_ptc_mktemp _ptc_consumed)" +_PTC_SAW_BARE_TOOL_FILE="$(_ptc_mktemp _ptc_saw_tool)" +_PTC_PRE_TOOL_JOBS_FILE="$(_ptc_mktemp _ptc_pre_tool_jobs)" +_PTC_PRE_TOOL_JOBS_READY_FILE="$(_ptc_mktemp _ptc_pre_tool_jobs_ready)" +_PTC_TOOL_JOBS_FILE="$(_ptc_mktemp _ptc_tool_jobs)" +_PTC_WAIT_RAN_FILE="$(_ptc_mktemp _ptc_wait_ran)" +_PTC_SUPPRESS_SUBSHELL_TOOL_FILE="$(_ptc_mktemp _ptc_suppress_subshell_tool)" +_PTC_SUPPRESS_SUBSHELL_TOOL_CLEAR_FILE="$(_ptc_mktemp _ptc_suppress_subshell_tool_clear)" # Counter must persist across subshells (command substitution) so call_ids # stay deterministic across cached/uncached calls. Bash variables set in a # subshell don't propagate back, so we use a file. -_PTC_COUNTER_FILE="$(mktemp -t _ptc_counter.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_counter.XXXXXX)" +_PTC_COUNTER_FILE="$(_ptc_mktemp _ptc_counter)" _PTC_LOCK_DIR="\${_PTC_PENDING_FILE}.lock" printf '0' > "$_PTC_COUNTER_FILE" : > "$_PTC_CONSUMED_FILE" @@ -243,7 +288,7 @@ _ptc_prune_finished_tool_jobs() { return 0 fi local _ptc_tmp_file - _ptc_tmp_file="$(mktemp -t _ptc_tool_jobs_live.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_tool_jobs_live.XXXXXX)" + _ptc_tmp_file="$(_ptc_mktemp _ptc_tool_jobs_live)" while IFS= read -r _ptc_pid; do [ -n "$_ptc_pid" ] || continue if kill -0 "$_ptc_pid" 2>/dev/null; then @@ -330,6 +375,17 @@ _ptc_maybe_emit_pending() { trap - DEBUG EXIT exit 1 fi + # Native BYOM workers use this private execution-scoped control file so a + # large stdout stream cannot truncate away the replay frame. Other + # backends continue to consume the stdout sentinel below. + if [ -n "$_PTC_CONTROL_PATH" ]; then + printf '%s' "$_ptc_payload" > "$_PTC_CONTROL_PATH" || { + printf 'failed to persist pending PTC tool calls\n' >&2 + _ptc_cleanup_tempfiles + trap - DEBUG EXIT + exit 1 + } + fi if [ "\${BASH_SUBSHELL:-0}" -eq 1 ]; then trap - DEBUG EXIT exit 0 @@ -512,7 +568,7 @@ _ptc_call_tool() { # Large input can exceed ARG_MAX via --argjson; write once, reuse path below. local _ptc_input_tmp - _ptc_input_tmp="$(mktemp -t _ptc_input.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_input.XXXXXX)" + _ptc_input_tmp="$(_ptc_mktemp _ptc_input)" printf '%s' "$_ptc_input" > "$_ptc_input_tmp" local _ptc_matches @@ -668,8 +724,12 @@ exit $_ptc_user_exit_code function generateBashToolStub(tool: LCTool): string { const fnName = normalizeBashFunctionName(tool.name); - const desc = (tool.description ?? '').split('\n').map(l => `# ${l}`).join('\n'); - const nameComment = fnName !== tool.name ? `# Original tool name: ${tool.name}\n` : ''; + const desc = (tool.description ?? '') + .split('\n') + .map(l => `# ${l}`) + .join('\n'); + const nameComment = + fnName !== tool.name ? `# Original tool name: ${tool.name}\n` : ''; const escapedToolName = escapeForBashDoubleQuote(tool.name); return `${nameComment}${desc ? desc + '\n' : ''}${fnName}() { local _default_input='{}' @@ -683,7 +743,8 @@ function generateBashToolStub(tool: LCTool): string { } function generateBashPendingDeferHelper(tools: readonly LCTool[]): string { - const toolNamesPattern = tools + const toolNamesPattern = + tools .map(tool => normalizeBashFunctionName(tool.name)) .map(escapeForBashEre) .join('|') || 'a^'; diff --git a/service/src/preamble.test.ts b/service/src/preamble.test.ts index bf50e3ec..803e1839 100644 --- a/service/src/preamble.test.ts +++ b/service/src/preamble.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { buildScopedSentinel, createProgrammaticPayload, + extractPendingFromControlPayload, extractPendingFromStdout, generatePreamble, } from './preamble'; @@ -33,7 +34,9 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { expect(preamble).toMatch(/AF_UNIX/); expect(preamble).toMatch(/\.connect\(_TOOL_CALL_SOCKET\)/); /* Regression guard against reintroducing the user-spoofable check. */ - expect(preamble).not.toMatch(/if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/); + expect(preamble).not.toMatch( + /if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/, + ); }); test('caches the probe verdict at module load (before user code can plant a spoof)', () => { @@ -41,10 +44,14 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { /* The probe call must appear at top level of the preamble, NOT * inside _do_request. Otherwise a user could plant a regular file * at the path between calls and flip the gate per-request. */ - const probeCallIdx = preamble.indexOf('_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()'); + const probeCallIdx = preamble.indexOf( + '_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()', + ); expect(probeCallIdx).toBeGreaterThan(-1); /* _do_request must consult the cached verdict, not re-probe. */ - const doReqMatch = preamble.match(/def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/); + const doReqMatch = preamble.match( + /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, + ); expect(doReqMatch).not.toBeNull(); expect(doReqMatch![0]).toContain('_USE_TOOL_CALL_SOCKET'); expect(doReqMatch![0]).not.toContain('_probe_tool_call_socket('); @@ -55,7 +62,9 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { /* The fallback must still construct the URL from _CALLBACK_URL and * delegate to _tcp_request. Without this, runners without the * proxy bind-mount would have no way to reach the orchestrator. */ - const doReqMatch = preamble.match(/def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/); + const doReqMatch = preamble.match( + /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, + ); expect(doReqMatch).not.toBeNull(); expect(doReqMatch![0]).toContain('_CALLBACK_URL + path'); expect(doReqMatch![0]).toContain('_tcp_request('); @@ -67,24 +76,47 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { * exported. The path must remain hardcoded so the preamble does * not depend on env-var injection. */ expect(preamble).toContain('_TOOL_CALL_SOCKET = "/tmp/tcs.sock"'); - expect(preamble).not.toMatch(/os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/); + expect(preamble).not.toMatch( + /os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/, + ); expect(preamble).not.toMatch(/os\.environ\[['"]TOOL_CALL_SOCKET['"]\]/); }); }); describe('extractPendingFromStdout — input hash metadata', () => { + test('normalizes native control payload hashes instead of trusting the sandbox', () => { + const forgedHash = hashToolInput({ resource: 'B' }); + const expectedHash = hashToolInput({ resource: 'A' }); + const pending = extractPendingFromControlPayload( + JSON.stringify({ + pending: [ + { + call_id: 'call_001', + tool_name: 'authorize', + input: { resource: 'A' }, + input_hash: forgedHash, + }, + ], + }), + ); + expect(pending?.[0]?.input_hash).toBe(expectedHash); + expect(pending?.[0]?.input_hash).not.toBe(forgedHash); + }); + test('ignores sandbox-supplied input_hash and uses the parsed input hash', () => { const executionId = 'exec_hash_guard'; const { start, end } = buildScopedSentinel(executionId); const forgedHash = hashToolInput({ resource: 'B' }); const expectedHash = hashToolInput({ resource: 'A' }); const payload = { - pending: [{ + pending: [ + { call_id: 'call_001', tool_name: 'authorize', input: { resource: 'A' }, input_hash: forgedHash, - }], + }, + ], }; const parsed = extractPendingFromStdout( diff --git a/service/src/preamble.ts b/service/src/preamble.ts index 9ff75dc7..94685c6c 100644 --- a/service/src/preamble.ts +++ b/service/src/preamble.ts @@ -2,7 +2,10 @@ import fs from 'fs'; import path from 'path'; import type * as t from './types'; import { planLimits } from './config'; -import { generateBashReplayPreamble, generateBashReplayPostamble } from './preamble-bash'; +import { + generateBashReplayPreamble, + generateBashReplayPostamble, +} from './preamble-bash'; import { PTC_HISTORY_FILENAME, PTC_HISTORY_SANDBOX_PATH, @@ -11,10 +14,16 @@ import { buildScopedSentinel, isReservedPtcFilename, } from './ptc-constants'; -import { hashToolInput, pendingInputHashesFromRawPayload } from './tool-input-signature'; +import { + hashToolInput, + pendingInputHashesFromRawPayload, +} from './tool-input-signature'; // Load async matplotlib template for programmatic tool calling -const templateCodeAsync = fs.readFileSync(path.join(__dirname, 'matplotlib-async.py'), 'utf8'); +const templateCodeAsync = fs.readFileSync( + path.join(__dirname, 'matplotlib-async.py'), + 'utf8', +); // ============================================================================= // Programmatic Tool Calling Types & Preamble Generation @@ -88,11 +97,41 @@ function normalizePythonFunctionName(name: string): string { // Python keywords to avoid const pythonKeywords = new Set([ - 'False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', - 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', - 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', - 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', - 'try', 'while', 'with', 'yield' + 'False', + 'None', + 'True', + 'and', + 'as', + 'assert', + 'async', + 'await', + 'break', + 'class', + 'continue', + 'def', + 'del', + 'elif', + 'else', + 'except', + 'finally', + 'for', + 'from', + 'global', + 'if', + 'import', + 'in', + 'is', + 'lambda', + 'nonlocal', + 'not', + 'or', + 'pass', + 'raise', + 'return', + 'try', + 'while', + 'with', + 'yield', ]); if (pythonKeywords.has(normalized)) { @@ -137,11 +176,14 @@ function jsonSchemaToPythonType(schema: JsonSchemaProperty): string { * Sort property names so required parameters come before optional ones. * Uses a Set for O(1) lookups instead of repeated array includes() calls. */ -function getSortedPropertyNames(propertyNames: string[], required: string[]): string[] { +function getSortedPropertyNames( + propertyNames: string[], + required: string[], +): string[] { const requiredSet = new Set(required); return [ ...propertyNames.filter(name => requiredSet.has(name)), - ...propertyNames.filter(name => !requiredSet.has(name)) + ...propertyNames.filter(name => !requiredSet.has(name)), ]; } @@ -155,7 +197,10 @@ function schemaToParams(schema?: JsonSchema): string { const required = schema.required ?? []; const requiredSet = new Set(required); - const sortedNames = getSortedPropertyNames(Object.keys(schema.properties), required); + const sortedNames = getSortedPropertyNames( + Object.keys(schema.properties), + required, + ); const params: string[] = []; @@ -192,7 +237,11 @@ function inferReturnType(description?: string): string { const desc = description.toLowerCase(); - if (desc.includes('returns list') || desc.includes('returns array') || desc.includes('list of')) { + if ( + desc.includes('returns list') || + desc.includes('returns array') || + desc.includes('list of') + ) { return 'List[Dict[str, Any]]'; } if (desc.includes('returns dict') || desc.includes('returns object')) { @@ -225,7 +274,10 @@ function generateDocstring(tool: LCTool): string { doc += '\n\n Parameters:'; const required = tool.parameters.required ?? []; const requiredSet = new Set(required); - const sortedNames = getSortedPropertyNames(Object.keys(tool.parameters.properties), required); + const sortedNames = getSortedPropertyNames( + Object.keys(tool.parameters.properties), + required, + ); for (const name of sortedNames) { const propSchema = tool.parameters.properties[name]; @@ -253,7 +305,8 @@ function generateToolStub(tool: LCTool): string { const pythonFunctionName = normalizePythonFunctionName(tool.name); // If name was changed, add a comment - const nameComment = pythonFunctionName !== tool.name + const nameComment = + pythonFunctionName !== tool.name ? ` # Original tool name: ${tool.name}\n` : ''; @@ -451,7 +504,8 @@ async def _execute_tool_internal_async(tool_name: str, tool_input: Dict[str, Any */ export function generateReplayPreamble(config: ReplayPreambleConfig): string { const { executionId, tools } = config; - const { start: scopedStart, end: scopedEnd } = buildScopedSentinel(executionId); + const { start: scopedStart, end: scopedEnd } = + buildScopedSentinel(executionId); let preamble = ` # ============================================================================ @@ -564,6 +618,63 @@ export interface ExtractPendingResult { }> | null; } +export function extractPendingFromControlPayload( + rawPayload: string, +): ExtractPendingResult['pending'] { + let parsed: { pending?: unknown } | null = null; + try { + parsed = JSON.parse(rawPayload) as { pending?: unknown }; + } catch { + return null; + } + + const pendingField = parsed?.pending; + if (!Array.isArray(pendingField)) return null; + + const rawInputHashes = pendingInputHashesFromRawPayload(rawPayload); + type PendingWithIndex = { + c: { call_id: string; tool_name: string; input: unknown }; + index: number; + }; + const isPendingWithIndex = (entry: { + c: unknown; + index: number; + }): entry is PendingWithIndex => { + const { c } = entry; + return ( + c != null && + typeof c === 'object' && + typeof (c as { call_id?: unknown }).call_id === 'string' && + typeof (c as { tool_name?: unknown }).tool_name === 'string' + ); + }; + return pendingField + .map((c, index) => ({ c, index })) + .filter(isPendingWithIndex) + .map(({ c, index }) => { + const callSite = (c as { call_site?: unknown }).call_site; + const rawInputHash = rawInputHashes[index]; + const hasObjectInput = + c.input != null && typeof c.input === 'object'; + const input = (hasObjectInput ? c.input : {}) as Record< + string, + unknown + >; + return { + call_id: c.call_id, + tool_name: c.tool_name, + input, + input_hash: + hasObjectInput && typeof rawInputHash === 'string' + ? rawInputHash + : hashToolInput(input), + ...(typeof callSite === 'string' + ? { call_site: callSite } + : {}), + }; + }); +} + /** * Locate the last line whose trimmed content exactly equals `marker`. * Using full-line anchoring prevents user-provided tool payloads that happen @@ -581,7 +692,8 @@ function findSentinelLine( for (let i = lines.length - 1; i >= searchFromLine; i--) { if (lines[i].trim() === marker) { const startOffset = lineStartOffsets[i]; - const endOffset = i + 1 < lineStartOffsets.length + const endOffset = + i + 1 < lineStartOffsets.length ? lineStartOffsets[i + 1] - 1 : startOffset + lines[i].length; return { line: i, startOffset, endOffset }; @@ -619,48 +731,8 @@ export function extractPendingFromStdout( const payloadLines = lines.slice(startLine.line + 1, endLine.line); const rawPayload = payloadLines.join('\n').trim(); - let parsed: { pending?: unknown } | null = null; - try { - parsed = JSON.parse(rawPayload) as { pending?: unknown }; - } catch { - return { stdout, pending: null }; - } - - const pendingField = parsed?.pending; - if (!Array.isArray(pendingField)) return { stdout, pending: null }; - - const rawInputHashes = pendingInputHashesFromRawPayload(rawPayload); - type PendingWithIndex = { - c: { call_id: string; tool_name: string; input: unknown }; - index: number; - }; - const isPendingWithIndex = (entry: { c: unknown; index: number }): entry is PendingWithIndex => { - const { c } = entry; - return ( - c != null && - typeof c === 'object' && - typeof (c as { call_id?: unknown }).call_id === 'string' && - typeof (c as { tool_name?: unknown }).tool_name === 'string' - ); - }; - const pending = pendingField - .map((c, index) => ({ c, index })) - .filter(isPendingWithIndex) - .map(({ c, index }) => { - const callSite = (c as { call_site?: unknown }).call_site; - const rawInputHash = rawInputHashes[index]; - const hasObjectInput = c.input != null && typeof c.input === 'object'; - const input = (hasObjectInput ? c.input : {}) as Record; - return { - call_id: c.call_id, - tool_name: c.tool_name, - input, - input_hash: hasObjectInput && typeof rawInputHash === 'string' - ? rawInputHash - : hashToolInput(input), - ...(typeof callSite === 'string' ? { call_site: callSite } : {}), - }; - }); + const pending = extractPendingFromControlPayload(rawPayload); + if (pending == null) return { stdout, pending: null }; /** Strip only the sentinel block and leave every other byte of user * stdout untouched. Both the Python and bash preambles defensively @@ -674,7 +746,10 @@ export function extractPendingFromStdout( * emission, and anything else that depends on byte-accurate stdout. */ const rawHead = stdout.slice(0, startLine.startOffset); const head = rawHead.endsWith('\n') ? rawHead.slice(0, -1) : rawHead; - const tailStart = endLine.endOffset < stdout.length ? endLine.endOffset + 1 : stdout.length; + const tailStart = + endLine.endOffset < stdout.length + ? endLine.endOffset + 1 + : stdout.length; const tail = stdout.slice(tailStart); const cleaned = head + tail; @@ -688,11 +763,14 @@ export function extractPendingFromStdout( function wrapUserCodeInAsync(userCode: string): string { const lines = userCode.split('\n'); - let wrapped = '# ============================================================================\n'; + let wrapped = + '# ============================================================================\n'; wrapped += '# USER CODE BEGINS BELOW\n'; - wrapped += '# ============================================================================\n\n'; + wrapped += + '# ============================================================================\n\n'; wrapped += 'async def __user_main__():\n'; - wrapped += ' """Auto-generated wrapper for user code to support top-level await"""\n'; + wrapped += + ' """Auto-generated wrapper for user code to support top-level await"""\n'; // Indent all user code for (const line of lines) { @@ -743,10 +821,21 @@ const PROGRAMMATIC_RUN_TIMEOUT = 300000; // 5 minutes wall time * Create a payload for programmatic tool calling execution * Combines the tool preamble with user code */ -export function createProgrammaticPayload(options: CreateProgrammaticPayloadOptions): t.PayloadBody { +export function createProgrammaticPayload( + options: CreateProgrammaticPayloadOptions, +): t.PayloadBody { const { - req, session_id, execution_id, callbackUrl, callbackToken, tools, timeout, - mode = 'blocking', history, codeOverride, filesOverride, + req, + session_id, + execution_id, + callbackUrl, + callbackToken, + tools, + timeout, + mode = 'blocking', + history, + codeOverride, + filesOverride, language = 'python', } = options; const body = req.body as t.ProgrammaticRequestBody; @@ -762,7 +851,14 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti throw new Error('bash PTC is only supported in replay mode'); } return buildBashPayload({ - req, execution_id, session_id, tools, userCode, files, history, timeout, + req, + execution_id, + session_id, + tools, + userCode, + files, + history, + timeout, }); } @@ -771,7 +867,9 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti preamble = generateReplayPreamble({ executionId: execution_id, tools }); } else { if (!callbackUrl || !callbackToken) { - throw new Error('blocking PTC mode requires callbackUrl and callbackToken'); + throw new Error( + 'blocking PTC mode requires callbackUrl and callbackToken', + ); } preamble = generatePreamble({ callbackUrl, @@ -781,15 +879,21 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti }); } - const isPyPlot = userCode.includes('import matplotlib') || userCode.includes('import seaborn'); + const isPyPlot = + userCode.includes('import matplotlib') || + userCode.includes('import seaborn'); let finalCode: string; if (isPyPlot) { - const indentedUserCode = userCode.trim().split('\n').map(line => ` ${line}`).join('\n'); + const indentedUserCode = userCode + .trim() + .split('\n') + .map(line => ` ${line}`) + .join('\n'); const wrappedUserCode = templateCodeAsync.replace( /# BEGIN USER CODE\n[\s\S]*?# END USER CODE/, - `# BEGIN USER CODE\n${indentedUserCode}\n # END USER CODE` + `# BEGIN USER CODE\n${indentedUserCode}\n # END USER CODE`, ); finalCode = preamble + '\n' + wrappedUserCode; } else { @@ -797,7 +901,9 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti finalCode = preamble + wrappedUserCode; } - const run_memory_limit = planLimits[req.planId ?? '']?.run_memory_limit ?? planLimits.default.run_memory_limit; + const run_memory_limit = + planLimits[req.planId ?? '']?.run_memory_limit ?? + planLimits.default.run_memory_limit; const run_timeout = timeout ?? PROGRAMMATIC_RUN_TIMEOUT; const payload: t.PayloadBody = { @@ -809,8 +915,8 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti files: [ { name: 'main.py', - content: finalCode - } + content: finalCode, + }, ], session_id, }; @@ -851,13 +957,27 @@ function buildBashPayload(args: { history?: Record; timeout?: number; }): t.PayloadBody { - const { req, execution_id, session_id, tools, userCode, files, history, timeout } = args; - - const preamble = generateBashReplayPreamble({ executionId: execution_id, tools }); + const { + req, + execution_id, + session_id, + tools, + userCode, + files, + history, + timeout, + } = args; + + const preamble = generateBashReplayPreamble({ + executionId: execution_id, + tools, + }); const postamble = generateBashReplayPostamble(); const finalCode = preamble + userCode + '\n' + postamble; - const run_memory_limit = planLimits[req.planId ?? '']?.run_memory_limit ?? planLimits.default.run_memory_limit; + const run_memory_limit = + planLimits[req.planId ?? '']?.run_memory_limit ?? + planLimits.default.run_memory_limit; const run_timeout = timeout ?? PROGRAMMATIC_RUN_TIMEOUT; const payload: t.PayloadBody = { @@ -865,6 +985,8 @@ function buildBashPayload(args: { run_timeout, language: 'bash', version: '5.2.0', + execution_id, + replay_tool_count: tools.length, files: [ { name: 'main.sh', diff --git a/service/src/ptc-constants.test.ts b/service/src/ptc-constants.test.ts new file mode 100644 index 00000000..5028e385 --- /dev/null +++ b/service/src/ptc-constants.test.ts @@ -0,0 +1,9 @@ +import { expect, test } from 'bun:test'; +import { isReservedPtcFilename } from './ptc-constants'; + +test('reserves replay inputs and output control channels after normalization', () => { + for (const name of ['_ptc_history.json', '_ptc_pending_result.json', '_PTC_PENDING_RESULT.JSON', 'sub/../_ptc_pending_result.json', 'sub\\_ptc_pending_result.json']) { + expect(isReservedPtcFilename(name)).toBe(true); + } + expect(isReservedPtcFilename('_ptc_data.csv')).toBe(false); +}); diff --git a/service/src/ptc-constants.ts b/service/src/ptc-constants.ts index 5009c3c3..b7b3999b 100644 --- a/service/src/ptc-constants.ts +++ b/service/src/ptc-constants.ts @@ -15,8 +15,9 @@ export const PTC_HISTORY_SANDBOX_PATH = `/mnt/data/${PTC_HISTORY_FILENAME}`; * Returns `true` for any filename the submission layer must refuse. * * Two things make a name "reserved": - * 1. Its post-normalization basename is `_ptc_history.json` — the single - * runtime fixture the replay preamble injects into the submission dir. + * 1. Its post-normalization basename is `_ptc_history.json` or + * `_ptc_pending_result.json`, compared case-insensitively for macOS. + * These are the replay input and output control channels. * Any user-supplied file with that exact basename would shadow our * injected history and silently corrupt replay correctness, so we * reject it on the request path. The bash preamble's `_ptc_pending.*` @@ -61,7 +62,7 @@ export function isReservedPtcFilename(name: string): boolean { } if (escapes) return true; const basename = segments.length > 0 ? segments[segments.length - 1] : ''; - return basename === PTC_HISTORY_FILENAME; + return [PTC_HISTORY_FILENAME, '_ptc_pending_result.json'].includes(basename.toLowerCase()); } /** diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index a65bc89e..697271ee 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -62,6 +62,34 @@ describe('RemoteBridgeSandboxBackend', () => { }); }); + test('preserves an authenticated selected workspace on remote dispatch', async () => { + let dispatched: Parameters[0] | undefined; + const store = { + dispatch: async ( + args: Parameters[0], + ): ReturnType => { + dispatched = args; + return { + protocolVersion: 1 as const, + generation: 1, + leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', + status: 'fulfilled' as const, + result: { session_id: 'session-1', language: 'bash', version: '5.2', files: [] }, + }; + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + + await backend.execute(request(), { ...context(), workspaceId: 'project-a' }); + + expect(dispatched).toMatchObject({ + workerId: 'user-vm', + workspaceId: 'project-a', + requireTenantBinding: true, + }); + }); + test('maps tenant authorization rejection to a bridge backend error', async () => { const store = { dispatch: async (): ReturnType => { diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 0bee0038..6e06eda8 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -60,6 +60,7 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { (this.dynamicWorkers || ctx.bridgeWorkerId !== this.workerId), body: req.body, headers: req.headers, + ...(ctx.workspaceId != null ? { workspaceId: ctx.workspaceId } : {}), runtimeSessionId: ctx.runtimeSessionId, deadlineAtMs: ctx.deadlineAtMs ?? Date.now() + env.JOB_TIMEOUT, signal: ctx.signal, diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index fbaa2d20..e21982d9 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -39,6 +39,8 @@ export interface SandboxExecuteContext { canonicalUserId?: string; /** Trusted API-selected outbound worker. Presence requires a tenant-bound credential. */ bridgeWorkerId?: string; + /** Trusted selected workspace for native replay-mode PTC. */ + workspaceId?: string; /** Stable identifier for this queued iteration, used to derive an idempotent * stateless launch token. PTC replay reuses one executionId across every * iteration, so the executionId alone cannot separate them; the request body @@ -63,6 +65,7 @@ export type SandboxRawResponse = t.ExecuteResponse & { session_id: string; files?: t.FileRefs; run?: t.ExecuteResponse['run']; + pending_tool_calls_payload?: string; }; export interface SandboxBackend { diff --git a/service/src/sandbox-dispatch.test.ts b/service/src/sandbox-dispatch.test.ts index 435be9aa..5b1ef749 100644 --- a/service/src/sandbox-dispatch.test.ts +++ b/service/src/sandbox-dispatch.test.ts @@ -12,8 +12,10 @@ import { } from './execution-manifest'; const SECRET = 'test-secret'; -const PRIVATE_KEY = 'MC4CAQAwBQYDK2VwBCIEIBoxzSJjQ5jTVyuohHtlD+uDGqv/tZ6hQS2CmxuOg2Wn'; -const PUBLIC_KEY = 'MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U='; +const PRIVATE_KEY = + 'MC4CAQAwBQYDK2VwBCIEIBoxzSJjQ5jTVyuohHtlD+uDGqv/tZ6hQS2CmxuOg2Wn'; +const PUBLIC_KEY = + 'MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U='; function payload(overrides: Partial = {}): t.PayloadBody { return { @@ -25,14 +27,22 @@ function payload(overrides: Partial = {}): t.PayloadBody { }; } -function claims(overrides: Partial = {}): ExecutionManifestClaims { +function claims( + overrides: Partial = {}, +): ExecutionManifestClaims { return { v: EXECUTION_MANIFEST_VERSION, exec_id: 'exec_123', tenant_id: 'tenant_abc', user_id: 'user_123', session_key: 'tenant:tenant_abc:user:user_123', - input_files: [{ id: 'file_123', session_id: 'sess_input', name: 'inputs/data.csv' }], + input_files: [ + { + id: 'file_123', + session_id: 'sess_input', + name: 'inputs/data.csv', + }, + ], read_sessions: ['sess_input'], output_session_id: 'sess_output', max_upload_bytes: 1024, @@ -46,6 +56,20 @@ function claims(overrides: Partial = {}): ExecutionMani } describe('sandbox execute request dispatch', () => { + test('budgets every input and output batch before signing the request', () => { + const request = buildSandboxExecuteRequest({ + payload: payload({ files: Array.from({ length: 9 }, (_, index) => ({ name: `${index}.txt`, id: `file_${index}`, storage_session_id: 'input' })) }), + programmaticTransferReserveMs: 60_000, + executionManifestClaims: claims({ max_output_files: 10 }), + executionManifestSecret: SECRET, + executionManifestTtlSeconds: 300, + nowSeconds: 1_000, + }); + // Three download batches plus three upload batches share one reserve. + expect(request.body.transfer_timeout_ms).toBe(10_000); + const verified = verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_000 }); + expect(verified.execute_body_sha256).toBe(executionManifestBodySha256(request.body)); + }); test('keeps large egress grants out of HTTP headers', () => { const largeGrant = `ceg1.${'a'.repeat(24_000)}`; const request = buildSandboxExecuteRequest({ @@ -64,18 +88,27 @@ describe('sandbox execute request dispatch', () => { const request = buildSandboxExecuteRequest({ payload: payload(), executionManifestClaims: claims(), + maxOutputFileBytes: 1_000, executionManifestSecret: SECRET, executionManifestTtlSeconds: 300, nowSeconds: 1_000, }); expect(request.headers[EXECUTION_MANIFEST_HEADER]).toBeUndefined(); + expect(request.body.max_output_files).toBe(10); + expect(request.body.max_output_file_bytes).toBe(1_000); expect(request.body.execution_manifest).toEqual(expect.any(String)); - expect(verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifest(request.body.execution_manifest!, SECRET, { + nowSeconds: 1_100, + }), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); test('signs execution manifests with a private key when configured', () => { @@ -88,11 +121,19 @@ describe('sandbox execute request dispatch', () => { nowSeconds: 1_000, }); - expect(verifyExecutionManifestWithPublicKey(request.body.execution_manifest!, PUBLIC_KEY, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifestWithPublicKey( + request.body.execution_manifest!, + PUBLIC_KEY, + { nowSeconds: 1_100 }, + ), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); test('binds body-carried egress grants into signed execution manifests', () => { @@ -106,10 +147,16 @@ describe('sandbox execute request dispatch', () => { }); expect(request.body.egress_grant).toBe('ceg1.sealed-grant'); - expect(verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifest(request.body.execution_manifest!, SECRET, { + nowSeconds: 1_100, + }), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); }); diff --git a/service/src/sandbox-dispatch.ts b/service/src/sandbox-dispatch.ts index e3066905..340834b5 100644 --- a/service/src/sandbox-dispatch.ts +++ b/service/src/sandbox-dispatch.ts @@ -1,5 +1,14 @@ import type * as t from './types'; -import { executionManifestBodySha256, signExecutionManifestWithKey, type ExecutionManifestClaims } from './execution-manifest'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, +} from '../../packages/code/src/protocol'; +import { + executionManifestBodySha256, + signExecutionManifestWithKey, + type ExecutionManifestClaims, +} from './execution-manifest'; interface BuildSandboxExecuteRequestArgs { payload: t.PayloadBody; @@ -9,6 +18,8 @@ interface BuildSandboxExecuteRequestArgs { executionManifestSecret: string; executionManifestTtlSeconds: number; nowSeconds?: number; + maxOutputFileBytes?: number; + programmaticTransferReserveMs?: number; } interface SandboxExecuteRequest { @@ -21,15 +32,31 @@ interface SandboxExecuteRequest { * ride in the JSON body instead of HTTP headers. Otherwise skill-heavy jobs can * fail with 431 before sandbox-runner reaches capability validation. */ -export function buildSandboxExecuteRequest(args: BuildSandboxExecuteRequestArgs): SandboxExecuteRequest { +export function buildSandboxExecuteRequest( + args: BuildSandboxExecuteRequestArgs, +): SandboxExecuteRequest { const body: t.PayloadBody = { ...args.payload }; - const headers: Record = { 'Content-Type': 'application/json' }; + const headers: Record = { + 'Content-Type': 'application/json', + }; if (args.egressGrantToken) { body.egress_grant = args.egressGrantToken; } + if (args.maxOutputFileBytes != null) { + body.max_output_file_bytes = args.maxOutputFileBytes; + } + if (args.programmaticTransferReserveMs != null) { + const batches = Math.max(1, + Math.ceil(body.files.filter(file => 'id' in file).length / BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY) + + Math.ceil((args.executionManifestClaims?.max_output_files ?? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY), + ); + body.transfer_timeout_ms = Math.min(BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, + Math.max(1, Math.floor(args.programmaticTransferReserveMs / batches))); + } if (args.executionManifestClaims) { + body.max_output_files = args.executionManifestClaims.max_output_files; const nowSeconds = args.nowSeconds ?? Math.floor(Date.now() / 1000); body.execution_manifest = signExecutionManifestWithKey( { diff --git a/service/src/sandbox-egress.ts b/service/src/sandbox-egress.ts index f6549c09..e47806a2 100644 --- a/service/src/sandbox-egress.ts +++ b/service/src/sandbox-egress.ts @@ -3,6 +3,7 @@ import { env } from './config'; import { createGatewayPtcCallbackToken } from './egress-gateway-client'; import type { ExecutionManifestClaims } from './execution-manifest'; import type * as t from './types'; +import { programmaticTransferReserveMs } from '../../packages/code/src/protocol'; export type SandboxJobSecurity = { payload: t.PayloadBody; @@ -65,6 +66,9 @@ export function timeoutMsToGrantSeconds(timeoutMs: number): number { } const DEFAULT_PROGRAMMATIC_TIMEOUT_MS = 300000; +const SELECTED_WORKSPACE_REPLAY_PASSES = 2; +const SELECTED_WORKSPACE_SETTLEMENT_RESERVE_MS = 5_000; +const SELECTED_WORKSPACE_MAX_QUEUE_RESERVE_MS = 30_000; export function normalizeProgrammaticTimeoutMs( rawTimeout: unknown, @@ -80,6 +84,31 @@ export function normalizeProgrammaticTimeoutMs( return Math.min(Math.ceil(rawTimeout), maxTimeout); } +/** + * Selected-workspace Bash replay may run one read-only probe and one commit + * pass in its final iteration. Bound each pass so both plus settlement reserve + * fit inside the worker-owned JOB_TIMEOUT instead of advertising a duration + * the assignment cannot complete. + */ +export function normalizeSelectedWorkspaceProgrammaticTimeoutMs( + rawTimeout: unknown, + jobTimeoutMs = env.JOB_TIMEOUT, +): number { + const totalBudget = Math.max(1, Math.floor(jobTimeoutMs)); + const queueReserve = Math.min( + SELECTED_WORKSPACE_MAX_QUEUE_RESERVE_MS, + Math.floor(totalBudget / 5), + ); + const executionBudget = Math.max( + 1, + totalBudget - queueReserve - SELECTED_WORKSPACE_SETTLEMENT_RESERVE_MS - programmaticTransferReserveMs(totalBudget), + ); + return normalizeProgrammaticTimeoutMs( + rawTimeout, + Math.max(1, Math.floor(executionBudget / SELECTED_WORKSPACE_REPLAY_PASSES)), + ); +} + export async function sealPtcCallbackTokenForGateway(args: { executionId: string; sessionId: string; diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 9896518c..4de4a4d8 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -11,12 +11,19 @@ import { connection, getExecutionQueueBinding, } from '../queue'; -import { createProgrammaticPayload, extractPendingFromStdout } from '../preamble'; +import { + createProgrammaticPayload, + extractPendingFromControlPayload, + extractPendingFromStdout, +} from '../preamble'; import { findBashToolNameCollision } from '../preamble-bash'; import type { LCTool } from '../preamble'; import { isReservedPtcFilename } from '../ptc-constants'; import { internalServiceHeaders } from '../internal-service-auth'; -import { resolveOutputBucketSessionKey, SessionKeyResolutionError } from '../session-key'; +import { + resolveOutputBucketSessionKey, + SessionKeyResolutionError, +} from '../session-key'; import { getCredentialId, getPrincipalOrReject } from '../auth/principal'; import { getExecutionIdentity } from '../execution-identity'; import { PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION } from '../runtime-session/job-policy'; @@ -34,15 +41,25 @@ import { publicExecutionFailure } from '../utils'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, + normalizeSelectedWorkspaceProgrammaticTimeoutMs, prepareSandboxJobSecurity, sealPtcCallbackTokenForGateway, timeoutMsToGrantSeconds, } from '../sandbox-egress'; import { findUnregisteredToolCall } from '../tool-scope'; import { summarizeRequestedFiles } from '../execution-log'; -import { pollBlockingExecution, type BlockingPendingState } from './blocking-poll'; -import { clearSessionOwnership, recordSessionOwnership } from '../session-ownership'; -import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; +import { + pollBlockingExecution, + type BlockingPendingState, +} from './blocking-poll'; +import { + clearSessionOwnership, + recordSessionOwnership, +} from '../session-ownership'; +import { + FileRefAuthorizationError, + authorizeRequestedFiles, +} from './file-authorization'; import { buildReplayExecutionState, resolveReplayStateSandboxBackend, @@ -50,8 +67,10 @@ import { import { BridgeWorkerSelectionError, CODEAPI_BRIDGE_WORKER_HEADER, + CODEAPI_BRIDGE_WORKSPACE_HEADER, resolveBridgeWorkerSelection, } from '../bridge/selection'; +import { isValidBridgeWorkerId, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES } from '../../../packages/code/src/protocol'; import logger from '../logger'; import { type ExecutionState, @@ -145,26 +164,41 @@ async function retryToolCallServerRequest( ): Promise { let lastError: Error | undefined; - for (let attempt = 1; attempt <= TOOL_CALL_SERVER_RETRY_ATTEMPTS; attempt++) { + for ( + let attempt = 1; + attempt <= TOOL_CALL_SERVER_RETRY_ATTEMPTS; + attempt++ + ) { try { return await requestFn(); } catch (error) { lastError = error as Error; if (axios.isAxiosError(error)) { - if (error.response && error.response.status >= 400 && error.response.status < 500) { + if ( + error.response && + error.response.status >= 400 && + error.response.status < 500 + ) { throw error; } } if (attempt < TOOL_CALL_SERVER_RETRY_ATTEMPTS) { - logger.warn(`${context} failed (attempt ${attempt}/${TOOL_CALL_SERVER_RETRY_ATTEMPTS}), retrying...`, { + logger.warn( + `${context} failed (attempt ${attempt}/${TOOL_CALL_SERVER_RETRY_ATTEMPTS}), retrying...`, + { error: lastError.message, - }); - await new Promise(resolve => setTimeout(resolve, TOOL_CALL_SERVER_RETRY_DELAY * attempt)); + }, + ); + await new Promise(resolve => + setTimeout(resolve, TOOL_CALL_SERVER_RETRY_DELAY * attempt), + ); } } } - logger.error(`${context} failed after ${TOOL_CALL_SERVER_RETRY_ATTEMPTS} attempts`); + logger.error( + `${context} failed after ${TOOL_CALL_SERVER_RETRY_ATTEMPTS} attempts`, + ); throw lastError; } @@ -179,7 +213,9 @@ setInterval(() => { }, STALE_CLEANUP_INTERVAL_MS); function generateContinuationToken(execution_id: string): string { - return Buffer.from(JSON.stringify({ execution_id, ts: Date.now() })).toString('base64'); + return Buffer.from( + JSON.stringify({ execution_id, ts: Date.now() }), + ).toString('base64'); } /** Map a replay-continuation HTTP status to its operational outcome @@ -200,14 +236,21 @@ function classifyContinuationOutcome(statusCode: number): string { * timestamp is older than the execution-state TTL — without this, the * `ts` field was dead data and a client could replay an ancient token * against a freshly-reused-execution-id window. */ -function decodeContinuationToken(token: string): { execution_id: string } | null { +function decodeContinuationToken( + token: string, +): { execution_id: string } | null { try { - const parsed: unknown = JSON.parse(Buffer.from(token, 'base64').toString('utf-8')); + const parsed: unknown = JSON.parse( + Buffer.from(token, 'base64').toString('utf-8'), + ); if (parsed === null || typeof parsed !== 'object') { return null; } const candidate = parsed as { execution_id?: unknown; ts?: unknown }; - if (typeof candidate.execution_id !== 'string' || candidate.execution_id.length === 0) { + if ( + typeof candidate.execution_id !== 'string' || + candidate.execution_id.length === 0 + ) { return null; } if (typeof candidate.ts === 'number' && Number.isFinite(candidate.ts)) { @@ -226,13 +269,17 @@ function decodeContinuationToken(token: string): { execution_id: string } | null // Blocking mode (legacy path) // --------------------------------------------------------------------------- -function waitForExecutionState(execution_id: string, timeout: number): ReturnType { +function waitForExecutionState( + execution_id: string, + timeout: number, +): ReturnType { return pollBlockingExecution(execution_id, timeout, { getExecutionState, getBlockingResult, - getPending: async (id) => { + getPending: async id => { const response = await retryToolCallServerRequest( - () => axios.get( + () => + axios.get( `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, { headers: internalServiceHeaders() }, ), @@ -240,7 +287,8 @@ function waitForExecutionState(execution_id: string, timeout: number): ReturnTyp ); return response.data; }, - isNotFound: (error) => axios.isAxiosError(error) && error.response?.status === 404, + isNotFound: error => + axios.isAxiosError(error) && error.response?.status === 404, sleep: () => new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)), now: Date.now, }); @@ -295,7 +343,8 @@ async function runReplayIteration( }); if (DEBUG_MODE) { - const firstFile = rawPayload.files[0] as { content?: string } | undefined; + const firstFile = rawPayload.files[0] as + { content?: string } | undefined; logger.debug('Replay enqueue details', { execution_id: state.execution_id, historySize: Object.keys(history).length, @@ -320,7 +369,9 @@ async function runReplayIteration( state.executionProfile ?? env.EXECUTION_PROFILE, state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, ); - const job = await queue.add(Jobs.execute, { + const job = await queue.add( + Jobs.execute, + { code: state.userCode ?? '', userId, payload: sandboxSecurity.payload, @@ -332,17 +383,24 @@ async function runReplayIteration( canonicalUserId: state.canonicalUserId, executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, sandboxBackend: replayBackend, - ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), + ...(state.bridgeWorkerId != null + ? { bridgeWorkerId: state.bridgeWorkerId } + : {}), + ...(state.workspaceId != null + ? { workspaceId: state.workspaceId } + : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, executionManifestClaims: sandboxSecurity.executionManifestClaims, egressGrantClaims: sandboxSecurity.egressGrantClaims, egressGrantToken: sandboxSecurity.egressGrantToken, - }, { + }, + { removeOnComplete: { age: 60, count: 1 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, - }); + }, + ); jobsSubmitted.inc({ language }); return job.waitUntilFinished(events, JOB_COMPLETION_WAIT_TIMEOUT_MS); @@ -365,18 +423,24 @@ async function handleReplayInitial( apiKeyId: string; userId: string; bridgeWorkerId?: string; + workspaceId?: string; }, ): Promise { - const { apiKeyId, userId, bridgeWorkerId } = params; - const { - code, - tools, - user_id, - files, - } = req.body as t.ProgrammaticRequestBody; + const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; + const { code, tools, user_id, files } = + req.body as t.ProgrammaticRequestBody; let timeout: number; try { - timeout = normalizeProgrammaticTimeoutMs((req.body as t.ProgrammaticRequestBody).timeout); + if (workspaceId != null && Array.isArray(files) && files.length > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES) { + throw new Error(`Selected-workspace execution allows at most ${BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES} input files; main and replay history occupy two reserved slots`); + } + timeout = workspaceId != null + ? normalizeSelectedWorkspaceProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ) + : normalizeProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ); } catch (error) { res.status(400).json({ error: (error as Error).message }); return; @@ -400,14 +464,23 @@ async function handleReplayInitial( }); return; } - const language: 'python' | 'bash' = requestedLanguage === 'bash' ? 'bash' : 'python'; + const language: 'python' | 'bash' = + requestedLanguage === 'bash' ? 'bash' : 'python'; + if (workspaceId != null && language !== 'bash') { + res.status(400).json({ + error: 'Selected-workspace programmatic execution supports bash only', + }); + return; + } if (!code) { res.status(400).json({ error: 'Missing required field: code' }); return; } - if (!tools || !Array.isArray(tools) || tools.length === 0) { - res.status(400).json({ error: 'Missing required field: tools (must be a non-empty array)' }); + if (!Array.isArray(tools) || (tools.length === 0 && workspaceId == null)) { + res.status(400).json({ + error: 'Missing required field: tools (must be non-empty unless a selected workspace executes bash)', + }); return; } if (tools.length > MAX_TOOLS_PER_REQUEST) { @@ -442,7 +515,8 @@ async function handleReplayInitial( files, store: connection, }); - (req.body as t.ProgrammaticRequestBody).files = authorizedFiles.length > 0 ? authorizedFiles : undefined; + (req.body as t.ProgrammaticRequestBody).files = + authorizedFiles.length > 0 ? authorizedFiles : undefined; } catch (error) { if (sendFileRefAuthorizationError(error, res, req)) return; logger.error('Error authorizing replay file refs:', error); @@ -456,7 +530,14 @@ async function handleReplayInitial( try { sessionKey = resolveOutputBucketSessionKey(req); } catch (error) { - if (sendSessionKeyResolutionError(error, res, req, 'programmatic /exec: resolveOutputBucketSessionKey')) { + if ( + sendSessionKeyResolutionError( + error, + res, + req, + 'programmatic /exec: resolveOutputBucketSessionKey', + ) + ) { return; } throw error; @@ -466,9 +547,9 @@ async function handleReplayInitial( const execution_id = nanoid(); const authContext = req.codeApiAuthContext; const identity = getExecutionIdentity(req, userId); - const isPyPlot = language === 'python' && ( - code.includes('import matplotlib') || code.includes('import seaborn') - ); + const isPyPlot = + language === 'python' && + (code.includes('import matplotlib') || code.includes('import seaborn')); await recordSessionOwnership(connection, session_id, sessionKey); @@ -487,6 +568,7 @@ async function handleReplayInitial( timeout, language, bridgeWorkerId, + workspaceId, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -506,13 +588,16 @@ async function handleReplayInitial( await setExecutionState(state); } catch (err) { if (err instanceof ExecutionStateTooLargeError) { - logger.warn('Rejecting replay request: ExecutionState exceeds Redis cap', { + logger.warn( + 'Rejecting replay request: ExecutionState exceeds Redis cap', + { execution_id, userId, apiKeyId, bytes: err.bytes, cap: err.cap, - }); + }, + ); await clearSessionOwnership(connection, session_id).catch(() => {}); ptcReplayStateOversize.inc(); res.status(413).json({ @@ -568,9 +653,15 @@ async function handleReplayContinuation( * adds outcome plumbing through `runAndRespond`. */ const startMs = performance.now(); res.once('finish', () => { - const labels = { mode: 'replay' as const, outcome: classifyContinuationOutcome(res.statusCode) }; + const labels = { + mode: 'replay' as const, + outcome: classifyContinuationOutcome(res.statusCode), + }; ptcReplayContinuations.inc(labels); - ptcReplayContinuationDuration.observe(labels, (performance.now() - startMs) / 1000); + ptcReplayContinuationDuration.observe( + labels, + (performance.now() - startMs) / 1000, + ); }); /** Reject oversized batches before we spend any CPU on per-entry @@ -621,9 +712,14 @@ async function handleReplayContinuation( call_site: emitted.call_site, }; }); - const deltaOrError = await computeToolHistoryDelta(state.execution_id, enrichedResults); + const deltaOrError = await computeToolHistoryDelta( + state.execution_id, + enrichedResults, + ); if ('error' in deltaOrError) { - res.status(deltaOrError.status ?? 400).json({ error: deltaOrError.error }); + res.status(deltaOrError.status ?? 400).json({ + error: deltaOrError.error, + }); return; } const delta = deltaOrError; @@ -642,7 +738,9 @@ async function handleReplayContinuation( }); if (!pre.ok) { if (pre.status === 403) { - logger.warn('Unauthorized replay continuation request rejected', { + logger.warn( + 'Unauthorized replay continuation request rejected', + { execution_id: state.execution_id, requestUserId: userId, requestApiKeyId: apiKeyId, @@ -650,7 +748,8 @@ async function handleReplayContinuation( executionUserId: state.userId, executionApiKeyId: state.apiKeyId, executionTenantId: state.tenantId, - }); + }, + ); } if (pre.cleanupOnReject === true) { await cleanupExecution(state.execution_id, 'replay'); @@ -672,7 +771,10 @@ async function handleReplayContinuation( * Redis MULTI/EXEC so counters and the hash can't drift out of sync * on a partial failure. */ state.callCount = (state.callCount ?? 0) + delta.newCallIds.length; - state.historyBytes = Math.max(0, (state.historyBytes ?? 0) + delta.bytesDelta); + state.historyBytes = Math.max( + 0, + (state.historyBytes ?? 0) + delta.bytesDelta, + ); state.lastActivity = Date.now(); try { await commitToolHistoryAndState(state, delta); @@ -687,14 +789,19 @@ async function handleReplayContinuation( * forward is a fresh execution with smaller inputs. Reap the * old execution to free the lock and Redis keys, then return * an actionable 413 instead of a generic 500. */ - logger.warn('Replay continuation rejected: ExecutionState exceeds Redis cap', { + logger.warn( + 'Replay continuation rejected: ExecutionState exceeds Redis cap', + { execution_id: state.execution_id, bytes: err.bytes, cap: err.cap, callCount: state.callCount, historyBytes: state.historyBytes, - }); - await cleanupExecution(state.execution_id, 'replay').catch(() => {}); + }, + ); + await cleanupExecution(state.execution_id, 'replay').catch( + () => {}, + ); ptcReplayStateOversize.inc(); res.status(413).json({ status: 'error', @@ -715,10 +822,13 @@ async function handleReplayContinuation( * the throw bubble to the top-level catch and become an opaque * 500 — clients (and load balancers) treat 5xx classes very * differently for retry policy. */ - logger.error('Failed to commit replay continuation; returning retryable 503', { + logger.error( + 'Failed to commit replay continuation; returning retryable 503', + { execution_id: state.execution_id, err: (err as Error).message, - }); + }, + ); res.status(503).json({ status: 'error', error: 'Failed to persist replay continuation; please retry the same request', @@ -764,11 +874,15 @@ async function runAndRespond( try { result = await runReplayIteration(req, state, apiKeyId, userId); } catch (err) { - logger.error('Replay iteration failed', { execution_id: state.execution_id, err }); + logger.error('Replay iteration failed', { + execution_id: state.execution_id, + err, + }); await cleanupExecution(state.execution_id, 'replay'); if (!isDisconnected()) { const publicFailure = publicExecutionFailure(err); - const message = publicFailure?.body.message ?? (err as Error).message; + const message = + publicFailure?.body.message ?? (err as Error).message; res.status(200).json({ status: 'error', error: message !== '' ? message : 'Sandbox execution failed', @@ -786,10 +900,19 @@ async function runAndRespond( return; } - const { stdout: cleanStdout, pending } = extractPendingFromStdout( + const extracted = extractPendingFromStdout( result.stdout, state.execution_id, ); + const cleanStdout = extracted.stdout; + const controlPayload = result.pending_tool_calls_payload; + const hasControlPayload = typeof controlPayload === 'string'; + const controlPending = hasControlPayload + ? extractPendingFromControlPayload(controlPayload) + : null; + const pending = hasControlPayload + ? (controlPending ?? []) + : extracted.pending; if (pending != null) { if (pending.length === 0) { @@ -806,7 +929,10 @@ async function runAndRespond( }); return; } - const unregisteredToolCall = findUnregisteredToolCall(pending, state.tools); + const unregisteredToolCall = findUnregisteredToolCall( + pending, + state.tools, + ); if (unregisteredToolCall != null) { logger.warn('Sandbox requested unregistered replay tool call', { execution_id: state.execution_id, @@ -866,11 +992,16 @@ async function runAndRespond( await setExecutionState(state); await refreshExecutionTtl(state.execution_id); } catch (err) { - logger.error('Failed to persist execution state before continuation; aborting', { + logger.error( + 'Failed to persist execution state before continuation; aborting', + { execution_id: state.execution_id, err: (err as Error).message, - }); - await cleanupExecution(state.execution_id, 'replay').catch(() => {}); + }, + ); + await cleanupExecution(state.execution_id, 'replay').catch( + () => {}, + ); if (!isDisconnected()) { if (err instanceof ExecutionStateTooLargeError) { /** A continuation that pushes `emittedCallIds` past the @@ -887,8 +1018,7 @@ async function runAndRespond( } else { res.status(503).json({ status: 'error', - error: - 'Failed to persist replay state; please retry the request from scratch', + error: 'Failed to persist replay state; please retry the request from scratch', session_id: state.session_id, }); } @@ -912,7 +1042,8 @@ async function runAndRespond( if (!isSandboxRunSuccess(result)) { await cleanupExecution(state.execution_id, 'replay'); - const errorMessage = result.message != null && result.message !== '' + const errorMessage = + result.message != null && result.message !== '' ? result.message : `Sandbox exited with code ${result.code ?? 'unknown'}`; res.status(200).json({ @@ -941,7 +1072,10 @@ async function runAndRespond( // Request entrypoint // --------------------------------------------------------------------------- -router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedRequest, res) => { +router.post( + '/exec/programmatic', + executionLimiter, + async (req: t.AuthenticatedRequest, res) => { const principal = getPrincipalOrReject(req, res); if (!principal) return; const apiKeyId = getCredentialId(req); @@ -954,13 +1088,12 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR return res.status(503).json({ error: 'Service is starting up' }); } - const { - continuation_token, - tool_results, - } = req.body as t.ProgrammaticRequestBody; + const { continuation_token, tool_results } = + req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; + let workspaceId: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -970,12 +1103,35 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), trustedWorkerId: principal.codeWorkerId, }); - bridgeWorkerId = bridgeSelection?.explicit === true - ? bridgeSelection.workerId - : undefined; + bridgeWorkerId = + bridgeSelection?.explicit === true || + (bridgeSelection != null && !env.BRIDGE_DYNAMIC_WORKERS) + ? bridgeSelection.workerId + : undefined; + const requestedWorkspaceId = req + .header(CODEAPI_BRIDGE_WORKSPACE_HEADER) + ?.trim(); + if ( + requestedWorkspaceId != null && + requestedWorkspaceId !== '' + ) { + if (bridgeWorkerId == null) { + return res.status(400).json({ + error: 'Workspace selection requires an authenticated bridge worker', + }); + } + if (!isValidBridgeWorkerId(requestedWorkspaceId)) { + return res + .status(400) + .json({ error: 'Invalid code workspace ID' }); + } + workspaceId = requestedWorkspaceId; + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { - return res.status(error.status).json({ error: error.message }); + return res + .status(error.status) + .json({ error: error.message }); } throw error; } @@ -1013,7 +1169,9 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } const decoded = decodeContinuationToken(continuation_token); if (!decoded) { - return res.status(400).json({ error: 'Invalid continuation token' }); + return res + .status(400) + .json({ error: 'Invalid continuation token' }); } const existing = await getExecutionState(decoded.execution_id); if (existing?.mode === 'replay') { @@ -1036,9 +1194,23 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR }); } if (env.PTC_MODE === 'replay') { - return await handleReplayInitial(req, res, { apiKeyId, userId, bridgeWorkerId }); + return await handleReplayInitial(req, res, { + apiKeyId, + userId, + bridgeWorkerId, + workspaceId, + }); } - return await handleBlocking(req, res, { apiKeyId, userId, bridgeWorkerId }); + if (workspaceId != null) { + return res.status(400).json({ + error: 'Selected-workspace programmatic execution requires replay mode', + }); + } + return await handleBlocking(req, res, { + apiKeyId, + userId, + bridgeWorkerId, + }); } catch (err) { logger.error(`[${INSTANCE_ID}] Programmatic routing error:`, err); if (!res.headersSent) { @@ -1046,7 +1218,8 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } return; } -}); + }, +); // --------------------------------------------------------------------------- // Blocking-mode handler (extracted from the original implementation). @@ -1059,46 +1232,47 @@ async function handleBlocking( params: { apiKeyId: string; userId: string; bridgeWorkerId?: string }, ): Promise> { const { apiKeyId, userId, bridgeWorkerId } = params; - const { - code, - tools, - user_id, - files, - continuation_token, - tool_results, - } = req.body as t.ProgrammaticRequestBody; + const { code, tools, user_id, files, continuation_token, tool_results } = + req.body as t.ProgrammaticRequestBody; let timeout: number; try { - timeout = normalizeProgrammaticTimeoutMs((req.body as t.ProgrammaticRequestBody).timeout); + timeout = normalizeProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ); } catch (error) { return res.status(400).json({ error: (error as Error).message }); } // CASE 1: Continuation - if (continuation_token != null && continuation_token !== '' && tool_results) { + if ( + continuation_token != null && + continuation_token !== '' && + tool_results + ) { const decoded = decodeContinuationToken(continuation_token); if (!decoded) { - return res.status(400).json({ error: 'Invalid continuation token' }); + return res + .status(400) + .json({ error: 'Invalid continuation token' }); } const { execution_id } = decoded; const execution = await getExecutionState(execution_id); if (!execution) { - return res.status(404).json({ error: 'Execution not found or expired' }); + return res + .status(404) + .json({ error: 'Execution not found or expired' }); } const identity = getExecutionIdentity(req, userId); if ( execution.userId !== userId || (execution.apiKeyId != null && execution.apiKeyId !== apiKeyId) || - ( - execution.tenantId != null && - execution.tenantId !== identity.storageNamespace - ) || - ( - execution.authContextHash != null && - execution.authContextHash !== req.codeApiAuthContext?.authContextHash - ) + (execution.tenantId != null && + execution.tenantId !== identity.storageNamespace) || + (execution.authContextHash != null && + execution.authContextHash !== + req.codeApiAuthContext?.authContextHash) ) { logger.warn('Unauthorized blocking continuation request rejected', { execution_id, @@ -1122,14 +1296,19 @@ async function handleBlocking( try { await retryToolCallServerRequest( - () => axios.post(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/results`, { + () => + axios.post( + `${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/results`, + { results: tool_results.map(r => ({ call_id: r.call_id, result: r.result, is_error: r.is_error ?? false, error_message: r.error_message, })), - }, { headers: internalServiceHeaders() }), + }, + { headers: internalServiceHeaders() }, + ), 'Submit tool results', ); @@ -1174,14 +1353,21 @@ async function handleBlocking( return res.status(400).json({ error: 'Missing required field: code' }); } if (!tools || !Array.isArray(tools) || tools.length === 0) { - return res.status(400).json({ error: 'Missing required field: tools (must be a non-empty array)' }); + return res + .status(400) + .json({ + error: 'Missing required field: tools (must be a non-empty array)', + }); } if (tools.length > MAX_TOOLS_PER_REQUEST) { - logger.warn(`Too many tools provided: ${tools.length}, limit is ${MAX_TOOLS_PER_REQUEST}`, { + logger.warn( + `Too many tools provided: ${tools.length}, limit is ${MAX_TOOLS_PER_REQUEST}`, + { execution_id: 'pre-creation', userId, toolCount: tools.length, - }); + }, + ); return res.status(400).json({ error: `Too many tools provided (${tools.length}). Maximum is ${MAX_TOOLS_PER_REQUEST}.`, }); @@ -1202,7 +1388,8 @@ async function handleBlocking( files, store: connection, }); - (req.body as t.ProgrammaticRequestBody).files = authorizedFiles.length > 0 ? authorizedFiles : undefined; + (req.body as t.ProgrammaticRequestBody).files = + authorizedFiles.length > 0 ? authorizedFiles : undefined; } catch (error) { if (sendFileRefAuthorizationError(error, res, req)) return; logger.error('Error authorizing programmatic file refs:', error); @@ -1215,7 +1402,14 @@ async function handleBlocking( try { sessionKey = resolveOutputBucketSessionKey(req); } catch (error) { - if (sendSessionKeyResolutionError(error, res, req, 'programmatic /exec-blocking: resolveOutputBucketSessionKey')) { + if ( + sendSessionKeyResolutionError( + error, + res, + req, + 'programmatic /exec-blocking: resolveOutputBucketSessionKey', + ) + ) { return; } throw error; @@ -1270,24 +1464,34 @@ async function handleBlocking( try { callbackUrl = normalizeEgressGatewayUrl(env.EGRESS_GATEWAY_URL); } catch (error) { - logger.error('Blocking PTC requires egress gateway callback URL:', error); + logger.error( + 'Blocking PTC requires egress gateway callback URL:', + error, + ); await cleanupExecution(execution_id, 'blocking'); - return res.status(503).json({ error: 'Egress gateway unavailable' }); + return res + .status(503) + .json({ error: 'Egress gateway unavailable' }); } let callbackToken: string; try { const toolCallResponse = await retryToolCallServerRequest( - () => axios.post<{ + () => + axios.post<{ success: boolean; callback_token: string; - }>(`${env.TOOL_CALL_SERVER_URL}/sessions`, { + }>( + `${env.TOOL_CALL_SERVER_URL}/sessions`, + { execution_id, session_id, timeout, tools, - }, { headers: internalServiceHeaders() }), + }, + { headers: internalServiceHeaders() }, + ), 'Create Tool Call Server session', ); @@ -1299,9 +1503,14 @@ async function handleBlocking( allowedToolNames: tools.map(tool => tool.name), }); } catch (error) { - logger.error('Failed to create Tool Call Server session or callback token:', error); + logger.error( + 'Failed to create Tool Call Server session or callback token:', + error, + ); await cleanupExecution(execution_id, 'blocking'); - return res.status(503).json({ error: 'Tool Call Server unavailable' }); + return res + .status(503) + .json({ error: 'Tool Call Server unavailable' }); } let rawPayload: t.PayloadBody; @@ -1316,10 +1525,15 @@ async function handleBlocking( timeout, }); } catch (error) { - logger.error('Failed to create payload', { execution_id, error: (error as Error).message }); + logger.error('Failed to create payload', { + execution_id, + error: (error as Error).message, + }); await cleanupExecution(execution_id, 'blocking'); return res.status(400).json({ - error: (error as Error).message || 'Failed to generate code payload', + error: + (error as Error).message || + 'Failed to generate code payload', }); } const sandboxSecurity = prepareSandboxJobSecurity({ @@ -1331,7 +1545,9 @@ async function handleBlocking( payload: rawPayload, }); - const job = await pyQueue.add(Jobs.execute, { + const job = await pyQueue.add( + Jobs.execute, + { code, userId, payload: sandboxSecurity.payload, @@ -1350,18 +1566,24 @@ async function handleBlocking( ...(bridgeWorkerId != null ? { bridgeWorkerId } : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, - executionManifestClaims: sandboxSecurity.executionManifestClaims, + executionManifestClaims: + sandboxSecurity.executionManifestClaims, egressGrantClaims: sandboxSecurity.egressGrantClaims, egressGrantToken: sandboxSecurity.egressGrantToken, - }, { + }, + { removeOnComplete: { age: 60, count: 1 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, jobId: session_id, - }); + }, + ); jobsSubmitted.inc({ language: 'python' }); - logger.info('Job queued, polling for tool calls', { execution_id, session_id }); + logger.info('Job queued, polling for tool calls', { + execution_id, + session_id, + }); let clientDisconnected = false; req.on('close', async () => { @@ -1372,21 +1594,27 @@ async function handleBlocking( await job.remove(); await cleanupExecution(execution_id, 'blocking'); } catch (error) { - logger.error('Error cleaning up after client disconnect:', error); + logger.error( + 'Error cleaning up after client disconnect:', + error, + ); } }); job.waitUntilFinished(pyQueueEvents, JOB_COMPLETION_WAIT_TIMEOUT_MS) - .then(async (result) => { + .then(async result => { if (clientDisconnected) return; await setExecutionResult(execution_id, result); }) - .catch(async (error) => { + .catch(async error => { if (clientDisconnected) return; await setExecutionError(execution_id, error); }); - const state = await waitForExecutionState(execution_id, Math.min(timeout, MAX_POLL_TIME)); + const state = await waitForExecutionState( + execution_id, + Math.min(timeout, MAX_POLL_TIME), + ); if (state.status === 'waiting' && state.pending_calls) { return res.status(200).json({ @@ -1416,7 +1644,10 @@ async function handleBlocking( session_id, }); } catch (error) { - logger.error(`[${INSTANCE_ID}] Session ID: ${session_id} | Execution ID: ${execution_id} | Error:`, error); + logger.error( + `[${INSTANCE_ID}] Session ID: ${session_id} | Execution ID: ${execution_id} | Error:`, + error, + ); await cleanupExecution(execution_id, 'blocking'); return res.status(500).json({ error: 'Internal server error' }); } diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index fc84d8f8..81405021 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -83,6 +83,7 @@ describe('buildReplayExecutionState', () => { const state = build({ authContext, bridgeWorkerId: 'code-user_123', + workspaceId: 'project-a', sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -102,6 +103,7 @@ describe('buildReplayExecutionState', () => { authContextHash: 'hash_123', apiKeyId: 'key_legacy', bridgeWorkerId: 'code-user_123', + workspaceId: 'project-a', sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index 25571fed..e6bda59a 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -38,6 +38,7 @@ export interface BuildReplayExecutionStateParams { timeout: number; language: 'python' | 'bash'; bridgeWorkerId?: string; + workspaceId?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -66,6 +67,7 @@ export function buildReplayExecutionState( authContextHash: identity.authContextHash, apiKeyId: params.apiKeyId, bridgeWorkerId: params.bridgeWorkerId, + workspaceId: params.workspaceId, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 2254ee21..3b65cedf 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -116,6 +116,8 @@ export interface ExecutionState { apiKeyId?: string; /** Authenticated worker selection retained across every replay iteration. */ bridgeWorkerId?: string; + /** Selected workspace retained and bound across every replay iteration. */ + workspaceId?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 2a90eac7..f0a6da3b 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -3,7 +3,10 @@ import type { Request } from 'express'; import type { ExecutionManifestClaims } from '../execution-manifest'; import type { ExecutionIdentity } from '../execution-identity'; import type { CodeApiPrincipal } from '../auth/principal'; -import type { ExecutionProfile, SandboxBackendName } from '../execution-profile'; +import type { + ExecutionProfile, + SandboxBackendName, +} from '../execution-profile'; import { Jobs } from '@/enum/service'; /** @@ -149,7 +152,11 @@ export interface RequestBody { runtime_session_hint?: string; } -export type CreatePayload = { req: AuthenticatedRequest, session_id: string; isPyPlot?: boolean }; +export type CreatePayload = { + req: AuthenticatedRequest; + session_id: string; + isPyPlot?: boolean; +}; export interface FileObject { name: string; id: string; @@ -160,10 +167,12 @@ export interface FileObject { size?: number; lastModified?: string; etag?: string; - metadata?: { + metadata?: + | { 'content-type': string; 'original-filename': string; - } | undefined; + } + | undefined; versionId?: string | null; contentType?: string; } @@ -184,6 +193,14 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Stable identity shared by all replay iterations of one execution. */ + execution_id?: string; + replay_tool_count?: number; + /** Manifest-bound upload ceiling exposed to remote workers. */ + max_output_files?: number; + /** Effective per-file ceiling after manifest and gateway policy intersect. */ + max_output_file_bytes?: number; + transfer_timeout_ms?: number; run_memory_limit?: number; run_timeout?: number; run_cpu_time?: number; @@ -238,6 +255,8 @@ export type ExecuteResult = { message?: string | null; status?: string | null; wall_time?: number | null; + /** Trusted worker control channel; avoids losing replay calls to stdout truncation. */ + pending_tool_calls_payload?: string; }; export interface LanguageConfig { @@ -265,6 +284,8 @@ export type JobData = { canonicalUserId?: string; /** Trusted dynamic outbound worker selection. */ bridgeWorkerId?: string; + /** Trusted selected workspace for native replay-mode PTC. */ + workspaceId?: string; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; /** Required sandbox transport. Optional only for jobs queued before fencing. */ diff --git a/service/src/workers.ts b/service/src/workers.ts index ad20fd0f..dc2f491b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -21,6 +21,7 @@ import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; +import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs } from '../../packages/code/src/protocol'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; @@ -91,9 +92,15 @@ async function processJobInner(job: t.ExecuteJob): Promise { const delivery = prepareInputDelivery(payload, sandboxPayload); const sandboxRequest = buildSandboxExecuteRequest({ + ...(job.data.workspaceId == null ? {} : { programmaticTransferReserveMs: programmaticTransferReserveMs(env.JOB_TIMEOUT) }), payload: delivery.payload, egressGrantToken, executionManifestClaims, + maxOutputFileBytes: Math.min( + executionManifestClaims?.max_upload_bytes ?? env.EGRESS_GATEWAY_MAX_FILE_BYTES, + env.EGRESS_GATEWAY_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + ), executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, @@ -149,6 +156,7 @@ async function processJobInner(job: t.ExecuteJob): Promise { tenantId: job.data.tenantId, canonicalUserId: job.data.canonicalUserId, bridgeWorkerId: job.data.bridgeWorkerId, + workspaceId: job.data.workspaceId, runtimeSessionId: runtimeSession.runtimeSessionId, runtimeSessionMode: runtimeSession.runtimeSessionMode, /* Stateful backends run this as a commit barrier after user code but @@ -184,6 +192,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { : {}), stdout, stderr, + ...(responseData.pending_tool_calls_payload != null + ? { pending_tool_calls_payload: responseData.pending_tool_calls_payload } + : {}), }; if (run) { From e4815fa6572a4380bd82678b5d500ce2d4c49fb2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 13:59:24 -0400 Subject: [PATCH 094/116] feat: Report Truncated Output Artifacts (#199) * feat: report truncated output artifacts * fix: classify omitted artifacts precisely * fix: preserve artifact scan invariants * fix: bound depth truncation probes * fix: bound capped directory enumeration * fix: constrain truncation probes across the job * fix: stop exhausted artifact probes --- api/README.md | 11 + api/src/job.ts | 279 ++++++++++++++- api/src/walker.test.ts | 375 +++++++++++++++++++++ service/src/execution-log.test.ts | 13 + service/src/execution-log.ts | 18 + service/src/service/blocking-poll.test.ts | 4 + service/src/service/blocking-poll.ts | 2 + service/src/service/programmatic-router.ts | 6 + service/src/types/service.ts | 12 + service/src/workers.ts | 3 + 10 files changed, 711 insertions(+), 12 deletions(-) diff --git a/api/README.md b/api/README.md index bbbe36ec..665a2931 100644 --- a/api/README.md +++ b/api/README.md @@ -94,6 +94,17 @@ Other package-format-compatible runtimes (Go, Rust, Java, GCC) can be installed Execute code in a sandboxed environment. +When supported output files are omitted because the response reaches its file +count limit, nesting or path limits, file-size limit, or a filesystem entry +cannot be read, the response includes `artifact_truncation`. Its `reasons` +object counts detected omissions by cause, `skipped_count` reports the total +detected omissions, and `skipped` contains up to 20 relative paths so callers +can match an expected output. Intentional filters such as unsupported file +extensions, hidden runtime directories, and unchanged session files do not +produce this marker when they can be classified within the bounded scan. A +depth-capped subtree that exceeds the metadata probe budget is reported +conservatively rather than allowing post-execution traversal to run unbounded. + ### `GET /api/v2/runtimes` List available language runtimes. diff --git a/api/src/job.ts b/api/src/job.ts index b7b82148..27604d29 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -36,9 +36,9 @@ import { SANDBOX_DIR_MODE, SANDBOX_FILE_MODE, ValidationError, + checkPathShape, hasRunnableSource, isDirkeep, - isValidPathShape, validateFilePath, isValidFilePath, } from './validation'; @@ -60,6 +60,15 @@ export { const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; const AUTO_LOAD_DIRKEEP_RETRIES = 2; +const PTC_HISTORY_FILENAME = '_ptc_history.json'; +const TRUNCATION_PROBE_MAX_ENTRIES = 1000; +const TRUNCATION_PROBE_MAX_LEVELS = 10; +const TRUNCATION_PROBE_MAX_HASH_BYTES = 50_000_000; + +interface TruncationProbeState { + remainingEntries: number; + remainingHashBytes: number; +} /** Replaying the same sealed grant cannot repair an authorization denial. */ class InputAuthorizationError extends Error { @@ -647,8 +656,20 @@ interface ExecuteResult { session_id: string; files: FileRef[]; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; } +export type ArtifactTruncationReason = 'max_files' | 'depth' | 'size' | 'path' | 'unreadable'; + +export interface ArtifactTruncation { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; +} + +const MAX_REPORTED_TRUNCATED_PATHS = 20; + const jobQueue: Array<() => void> = []; async function acquireJobIdentity(log: Logger): Promise { @@ -693,6 +714,11 @@ export class Job { private pendingSurfaced = new Map(); private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; + private artifactTruncation: ArtifactTruncation | undefined; + private truncationProbeState: TruncationProbeState = { + remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, + remainingHashBytes: TRUNCATION_PROBE_MAX_HASH_BYTES, + }; private inputFileHashes = new Map(); private inputManifest = new Map(); private inputDestinations = new Map(); @@ -1673,6 +1699,7 @@ export class Job { version: this.runtime.version.raw, session_id: this.outputSessionId, files: this.sessionFiles, + ...(this.artifactTruncation ? { artifact_truncation: this.artifactTruncation } : {}), }; } @@ -1680,6 +1707,11 @@ export class Job { this.generatedFiles = []; this.sessionFiles = []; this.inheritedRefs = []; + this.artifactTruncation = undefined; + this.truncationProbeState = { + remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, + remainingHashBytes: TRUNCATION_PROBE_MAX_HASH_BYTES, + }; const inputByName = new Map(); for (const f of this.files) inputByName.set(f.name, f); @@ -1699,6 +1731,23 @@ export class Job { if (remaining > 0 && this.inheritedRefs.length > 0) { this.sessionFiles.push(...this.inheritedRefs.slice(0, remaining)); } + for (const ref of this.inheritedRefs.slice(remaining)) { + this.recordArtifactTruncation('max_files', ref.name); + } + } + + private recordArtifactTruncation(reason: ArtifactTruncationReason, relativePath: string): void { + this.artifactTruncation ??= { + code: 'artifact_truncated', + reasons: {}, + skipped: [], + skipped_count: 0, + }; + this.artifactTruncation.reasons[reason] = (this.artifactTruncation.reasons[reason] ?? 0) + 1; + this.artifactTruncation.skipped_count++; + if (this.artifactTruncation.skipped.length < MAX_REPORTED_TRUNCATED_PATHS) { + this.artifactTruncation.skipped.push(relativePath); + } } /** @@ -1722,6 +1771,7 @@ export class Job { isRegularFile = st.isFile(); } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: failed to lstat entry'); + this.recordArtifactTruncation('unreadable', relativePath); return 'skip'; } } @@ -1742,7 +1792,14 @@ export class Job { inputByName: Map, ): Promise<{ collected: boolean; truncated: boolean }> { const keepPath = path.join(relativePath, DIRKEEP); - if (!isValidPathShape(keepPath)) return { collected: false, truncated: false }; + const pathShapeError = checkPathShape(keepPath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + keepPath, + ); + return { collected: false, truncated: true }; + } const keepFullPath = path.join(fullPath, DIRKEEP); const inheritedKeep = inputByName.get(keepPath); @@ -1770,6 +1827,7 @@ export class Job { return this.createDirkeepMarker(keepPath, keepFullPath); } if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } const id = nanoid(); @@ -1819,6 +1877,7 @@ export class Job { if (!keepModified || keepInfo?.readOnly === true) return this.echoInheritedKeep(keepPath, inheritedKeep); if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } const refreshedId = nanoid(); @@ -1860,6 +1919,7 @@ export class Job { inheritedKeep: TFile, ): { collected: boolean; truncated: boolean } { if (this.inheritedRefs.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } this.inheritedRefs.push({ @@ -1886,6 +1946,7 @@ export class Job { keepFullPath: string, ): Promise<{ collected: boolean; truncated: boolean }> { if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } try { @@ -1944,6 +2005,7 @@ export class Job { if (existingFile.id && existingFile.storage_session_id) { if (this.inheritedRefs.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', relativePath); return { collected: false, truncated: true }; } this.inheritedRefs.push({ @@ -2003,9 +2065,29 @@ export class Job { size = st.size; } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: unable to stat file'); + this.recordArtifactTruncation('unreadable', relativePath); return { collected: false, truncated: false, stopLoop: false }; } + + const inputFileInfo = this.inputFileHashes.get(relativePath); + const existingFile = inputByName.get(relativePath); if (size > this.runtime.max_file_size) { + /* Only an inline entrypoint needs hashing to decide whether this is + * intentional request-input suppression. Every other oversized file + * is rejected immediately, preserving the scan's bounded I/O cost. */ + if (!inputFileInfo || existingFile?.id != null || relativePath !== this.entryPointName) { + this.recordArtifactTruncation('size', relativePath); + return { collected: false, truncated: false, stopLoop: false }; + } + try { + const currentHash = await this.computeFileHash(fullPath, true); + if (currentHash === inputFileInfo.hash) { + return { collected: true, truncated: false, stopLoop: false }; + } + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed to hash oversized entrypoint'); + } + this.recordArtifactTruncation('size', relativePath); return { collected: false, truncated: false, stopLoop: false }; } @@ -2015,8 +2097,6 @@ export class Job { * stat-only signature would wrongly suppress. Compute once per session/input * file and reuse for the suppression check, wasModified, and the surfaced * mark; non-session jobs still only hash their inputs. */ - const inputFileInfo = this.inputFileHashes.get(relativePath); - const existingFile = inputByName.get(relativePath); let contentHash: string | undefined; if (inputFileInfo != null || this.session != null) { try { @@ -2058,6 +2138,15 @@ export class Job { if (wasModified) this.log.info({ file: relativePath }, 'Input file was modified'); } + /* The unchanged inline entrypoint is executable request input, not an + * output artifact. Suppress it before applying output-size reporting; + * downloaded inputs still flow through the size limit below, preserving + * the existing response-cap behavior for inherited refs. */ + if (!wasModified && inputFileInfo && existingFile?.id == null + && relativePath === this.entryPointName) { + return { collected: true, truncated: false, stopLoop: false }; + } + const echoed = this.tryEchoUnchangedInput({ wasModified, inputFileInfo, @@ -2067,6 +2156,7 @@ export class Job { if (echoed) return { ...echoed, stopLoop: false }; if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', relativePath); return { collected: false, truncated: true, stopLoop: true }; } @@ -2105,10 +2195,130 @@ export class Job { const childStatus = await this.walkDir(fullPath, parentDepth + 1, inputByName); if (childStatus === 'collected') return { collected: true, truncated: false }; if (childStatus === 'skipped') return { collected: false, truncated: true }; - if (this.isOutputCapFull()) return { collected: false, truncated: true }; return this.handleEmptyDirectory(relativePath, fullPath, inputByName); } + /** Finds the first artifact that a scan cap would hide without reading file + * contents. Files below a depth boundary cannot be valid primed inputs, and + * symlinks/unsupported files/hidden runtime directories remain intentional + * exclusions. An empty directory represents a reportable `.dirkeep`. */ + private async findTruncatedArtifact( + dir: string, + inputByName: Map, + state = this.truncationProbeState, + probeDepth = 0, + rootPath = path.relative(this.submissionDir, dir) || '.', + respectSessionSuppression = false, + ): Promise { + /* The state is shared by every probe in this job. Once exhausted, return + * conservatively before opening yet another capped sibling directory. */ + if (state.remainingEntries <= 0) return rootPath; + let directory: fs.Dir; + try { + directory = await fsp.opendir(dir); + } catch (err) { + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + this.log.debug({ dir, err }, 'walkDir: unable to inspect depth-capped directory'); + this.recordArtifactTruncation('unreadable', relativeDir); + return undefined; + } + + let sawVisibleEntry = false; + let sawVisibleNonHiddenEntry = false; + try { + for await (const entry of directory) { + if (entry.name === PTC_HISTORY_FILENAME) continue; + sawVisibleEntry = true; + state.remainingEntries--; + if (state.remainingEntries < 0) return rootPath; + const fullPath = path.join(dir, entry.name); + const relativePath = path.relative(this.submissionDir, fullPath); + const kind = await this.classifyDirent(entry, fullPath, relativePath); + if (kind === 'skip') { + /* Ordinary walking counts symlinks/special entries as non-empty even + * though it does not surface them, so the probe must not invent a + * parent .dirkeep for that shape. */ + sawVisibleNonHiddenEntry = true; + continue; + } + if (kind === 'file') { + sawVisibleNonHiddenEntry = true; + if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; + const existingFile = inputByName.get(relativePath); + const inputFileInfo = this.inputFileHashes.get(relativePath); + if ( + respectSessionSuppression + && relativePath === this.entryPointName + && existingFile?.id == null + && inputFileInfo + ) { + try { + const st = await fsp.lstat(fullPath); + if (!st.isFile()) continue; + if (st.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= st.size; + if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during entrypoint cap probe'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + } + /* Once generated outputs fill the response cap, a persistent + * workspace may still contain unchanged artifacts from earlier + * turns. Ordinary walking suppresses those via their content hash, + * so the bounded cap probe must do the same or it reports a false + * max_files warning. Current-request inputs remain reportable: they + * would otherwise have been echoed into this response. */ + if (respectSessionSuppression && this.session && !existingFile) { + if (this.session.isPrimedReadOnly(relativePath)) continue; + try { + const st = await fsp.lstat(fullPath); + if (!st.isFile()) continue; + if (st.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= st.size; + const hash = await this.computeFileHash(fullPath, true); + if (this.session.isSurfaced(relativePath, hash)) continue; + if ( + this.session.isPrimedInput(relativePath) + && this.session.primedHash(relativePath) === hash + ) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during cap-probe hashing'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + } + return relativePath; + } + if (isHiddenDirectory(entry.name) && !inputsLiveUnder(inputByName, relativePath)) continue; + sawVisibleNonHiddenEntry = true; + /* The probe exists only to avoid false warnings for small, obviously + * unsupported-only subtrees. Once either budget is exhausted, report + * the capped root conservatively instead of defeating the scan bound. */ + if (probeDepth >= TRUNCATION_PROBE_MAX_LEVELS) return rootPath; + const nested = await this.findTruncatedArtifact( + fullPath, + inputByName, + state, + probeDepth + 1, + rootPath, + respectSessionSuppression, + ); + if (nested) return nested; + } + } catch (err) { + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + this.log.debug({ dir, err }, 'walkDir: failed during bounded directory inspection'); + this.recordArtifactTruncation('unreadable', relativeDir); + return undefined; + } + + return sawVisibleEntry && sawVisibleNonHiddenEntry + ? undefined + : path.join(path.relative(this.submissionDir, dir), DIRKEEP); + } + /** * Recursively scans the submission directory for output files. Returns a * status distinguishing truly empty directories from scans truncated by @@ -2120,14 +2330,30 @@ export class Job { depth: number, inputByName: Map, ): Promise<'collected' | 'empty' | 'skipped'> { - if (depth >= config.max_nesting_depth) return 'skipped'; - if (this.isOutputCapFull()) return 'skipped'; - + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + if (depth >= config.max_nesting_depth) { + const skippedPath = await this.findTruncatedArtifact(dir, inputByName); + if (skippedPath) this.recordArtifactTruncation('depth', skippedPath); + return 'skipped'; + } + if (this.isOutputCapFull()) { + const skippedPath = await this.findTruncatedArtifact( + dir, + inputByName, + this.truncationProbeState, + 0, + relativeDir, + true, + ); + if (skippedPath) this.recordArtifactTruncation('max_files', skippedPath); + return 'skipped'; + } let entries: fs.Dirent[]; try { entries = await fsp.readdir(dir, { withFileTypes: true }); } catch (err) { this.log.debug({ dir, err }, 'walkDir: unable to read directory'); + this.recordArtifactTruncation('unreadable', relativeDir); return 'skipped'; } @@ -2146,7 +2372,6 @@ export class Job { * separate npm packages so we can't import directly; the filename literal * is asserted-equal in `service/scripts/test-ptc-sentinel.ts` to catch * accidental drift in CI. */ - const PTC_HISTORY_FILENAME = '_ptc_history.json'; const isPtcReserved = (name: string): boolean => name === PTC_HISTORY_FILENAME; const nonDirkeepCount = entries.reduce( @@ -2166,13 +2391,10 @@ export class Job { let skippedHiddenDirs = 0; for (const entry of entries) { - if (this.isOutputCapFull()) { truncated = true; break; } if (isPtcReserved(entry.name)) continue; const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); - if (!isValidPathShape(relativePath)) continue; - const kind = await this.classifyDirent(entry, fullPath, relativePath); if (kind === 'skip') continue; @@ -2189,9 +2411,42 @@ export class Job { skippedHiddenDirs++; continue; } + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + const skippedPath = await this.findTruncatedArtifact( + fullPath, + inputByName, + this.truncationProbeState, + 0, + relativePath, + ); + if (skippedPath) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + skippedPath, + ); + truncated = true; + } + continue; + } const res = await this.walkSubdirectory(relativePath, fullPath, depth, inputByName); if (res.collected) hasCollectedChild = true; if (res.truncated) truncated = true; + if (this.isOutputCapFull() && this.artifactTruncation?.reasons.max_files) break; + continue; + } + + /* Check intentional filename filtering before path limits. Unsupported + * files never belong in files[], regardless of how long their path is. */ + if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; + + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + relativePath, + ); + truncated = true; continue; } diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index 91382468..543505a9 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -26,11 +26,19 @@ interface WalkerInternals { generatedFiles: Array<{ id: string; name: string; path: string }>; sessionFiles: Array<{ id: string; name: string; storage_session_id: string; modified_from?: { id: string; storage_session_id: string }; inherited?: true; entity_id?: string }>; inheritedRefs: Array<{ id: string; name: string; storage_session_id: string; inherited?: true; entity_id?: string }>; + artifactTruncation?: { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; + }; + truncationProbeState: { remainingEntries: number; remainingHashBytes: number }; pendingSurfaced: Map; inputFileHashes: Map; files: TFile[]; reusePrimedInput: (file: TFile) => Promise; writeFile: (file: TFile) => Promise; + computeFileHash: (filePath: string, noFollow?: boolean) => Promise; walkDir: (dir: string, depth: number, inputByName: Map) => Promise<'collected' | 'empty' | 'skipped'>; handleSessionFiles: () => Promise; } @@ -743,6 +751,10 @@ describe('walkDir / output caps', () => { await internals.walkDir(tmpDir, 0, new Map()); expect(internals.generatedFiles.length).toBeLessThanOrEqual(cap); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + }); }); it('respects max_output_files cap on inherited refs', async () => { @@ -771,6 +783,11 @@ describe('walkDir / output caps', () => { expect(internals.inheritedRefs.length).toBeLessThanOrEqual(cap); expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { max_files: 5 }, + skipped_count: 5, + }); }); }); @@ -792,6 +809,364 @@ describe('walkDir / depth cap', () => { const deepName = path.relative(tmpDir, path.join(cursor, 'deep.py')); expect(internals.generatedFiles.map(f => f.name)).not.toContain(deepName); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { depth: 1 }, + skipped_count: 1, + }); + expect(internals.artifactTruncation?.skipped[0]).toBe(deepName); + }); + + it('does not report a depth cap when the skipped subtree has only unsupported files', async () => { + let cursor = tmpDir; + for (let i = 0; i < config.max_nesting_depth; i++) { + cursor = path.join(cursor, `d${i}`); + await fsp.mkdir(cursor); + } + await fsp.writeFile(path.join(cursor, 'cache.bin'), 'ignored'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('bounds depth-cap eligibility probes and reports the capped subtree conservatively', async () => { + let cursor = tmpDir; + const totalDepth = config.max_nesting_depth + 12; + for (let i = 0; i < totalDepth; i++) { + cursor = path.join(cursor, `d${i}`); + await fsp.mkdir(cursor); + } + await fsp.writeFile(path.join(cursor, 'cache.bin'), 'ignored'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + const cappedRoot = Array.from( + { length: config.max_nesting_depth }, + (_, i) => `d${i}`, + ).join(path.sep); + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { depth: 1 }, + skipped: [cappedRoot], + skipped_count: 1, + }); + }); +}); + +describe('walkDir / artifact truncation details', () => { + it('reports oversized supported outputs while leaving them out of files', async () => { + await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); + const job = makeJob({ maxFileSize: 3 }); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { size: 1 }, + skipped: ['large.txt'], + skipped_count: 1, + }); + }); + + it('reports overlong output paths', async () => { + const directory = 'a'.repeat(200); + await fsp.mkdir(path.join(tmpDir, directory)); + const name = path.join(directory, `${'b'.repeat(60)}.txt`); + await fsp.writeFile(path.join(tmpDir, name), 'content'); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [name], + skipped_count: 1, + }); + }); + + it('does not report an overlong path for an unsupported output', async () => { + const directory = 'a'.repeat(200); + await fsp.mkdir(path.join(tmpDir, directory)); + await fsp.writeFile(path.join(tmpDir, directory, `${'b'.repeat(60)}.bin`), 'ignored'); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('reports an empty-directory marker whose appended path is too long', async () => { + const directory = 'a'.repeat(config.max_path_length - 6); + await fsp.mkdir(path.join(tmpDir, directory)); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [path.join(directory, DIRKEEP)], + skipped_count: 1, + }); + }); + + it('reports an explicit overlong .dirkeep exactly once', async () => { + const directory = 'a'.repeat(config.max_path_length - 6); + await fsp.mkdir(path.join(tmpDir, directory)); + const keepName = path.join(directory, DIRKEEP); + await fsp.writeFile(path.join(tmpDir, keepName), ''); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [keepName], + skipped_count: 1, + }); + }); + + it('uses a bounded probe instead of recursively walking an overlong directory', async () => { + const first = 'a'.repeat(200); + const second = 'b'.repeat(60); + const overlongDir = path.join(first, second); + await fsp.mkdir(path.join(tmpDir, overlongDir), { recursive: true }); + for (let i = 0; i < 1001; i++) { + await fsp.writeFile(path.join(tmpDir, overlongDir, `ignored-${i}.bin`), 'ignored'); + } + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [overlongDir], + skipped_count: 1, + }); + }); + + it('does not report an unchanged oversized inline entrypoint', async () => { + const name = 'main.py'; + const content = 'print(1)'; + const full = path.join(tmpDir, name); + await fsp.writeFile(full, content); + const inline: TFile = { name, content }; + const job = makeJob({ files: [inline], maxFileSize: 3 }); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.entryPointName = name; + internals.inputFileHashes.set(name, { hash: sha256(content), path: full }); + + await internals.walkDir(tmpDir, 0, buildInputByName([inline])); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('inspects a capped directory before deciding whether an artifact was omitted', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + await fsp.mkdir(path.join(tmpDir, 'ignored')); + await fsp.writeFile(path.join(tmpDir, 'ignored', 'cache.bin'), 'ignored'); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('reports a capped empty-directory marker', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + await fsp.mkdir(path.join(tmpDir, 'empty')); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: [path.join('empty', DIRKEEP)], + skipped_count: 1, + }); + }); + + it('does not hash ordinary oversized files in session mode', async () => { + await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); + const session = new SessionWorkspace({ runtimeSessionId: 'rt_large' }); + const internals = asInternals(makeJob({ maxFileSize: 3, session })); + internals.submissionDir = tmpDir; + let hashCalls = 0; + internals.computeFileHash = async () => { + hashCalls++; + return sha256('too large'); + }; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(hashCalls).toBe(0); + expect(internals.artifactTruncation?.reasons).toEqual({ size: 1 }); + }); + + it('keeps scanning for generated outputs when only inherited refs are capped', async () => { + await fsp.mkdir(path.join(tmpDir, 'a-ignored')); + await fsp.writeFile(path.join(tmpDir, 'a-ignored', 'cache.bin'), 'ignored'); + await fsp.writeFile(path.join(tmpDir, 'z-generated.txt'), 'new'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.inheritedRefs = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `inherited-${i}.txt`, + storage_session_id: 'previous', + inherited: true, + })); + internals.artifactTruncation = { + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['another-inherited.txt'], + skipped_count: 1, + }; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles.map(file => file.name)).toContain('z-generated.txt'); + }); + + it('bounds output-cap eligibility probes for wide unsupported-only directories', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + for (let i = 0; i < 1001; i++) { + await fsp.writeFile(path.join(tmpDir, `ignored-${i}.bin`), 'ignored'); + } + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['.'], + skipped_count: 1, + }); + }); + + it('shares the output-cap probe budget across sibling subtrees', async () => { + for (const dirname of ['b-ignored', 'c-ignored']) { + await fsp.mkdir(path.join(tmpDir, dirname)); + for (let i = 0; i < 600; i++) { + await fsp.writeFile(path.join(tmpDir, dirname, `ignored-${i}.bin`), 'ignored'); + } + } + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(path.join(tmpDir, 'b-ignored'), 1, new Map()); + await internals.walkDir(path.join(tmpDir, 'c-ignored'), 1, new Map()); + + expect(internals.artifactTruncation?.reasons).toEqual({ max_files: 1 }); + expect(internals.artifactTruncation?.skipped).toEqual(['c-ignored']); + }); + + it('does not report surfaced session artifacts during output-cap probing', async () => { + const name = 'old-output.txt'; + const content = 'already returned'; + await fsp.writeFile(path.join(tmpDir, name), content); + const session = new SessionWorkspace({ runtimeSessionId: 'rt_capped' }); + session.markSurfaced(name, sha256(content)); + const internals = asInternals(makeJob({ session })); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('does not reopen capped directories after the shared probe budget is exhausted', async () => { + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.truncationProbeState.remainingEntries = 0; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + const absentDir = path.join(tmpDir, 'not-opened'); + + await internals.walkDir(absentDir, 1, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['not-opened'], + skipped_count: 1, + }); + }); + + it('does not report an unchanged inline entrypoint during output-cap probing', async () => { + const directory = path.join(tmpDir, 'src'); + const name = path.join('src', 'main.py'); + const content = 'print(1)'; + await fsp.mkdir(directory); + await fsp.writeFile(path.join(tmpDir, name), content); + const inline: TFile = { name, content }; + const internals = asInternals(makeJob({ files: [inline] })); + internals.submissionDir = tmpDir; + internals.entryPointName = name; + internals.inputFileHashes.set(name, { hash: sha256(content), path: path.join(tmpDir, name) }); + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(directory, 1, buildInputByName([inline])); + + expect(internals.artifactTruncation).toBeUndefined(); }); }); diff --git a/service/src/execution-log.test.ts b/service/src/execution-log.test.ts index f04bad90..0f538125 100644 --- a/service/src/execution-log.test.ts +++ b/service/src/execution-log.test.ts @@ -28,6 +28,12 @@ describe('execution log summaries', () => { failed: 1, detail: 'private storage failure', }, + artifact_truncation: { + code: 'artifact_truncated', + reasons: { max_files: 2 }, + skipped: ['secret-one.txt', 'secret-two.txt'], + skipped_count: 2, + }, run: { code: 0, stdout: 'top secret stdout', @@ -42,6 +48,7 @@ describe('execution log summaries', () => { expect(JSON.stringify(summary)).not.toContain('sensitive stderr'); expect(JSON.stringify(summary)).not.toContain('combined output'); expect(JSON.stringify(summary)).not.toContain('private storage failure'); + expect(JSON.stringify(summary)).not.toContain('secret-one.txt'); expect(summary).toMatchObject({ session_id: 'sess_123', files: { count: 2, inheritedCount: 1, modifiedCount: 1 }, @@ -52,6 +59,12 @@ describe('execution log summaries', () => { delivered: 2, failed: 1, }, + artifact_truncation: { + code: 'artifact_truncated', + reasons: { max_files: 2 }, + skipped_count: 2, + reported_paths: 2, + }, run: { stdout: { length: 17, present: true }, stderr: { length: 16, present: true }, diff --git a/service/src/execution-log.ts b/service/src/execution-log.ts index 48a93cce..143e5541 100644 --- a/service/src/execution-log.ts +++ b/service/src/execution-log.ts @@ -19,6 +19,7 @@ type SandboxResponseLike = { version?: unknown; files?: unknown; artifact_delivery?: unknown; + artifact_truncation?: unknown; run?: RunLike; }; @@ -40,6 +41,22 @@ function summarizeArtifactDelivery(value: unknown): Record | un }; } +function summarizeArtifactTruncation(value: unknown): Record | undefined { + if (value == null || typeof value !== 'object' || Array.isArray(value)) return undefined; + const truncation = value as { + code?: unknown; + reasons?: unknown; + skipped?: unknown; + skipped_count?: unknown; + }; + return { + code: truncation.code, + reasons: truncation.reasons, + skipped_count: truncation.skipped_count, + reported_paths: Array.isArray(truncation.skipped) ? truncation.skipped.length : undefined, + }; +} + export function summarizeText(value: unknown): { length: number; present: boolean } { if (typeof value !== 'string') { return { length: 0, present: false }; @@ -87,6 +104,7 @@ export function summarizeSandboxResponse(data: SandboxResponseLike): Record { expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ status: 'completed', stdout: result.stdout, stderr: '', files: [], artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, }); expect(deps.now()).toBe(2); }); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts index 8ce07fa8..03e386d4 100644 --- a/service/src/service/blocking-poll.ts +++ b/service/src/service/blocking-poll.ts @@ -34,6 +34,7 @@ export async function pollBlockingExecution( stderr?: string; files?: t.FileRefs; artifact_delivery?: t.ArtifactDeliveryFailure; + artifact_truncation?: t.ArtifactTruncation; }> { const start = deps.now(); while (deps.now() - start < timeout) { @@ -48,6 +49,7 @@ export async function pollBlockingExecution( stderr: result.stderr, files: result.files, artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, }; } } diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 4de4a4d8..93bbc0c5 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -1051,6 +1051,9 @@ async function runAndRespond( error: errorMessage, stdout: cleanStdout, stderr: result.stderr, + files: result.files, + artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, session_id: state.session_id, }); return; @@ -1064,6 +1067,7 @@ async function runAndRespond( stderr: result.stderr, files: result.files, artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, session_id: state.session_id, }); } @@ -1331,6 +1335,7 @@ async function handleBlocking( stderr: state.stderr ?? '', files: state.files ?? [], artifact_delivery: state.artifact_delivery, + artifact_truncation: state.artifact_truncation, session_id: execution.session_id, }); } @@ -1633,6 +1638,7 @@ async function handleBlocking( stderr: state.stderr ?? '', files: state.files ?? [], artifact_delivery: state.artifact_delivery, + artifact_truncation: state.artifact_truncation, session_id, }); } diff --git a/service/src/types/service.ts b/service/src/types/service.ts index f0a6da3b..d17b5a99 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -114,6 +114,15 @@ export interface ArtifactDeliveryFailure { failed: number; } +export type ArtifactTruncationReason = 'max_files' | 'depth' | 'size' | 'path' | 'unreadable'; + +export interface ArtifactTruncation { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; +} + export type ExecuteResponse = { run?: { stdout: string; @@ -133,6 +142,7 @@ export type ExecuteResponse = { session_id: string; files: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; }; export interface RequestBody { @@ -250,6 +260,7 @@ export type ExecuteResult = { stderr: string; files: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; code?: number | null; signal?: string | null; message?: string | null; @@ -392,6 +403,7 @@ export interface ProgrammaticResponse { stderr?: string; files?: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; /** Top-level execution session id (one sandbox PTC invocation). */ session_id?: string; tool_calls_made?: number; diff --git a/service/src/workers.ts b/service/src/workers.ts index dc2f491b..f11a5420 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -190,6 +190,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { ...(responseData.artifact_delivery != null ? { artifact_delivery: responseData.artifact_delivery } : {}), + ...(responseData.artifact_truncation != null + ? { artifact_truncation: responseData.artifact_truncation } + : {}), stdout, stderr, ...(responseData.pending_tool_calls_payload != null From f181b4deaa4ddf37a214875d9256932cb812a656 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 14:12:25 -0400 Subject: [PATCH 095/116] fix: cancel selected-workspace PTC across processes (#196) * fix: cancel replay jobs across API and worker processes * fix: drain worker cancellation watches promptly * fix: close programmatic cancellation races * fix: preserve cancellation response ordering * fix: close distributed cancellation races * fix: harden cancellation under concurrent load * fix: make cancellation ownership durable through completion * fix: recover durable replay outcomes across lost replies * fix: return atomic cancellation outcomes with aligned retention * fix: commit native results inside the workspace mutation fence * fix: claim programmatic execution before stalled-job redelivery --- docs/remote-bridge/README.md | 17 + packages/code/src/worker.test.ts | 643 ++++++++++++++--- packages/code/src/worker.ts | 203 ++++-- packages/code/src/workspace-worker.test.ts | 244 +++++++ service/src/bridge/concurrent-worker.test.ts | 20 +- service/src/config.spec.ts | 5 + service/src/config.ts | 9 +- service/src/job-cancellation-commit.test.ts | 431 ++++++++++++ service/src/job-cancellation.test.ts | 599 ++++++++++++++++ service/src/job-cancellation.ts | 664 ++++++++++++++++++ service/src/metrics.ts | 6 + service/src/middleware/limits.ts | 13 + service/src/programmatic-cancellation.test.ts | 234 ++++++ service/src/programmatic-cancellation.ts | 182 +++++ service/src/queue.ts | 39 +- service/src/redis-options.test.ts | 12 +- service/src/redis-options.ts | 6 + service/src/request-disconnect.test.ts | 60 ++ service/src/request-disconnect.ts | 49 ++ service/src/service/programmatic-router.ts | 351 +++++++-- service/src/types/service.ts | 6 + service/src/workers.ts | 399 ++++++++--- 22 files changed, 3872 insertions(+), 320 deletions(-) create mode 100644 service/src/job-cancellation-commit.test.ts create mode 100644 service/src/job-cancellation.test.ts create mode 100644 service/src/job-cancellation.ts create mode 100644 service/src/programmatic-cancellation.test.ts create mode 100644 service/src/programmatic-cancellation.ts create mode 100644 service/src/request-disconnect.test.ts create mode 100644 service/src/request-disconnect.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 99a0dece..b9de3ac2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -256,6 +256,23 @@ execution. the currently registered incarnation. - Request cancellation is polled by the worker and aborts the local sandbox request. +- Replay PTC clients may attach a fresh `X-LibreChat-Code-Request-ID` to each + `/exec/programmatic` request and send that same opaque ID to + `POST /v1/exec/programmatic/cancel`. Code API binds the short-lived request + record to the authenticated principal, durably marks cancellation in Redis, + and publishes it to the worker process holding the BullMQ job. This explicit + path avoids relying on HTTP connection teardown, frees waiting jobs + immediately, and interrupts active remote-bridge assignments without polling + once per active job. + Cancellation and completed-result publication use an atomic Redis decision: + a late cancel returns `already_completed` instead of acknowledging Stop after + completion won. Ambiguous enqueue/cancellation errors retain replay ownership + until a durable fence or the original job deadline. Completed results are + retained temporarily (bounded to 16 MiB) so a lost BullMQ completion reply + does not cause sandbox effects to be repeated. Reconnect reconciliation reads + only small status markers, using one subscriber per process. + Roll out the matching Code API queue-worker processes before enabling this + endpoint on API replicas; pre-cancellation workers do not observe its markers. - A leased assignment remains in a Redis-backed delivery claim until the worker explicitly acknowledges it; reconnecting before acknowledgement redelivers the same fenced assignment instead of losing it after an HTTP disconnect. diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index 68d95b28..e26f212a 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -252,7 +252,10 @@ test('worker asks its supervisor to quarantine an ambiguous stateful runtime', a const quarantined: Array<{ sessionId: string; reason: string }> = []; const supervisor: RuntimeSupervisor = { async acquire() { - return { endpoint: 'http://127.0.0.1:3000/runtime', sessionId: 'rt-user-1' }; + return { + endpoint: 'http://127.0.0.1:3000/runtime', + sessionId: 'rt-user-1', + }; }, async reset() {}, async quarantine(sessionId, reason) { @@ -388,7 +391,10 @@ test('worker continues after an assignment-scoped settlement conflict', async () registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (init?.signal?.aborted === true) { @@ -397,15 +403,25 @@ test('worker continues after an assignment-scoped settlement conflict', async () if (url.endsWith('/lease')) { leases += 1; return new Response( - JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/ack')) { leaseAcknowledged = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -511,7 +527,10 @@ test('worker refreshes its registration during a long assignment', async () => { registeredAt: new Date().toISOString(), leaseTtlMs: 100, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -576,7 +595,10 @@ test('worker schedules registration freshness from request start', async () => { registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -589,7 +611,10 @@ test('worker schedules registration freshness from request start', async () => { if (url.endsWith('/cancelled')) { return new Response( JSON.stringify({ protocolVersion: 1, cancelled: false }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -658,10 +683,13 @@ test('worker continues cancellation polling after a stalled response', async () }); } settlementAttempted = true; - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', @@ -694,6 +722,175 @@ test('worker continues cancellation polling after a stalled response', async () assert.equal(settlementAttempted, true); }); +test('worker stops an outstanding cancellation request before settling completed work', async () => { + let startCancellation!: () => void; + const cancellationStarted = new Promise((resolve) => { + startCancellation = resolve; + }); + let cancellationAborted = false; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + await cancellationStarted; + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + startCancellation(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + cancellationAborted = true; + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 1, + cancellationTransportTimeoutMs: 10_000, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-while-cancellation-polling', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(cancellationAborted, true); + assert.equal(settlementAttempted, true); +}); + +test('worker aborts a retryable cancellation error body before settling completed work', async () => { + let cancellationBodyStarted!: () => void; + const cancellationStarted = new Promise((resolve) => { + cancellationBodyStarted = resolve; + }); + let cancellationBodyAborted = false; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + await cancellationStarted; + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + return new Response( + new ReadableStream({ + start(controller) { + cancellationBodyStarted(); + init?.signal?.addEventListener( + 'abort', + () => { + cancellationBodyAborted = true; + controller.error(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }, + }), + { status: 500 }, + ); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 1, + cancellationTransportTimeoutMs: 10_000, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-during-retryable-cancellation-response', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(cancellationBodyAborted, true); + assert.equal(settlementAttempted, true); +}); + +test('worker stops its cancellation delay before settling immediately completed work', async () => { + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 10_000, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/execute')) { + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + throw new Error('cancellation transport should not start'); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-before-cancellation-polling', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(settlementAttempted, true); +}); + test('worker routes a hintless assignment to an ephemeral template session', async () => { let executeUrl = ''; let runtimeSessionHeader = ''; @@ -852,10 +1049,13 @@ test('worker preserves status for a non-JSON settlement rejection', async () => }, fetchImpl: async (input) => { if (String(input).endsWith('/execute')) { - return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ session_id: 'run-1', files: [] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); } settlementAttempts += 1; return new Response('assignment fenced', { @@ -1032,7 +1232,10 @@ test('worker retries a known-clean rejection after shutdown until acknowledged', registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/execute')) { @@ -1046,18 +1249,20 @@ test('worker retries a known-clean rejection after shutdown until acknowledged', controller.abort(); throw new TypeError('connection reset'); } - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1097,8 +1302,7 @@ test('worker preserves a definite rejection when its heartbeat fails', async () token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1118,7 +1322,10 @@ test('worker preserves a definite rejection when its heartbeat fails', async () registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/execute')) { @@ -1139,7 +1346,10 @@ test('worker preserves a definite rejection when its heartbeat fails', async () } return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1170,8 +1380,7 @@ test('worker quarantines a stateful workspace after a sandbox 5xx response', asy token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1187,7 +1396,10 @@ test('worker quarantines a stateful workspace after a sandbox 5xx response', asy settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1217,8 +1429,7 @@ test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1235,7 +1446,10 @@ test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => JSON.parse(String(init?.body) || '{}').status === 'rejected'; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1268,18 +1482,20 @@ test('worker quarantines a stateful workspace after the sandbox request aborts', }); } settlementAttempted = true; - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1331,13 +1547,23 @@ test('worker surfaces quarantine when shutdown aborts stateful execution', async registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/lease')) { return new Response( - JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -1351,18 +1577,20 @@ test('worker surfaces quarantine when shutdown aborts stateful execution', async ); }); } - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1510,7 +1738,10 @@ test('worker subtracts lease response transit from the server budget', async () if (String(input).endsWith('/ack')) { return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } now += 50; @@ -1527,10 +1758,16 @@ test('worker subtracts lease response transit from the server budget', async () leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(0).toISOString(), remainingMs: 1_000, - request: { body: { language: 'bash' }, headers: {} }, + request: { + body: { language: 'bash' }, + headers: {}, + }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1572,21 +1809,28 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/ack')) { now += 10; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/settle')) { settlementAttempts += 1; - abandonedSettlement = JSON.parse( - String(init?.body), - ) as Record; + abandonedSettlement = JSON.parse(String(init?.body)) as Record< + string, + unknown + >; if (settlementAttempts === 1) { return new Response(JSON.stringify({ error: 'unavailable' }), { status: 503, @@ -1595,7 +1839,10 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( } return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -1611,15 +1858,24 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(0).toISOString(), remainingMs: 10, - request: { body: { language: 'bash' }, headers: {} }, + request: { + body: { language: 'bash' }, + headers: {}, + }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); - await assert.rejects(worker.lease(), /expired during lease acknowledgement/); + await assert.rejects( + worker.lease(), + /expired during lease acknowledgement/, + ); assert.equal(abandonedSettlement?.status, 'rejected'); assert.ok(registrations > 0); assert.equal(settlementAttempts, 2); @@ -1653,7 +1909,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as JSON.parse(String(init?.body) || '{}').status === 'rejected'; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/workers/register')) { @@ -1665,7 +1924,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -1685,7 +1947,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as request: { body: { language: 'bash' }, headers: {} }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1702,8 +1967,7 @@ test('worker clamps rejected settlement errors to the protocol limit', async () token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1716,11 +1980,16 @@ test('worker clamps rejected settlement errors to the protocol limit', async () headers: { 'Content-Type': 'application/json' }, }); } - const settlement = JSON.parse(String(init?.body)) as { error: string }; + const settlement = JSON.parse(String(init?.body)) as { + error: string; + }; rejection = settlement.error; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1747,8 +2016,7 @@ test('worker quarantines an explicitly dirty stateful sandbox response', async ( token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1761,13 +2029,19 @@ test('worker quarantines an explicitly dirty stateful sandbox response', async ( error: 'session_workspace_dirty', message: 'restore required', }), - { status: 409, headers: { 'Content-Type': 'application/json' } }, + { + status: 409, + headers: { 'Content-Type': 'application/json' }, + }, ); } settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1859,15 +2133,21 @@ test('worker uses the server-relative lease budget despite VM clock skew', async }, fetchImpl: async (input) => { if (String(input).endsWith('/execute')) { - return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ session_id: 'run-1', files: [] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); } settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1886,7 +2166,6 @@ test('worker uses the server-relative lease budget despite VM clock skew', async assert.equal(settlementAttempted, true); }); - test('worker continues after an expired assignment settlement conflict', async () => { const controller = new AbortController(); let registrations = 0; @@ -1921,18 +2200,22 @@ test('worker continues after an expired assignment settlement conflict', async ( leases += 1; return Response.json({ protocolVersion: 1, - assignment: leases === 1 - ? { - protocolVersion: 1, - assignmentId: 'assignment-expired', - workerId: 'vm-1', - incarnationId, - generation: 1, - leaseToken: 'lease-token-that-is-long-enough-for-testing', - expiresAt: new Date(Date.now() + 10_000).toISOString(), - request: { body: { language: 'bash' }, headers: {} }, - } - : undefined, + assignment: + leases === 1 + ? { + protocolVersion: 1, + assignmentId: 'assignment-expired', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + request: { + body: { language: 'bash' }, + headers: {}, + }, + } + : undefined, }); } if (url.endsWith('/execute')) { @@ -1940,7 +2223,10 @@ test('worker continues after an expired assignment settlement conflict', async ( } if (url.endsWith('/settle')) { return Response.json( - { error: 'Bridge assignment has expired', code: 'ASSIGNMENT_EXPIRED' }, + { + error: 'Bridge assignment has expired', + code: 'ASSIGNMENT_EXPIRED', + }, { status: 409 }, ); } @@ -2172,7 +2458,10 @@ test('paired worker rotates credentials throughout a long assignment', async () } if (url.endsWith('/execute')) { await new Promise((resolve) => setTimeout(resolve, 55)); - return Response.json({ session_id: 'run-long-rotation', files: [] }); + return Response.json({ + session_id: 'run-long-rotation', + files: [], + }); } return Response.json({ protocolVersion: 1, accepted: true }); }; @@ -2276,6 +2565,117 @@ test('paired worker cancels a stalled credential refresh after execution', async assert.equal(refreshAborted, true); }); +test('one concurrent caller cannot abort a credential refresh another caller still needs', async () => { + const key = createBridgeIdentity(); + const first = new AbortController(); + const second = new AbortController(); + let releaseRefresh!: () => void; + let refreshStarted!: () => void; + const started = new Promise((resolve) => { + refreshStarted = resolve; + }); + const released = new Promise((resolve) => { + releaseRefresh = resolve; + }); + let transportAborted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-shared-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + assert.match(String(input), /credentials\/refresh$/); + refreshStarted(); + init?.signal?.addEventListener('abort', () => { + transportAborted = true; + }); + await released; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-shared-refresh-value', + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }); + }, + }); + + const firstRefresh = worker.refreshCredential(first.signal); + await started; + const secondRefresh = worker.refreshCredential(second.signal); + first.abort(); + + await assert.rejects(firstRefresh, { name: 'AbortError' }); + assert.equal(transportAborted, false); + releaseRefresh(); + await secondRefresh; + assert.equal(transportAborted, false); +}); + +test('a new caller starts a fresh credential refresh after the last waiter aborts', async () => { + const key = createBridgeIdentity(); + const first = new AbortController(); + let refreshCount = 0; + let firstRefreshStarted!: () => void; + const started = new Promise((resolve) => { + firstRefreshStarted = resolve; + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-replacement-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (_input, init) => { + refreshCount += 1; + if (refreshCount === 1) { + firstRefreshStarted(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-replacement-refresh', + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }); + }, + }); + + const abandoned = worker.refreshCredential(first.signal, Date.now() + 1_000); + await started; + first.abort(); + await assert.rejects(abandoned, { name: 'AbortError' }); + await worker.refreshCredential(undefined, Date.now() + 1_000); + + assert.equal(refreshCount, 2); +}); + test('paired worker refreshes conservatively before server clock calibration', async () => { const key = createBridgeIdentity(); let refreshCount = 0; @@ -2347,8 +2747,7 @@ test('paired worker charges initial credential refresh against the assignment de sandboxStarted = true; } if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2389,8 +2788,7 @@ test('paired worker rechecks the deadline after request serialization', async () codeApiUrl: 'https://code.example/v1', workerId: 'vm-1', incarnationId, - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', identity: { privateKey: key.privateKey, credential: 'credential-valid-during-serialization', @@ -2408,8 +2806,7 @@ test('paired worker rechecks the deadline after request serialization', async () sandboxStarted = true; } if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2471,8 +2868,7 @@ test('paired worker keeps endpoint validation failures known-clean', async () => const url = String(input); if (url.endsWith('/execute')) sandboxStarted = true; if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2758,11 +3154,13 @@ test('sandbox completion does not cancel an in-flight credential rotation', asyn } if (url.endsWith('/execute')) { await refreshStartedPromise; - return Response.json({ session_id: 'run-rotation-race', files: [] }); + return Response.json({ + session_id: 'run-rotation-race', + files: [], + }); } - settleAuthorization = ( - init?.headers as Record - ).Authorization; + settleAuthorization = (init?.headers as Record) + .Authorization; return Response.json({ protocolVersion: 1, accepted: true }); }; const worker = new BridgeWorker({ @@ -2799,8 +3197,41 @@ test('sandbox completion does not cancel an in-flight credential rotation', asyn ); }); +test('settlement does not drain another lane credential renewal', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', workerId: 'vm-1', token: 'fixture', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + fetchImpl: async (input) => String(input).endsWith('/execute') + ? Response.json({ session_id: 'independent-lane', files: [] }) + : Response.json({ protocolVersion: 1, accepted: true }), + }); + // A different lane owns this pending renewal. The settling lane has no + // maintenance waiter and must not consume its own lease on that promise. + Object.assign(worker, { credentialInFlight: { + promise: new Promise(() => {}), controller: new AbortController(), waiters: 1, + }, refreshCredential: async () => {} }); + const startedAt = Date.now(); + await worker.executeAndSettle({ + protocolVersion: 1, assignmentId: 'independent-lane', workerId: 'vm-1', + incarnationId, generation: 5, leaseToken: 'independent-lane-lease-token', + expiresAt: new Date(Date.now() + 600_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.ok(Date.now() - startedAt < 500, 'unrelated renewal must not add a one-second drain'); +}); + test('reconnect delay uses bounded exponential jitter', () => { - assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 0), 500); - assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 1), 1_000); - assert.equal(reconnectDelayMs(10, 1_000, 30_000, () => 1), 30_000); + assert.equal( + reconnectDelayMs(0, 1_000, 30_000, () => 0), + 500, + ); + assert.equal( + reconnectDelayMs(0, 1_000, 30_000, () => 1), + 1_000, + ); + assert.equal( + reconnectDelayMs(10, 1_000, 30_000, () => 1), + 30_000, + ); }); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index a0517bd9..ffe1b350 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -38,6 +38,11 @@ export interface BridgeWorkerOptions { capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; workspaceProgrammatic?: { + /** + * True when a WorkspaceToolError without mutation uncertainty proves the + * selected workspace was not changed. + */ + mutationFailuresAreAtomic?: true; executeProgrammatic( workspaceId: string, request: BridgeWorkspaceProgrammaticRequest, @@ -99,6 +104,7 @@ const DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CANCELLATION_POLL_INTERVAL_MS = 500; const DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS = 2_000; +const CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS = 1_000; const MIN_REGISTRATION_HEARTBEAT_MS = 25; const REGISTRATION_RETRY_DELAY_MS = 100; const CREDENTIAL_REFRESH_RETRY_DELAY_MS = 100; @@ -158,26 +164,29 @@ function workspaceCapabilitiesMatch( advertised.writeFileModes?.length === executor.writeFileModes?.length && (advertised.writeFileModes?.every( (mode, index) => mode === executor.writeFileModes?.[index], - ) ?? executor.writeFileModes == null) && + ) ?? + executor.writeFileModes == null) && advertised.editFileModes?.length === executor.editFileModes?.length && (advertised.editFileModes?.every( (mode, index) => mode === executor.editFileModes?.[index], - ) ?? executor.editFileModes == null) && - advertised.editFileFeatures?.length === - executor.editFileFeatures?.length && + ) ?? + executor.editFileModes == null) && + advertised.editFileFeatures?.length === executor.editFileFeatures?.length && (advertised.editFileFeatures?.every( (feature, index) => feature === executor.editFileFeatures?.[index], - ) ?? executor.editFileFeatures == null) && - advertised.listFileFeatures?.length === - executor.listFileFeatures?.length && + ) ?? + executor.editFileFeatures == null) && + advertised.listFileFeatures?.length === executor.listFileFeatures?.length && (advertised.listFileFeatures?.every( (feature, index) => feature === executor.listFileFeatures?.[index], - ) ?? executor.listFileFeatures == null) && + ) ?? + executor.listFileFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( (language, index) => language === executor.programmaticLanguages?.[index], - ) ?? executor.programmaticLanguages == null) && + ) ?? + executor.programmaticLanguages == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -189,7 +198,8 @@ function workspaceCapabilitiesMatch( (operation, operationIndex) => operation === executor.workspaces[index]?.operations?.[operationIndex], - ) ?? executor.workspaces[index]?.operations == null), + ) ?? + executor.workspaces[index]?.operations == null), ) ); } @@ -201,8 +211,7 @@ function registrationCompatibleCapabilities( if ( workspaceTools == null || (workspaceTools.operations.every( - (operation) => - operation === 'read_file' || operation === 'search_text', + (operation) => operation === 'read_file' || operation === 'search_text', ) && workspaceTools.workspaces.every( (workspace) => workspace.operations == null, @@ -211,8 +220,7 @@ function registrationCompatibleCapabilities( return capabilities; } const operations = workspaceTools.operations.filter( - (operation) => - operation === 'read_file' || operation === 'search_text', + (operation) => operation === 'read_file' || operation === 'search_text', ); if (operations.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; @@ -221,7 +229,9 @@ function registrationCompatibleCapabilities( const workspaces = workspaceTools.workspaces.flatMap((workspace) => { if ( workspace.operations != null && - !operations.every((operation) => workspace.operations?.includes(operation)) + !operations.every((operation) => + workspace.operations?.includes(operation), + ) ) { return []; } @@ -386,7 +396,11 @@ export class BridgeWorker { private negotiatedWorkspaceSlots = 1; private concurrentRunning = false; private registrationInFlight?: Promise; - private credentialInFlight?: Promise; + private credentialInFlight?: { + promise: Promise; + controller: AbortController; + waiters: number; + }; private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; constructor(private readonly options: BridgeWorkerOptions) { @@ -434,7 +448,8 @@ export class BridgeWorker { (options.workspaceProgrammatic != null) !== (options.capabilities.workspaceTools?.programmaticLanguages?.includes( 'bash', - ) === true) + ) === + true) ) { throw new BridgeProtocolError( 'Workspace programmatic capability requires a matching executor', @@ -553,7 +568,9 @@ export class BridgeWorker { if (signal?.aborted) { abortRegistration(); } else { - signal?.addEventListener('abort', abortRegistration, { once: true }); + signal?.addEventListener('abort', abortRegistration, { + once: true, + }); } const timeoutMs = Math.min( Math.max(1, this.registrationTtlMs - 1), @@ -575,7 +592,10 @@ export class BridgeWorker { workerId: this.options.workerId, incarnationId: this.incarnationId, capabilities: this.maintenanceOnly - ? { ...capabilities, requiresReadyConfirmation: true } + ? { + ...capabilities, + requiresReadyConfirmation: true, + } : capabilities, }, registrationController.signal, @@ -709,7 +729,9 @@ export class BridgeWorker { } await this.runtimeSupervisor.reset(runtimeSessionId, signal); await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + `${this.codeApiUrl}${bridgeWorkerPath( + this.options.workerId, + )}/workspaces/reset`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, @@ -745,7 +767,9 @@ export class BridgeWorker { // machine-local guard before the remote fence can be removed. await guard.assertAvailable(); await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + `${this.codeApiUrl}${bridgeWorkerPath( + this.options.workerId, + )}/workspaces/reset`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, @@ -1051,20 +1075,58 @@ export class BridgeWorker { transportTimeoutMs = Number.POSITIVE_INFINITY, ): Promise { while (this.credentialInFlight) { - await this.credentialInFlight; + await this.waitForCredentialRefresh(this.credentialInFlight, signal); // A longer-lived caller may still need another refresh after this one. } + const controller = new AbortController(); const pending = this.refreshCredentialOwned( - signal, + controller.signal, validThroughMs, transportTimeoutMs, ); - this.credentialInFlight = pending; + const entry = { promise: pending, controller, waiters: 0 }; + this.credentialInFlight = entry; + void pending.then( + () => { + if (this.credentialInFlight === entry) + this.credentialInFlight = undefined; + }, + () => { + if (this.credentialInFlight === entry) + this.credentialInFlight = undefined; + }, + ); + await this.waitForCredentialRefresh(entry, signal); + } + + private async waitForCredentialRefresh( + entry: NonNullable, + signal?: AbortSignal, + ): Promise { + entry.waiters += 1; + let removeAbortListener = (): void => {}; + const aborted = new Promise((_, reject) => { + if (signal == null) return; + const abort = (): void => + reject( + signal.reason instanceof Error + ? signal.reason + : new DOMException('aborted', 'AbortError'), + ); + removeAbortListener = (): void => + signal.removeEventListener('abort', abort); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + }); try { - await pending; + await Promise.race([entry.promise, aborted]); } finally { - if (this.credentialInFlight === pending) + removeAbortListener(); + entry.waiters -= 1; + if (entry.waiters === 0 && this.credentialInFlight === entry) { this.credentialInFlight = undefined; + entry.controller.abort(); + } } } @@ -1118,7 +1180,7 @@ export class BridgeWorker { assignment: BridgeAssignment, stopSignal: AbortSignal, serverClockOffsetMs: number, - requestSignal?: AbortSignal, + maintenance: { refresh?: Promise }, ): Promise { const identity = this.options.identity; if (identity == null) return; @@ -1136,18 +1198,18 @@ export class BridgeWorker { await abortableDelay(waitMs, stopSignal); if (stopSignal.aborted || Date.now() >= assignmentDeadlineMs) return; try { - await this.refreshCredential( - requestSignal, + maintenance.refresh = this.refreshCredential( + stopSignal, Date.now() + serverClockOffsetMs + refreshWindowMs, ); + await maintenance.refresh; } catch (error) { if (stopSignal.aborted) return; const terminal = error instanceof BridgeProtocolError && (error.status === 401 || error.status === 403); const credentialRemainingMs = - Date.parse(identity.expiresAt) - - (Date.now() + serverClockOffsetMs); + Date.parse(identity.expiresAt) - (Date.now() + serverClockOffsetMs); if (terminal || credentialRemainingMs <= 0) throw error; await abortableDelay( Math.min( @@ -1156,6 +1218,8 @@ export class BridgeWorker { ), stopSignal, ); + } finally { + maintenance.refresh = undefined; } } } @@ -1352,6 +1416,7 @@ export class BridgeWorker { ); let credentialMaintenanceError: unknown; let credentialMaintenance: Promise | undefined; + const ownCredentialMaintenance: { refresh?: Promise } = {}; let settlement: BridgeSettlement; let ambiguousSandboxError: unknown; let ambiguousWorkspaceMutationError: unknown; @@ -1368,7 +1433,7 @@ export class BridgeWorker { assignment, credentialController.signal, serverClockOffsetMs, - signal, + ownCredentialMaintenance, ).catch((error) => { credentialMaintenanceError = error; executionController.abort(); @@ -1674,14 +1739,22 @@ export class BridgeWorker { ) { workspaceMutationGuardError = error; } + const knownAtomicWorkspaceToolFailure = + assignment.executionKind === 'workspace_tool' && + error instanceof WorkspaceToolError && + this.options.workspaceTools?.mutationFailuresAreAtomic === true && + !error.requiresQuarantine; + const knownAtomicProgrammaticFailure = + assignment.executionKind === 'workspace_programmatic' && + error instanceof WorkspaceToolError && + this.options.workspaceProgrammatic?.mutationFailuresAreAtomic === + true && + !error.requiresQuarantine; if ( workspaceMutationApplied || (workspaceMutationArmed && - !( - error instanceof WorkspaceToolError && - this.options.workspaceTools?.mutationFailuresAreAtomic === true && - !error.requiresQuarantine - )) + !knownAtomicWorkspaceToolFailure && + !knownAtomicProgrammaticFailure) ) { ambiguousWorkspaceMutationError = error; } @@ -1713,12 +1786,30 @@ export class BridgeWorker { clearTimeout(deadlineTimer); cancellationController.abort(); await cancellationWatcher; + // Only drain renewal joined by this assignment, never an unrelated lane's + // refresh. Leave settlement time inside the original assignment budget. + const credentialInFlight = ownCredentialMaintenance.refresh; + if (credentialInFlight != null && !credentialController.signal.aborted) { + let drainTimer: ReturnType | undefined; + await Promise.race([ + credentialInFlight.catch(() => undefined), + new Promise((resolve) => { + drainTimer = setTimeout( + resolve, + Math.min(CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS, + Math.max(0, Date.parse(assignment.expiresAt) - serverClockOffsetMs - Date.now() - 5_000)), + ); + }), + ]); + if (drainTimer != null) clearTimeout(drainTimer); + } credentialController.abort(); await credentialMaintenance; try { if (workspaceMutationGuardError != null) throw workspaceMutationGuardError; if (ambiguousWorkspaceMutationError != null) { + this.options.onError?.(ambiguousWorkspaceMutationError); throw await this.quarantineWorkspace( undefined, 'Worker stopped after a workspace mutation completed without a fulfilled settlement', @@ -1913,7 +2004,9 @@ export class BridgeWorker { return await lease.execute({ body, headers, signal }); } if (lease.endpoint == null) { - throw new BridgeProtocolError('Runtime lease does not provide an execution transport'); + throw new BridgeProtocolError( + 'Runtime lease does not provide an execution transport', + ); } const endpoint = lease.endpoint.replace(/\/+$/, ''); const response = await this.fetchImpl(`${endpoint}/execute`, { @@ -2170,17 +2263,23 @@ export class BridgeWorker { signal: AbortSignal, ): Promise { while (!signal.aborted && !executionController.signal.aborted) { - await this.delay( - Math.max( - 1, - this.options.cancellationPollIntervalMs ?? - DEFAULT_CANCELLATION_POLL_INTERVAL_MS, - ), - signal, - ); + try { + await this.delay( + Math.max( + 1, + this.options.cancellationPollIntervalMs ?? + DEFAULT_CANCELLATION_POLL_INTERVAL_MS, + ), + signal, + ); + } catch (error) { + if (signal.aborted || executionController.signal.aborted) return; + throw error; + } if (signal.aborted || executionController.signal.aborted) return; const pollController = new AbortController(); const abortPoll = (): void => pollController.abort(); + signal.addEventListener('abort', abortPoll, { once: true }); executionController.signal.addEventListener('abort', abortPoll, { once: true, }); @@ -2200,6 +2299,15 @@ export class BridgeWorker { incarnationId: this.incarnationId, }, pollController.signal, + (response) => { + // Once response headers arrive, drain the bounded body before a + // successful execution can settle. Otherwise a cancellation=true + // response racing command completion can be discarded. The + // transport timer and execution signal still cap the drain. + if (response.ok || response.status === 404) { + signal.removeEventListener('abort', abortPoll); + } + }, ); if (response.cancelled) { executionController.abort(); @@ -2213,6 +2321,7 @@ export class BridgeWorker { if (signal.aborted) return; } finally { clearTimeout(timeout); + signal.removeEventListener('abort', abortPoll); executionController.signal.removeEventListener('abort', abortPoll); } } @@ -2222,6 +2331,7 @@ export class BridgeWorker { url: string, body: object, signal?: AbortSignal, + onResponseHeaders?: (response: Response) => void, ): Promise { const requestBody = JSON.stringify(body); const response = await this.fetchImpl(url, { @@ -2233,6 +2343,7 @@ export class BridgeWorker { body: requestBody, signal, }); + onResponseHeaders?.(response); let payload: unknown; try { payload = await response.json(); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index fa08a84c..f6205cd4 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1032,6 +1032,250 @@ test('worker executes programmatic Bash in the selected workspace and preserves ]); }); +test('worker keeps a selected workspace usable after an atomic programmatic setup failure', async () => { + const lifecycle: string[] = []; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw new WorkspaceToolError( + 'Programmatic input download failed', + 'COMMAND_UNAVAILABLE', + ); + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + ], + ]), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-setup-failure', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'clear']); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); +}); + +test('worker keeps a selected workspace usable after confirmed programmatic cancellation cleanup', async () => { + const lifecycle: string[] = []; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + ], + ]), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-cancelled-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'sleep 30' }], + }, + headers: {}, + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'clear']); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'EXECUTION_ABORTED'); +}); + +test('worker reports the underlying cause before quarantining an uncertain programmatic mutation', async () => { + const rootCause = new WorkspaceToolError( + 'Programmatic output upload failed', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + let reported: unknown; + let quarantined = false; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw rootCause; + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine(() => { + quarantined = true; + }), + ], + ]), + onError(error) { + reported = error; + }, + fetchImpl: async () => { + throw new Error('settlement must not run'); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-uncertain-failure', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }, + }), + BridgeWorkspaceQuarantinedError, + ); + + assert.equal(reported, rootCause); + assert.equal(quarantined, true); +}); + test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { let quarantinedReason: string | undefined; let armed = 0; diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts index 5aec49cf..ea5ac601 100644 --- a/service/src/bridge/concurrent-worker.test.ts +++ b/service/src/bridge/concurrent-worker.test.ts @@ -243,11 +243,25 @@ for (const failure of [ status: 'fulfilled', value: { status: 'fulfilled' }, }); - for (let i = 0; i < 300 && errors.length === 0; i++) + const diagnosticCount = cleanupFailure ? 1 : 2; + const quarantineAttemptCount = + failure === 'lost-response' + ? 2 + : failure === 'all-responses-lost' || failure === 'delivery-outage' + ? 3 + : 1; + for ( + let i = 0; + i < 300 && + (errors.length < diagnosticCount || + (!cleanupFailure && quarantineAttempts < quarantineAttemptCount)); + i++ + ) { await new Promise((resolve) => setTimeout(resolve, 5)); + } if (failure === 'delivery-outage') - expect(errors.length).toBeGreaterThanOrEqual(1); - else expect(errors.length).toBe(1); + expect(errors.length).toBeGreaterThanOrEqual(2); + else expect(errors.length).toBe(diagnosticCount); if (failure === 'lost-response') expect(quarantineAttempts).toBe(2); if (failure === 'delivery-outage') expect(quarantineAttempts).toBeGreaterThanOrEqual(3); diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 44b64b8b..69b416d8 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -99,6 +99,11 @@ describe('egress grant TTL configuration', () => { }); describe('job deadline accounting', () => { + it('never extends the producer deadline when worker configuration differs', () => { + expect(jobDeadlineAtMs(1_000, 300_000, 50_000, 91_000)).toBe(91_000); + expect(jobDeadlineAtMs(1_000, 30_000, 50_000, 91_000)).toBe(31_000); + expect(jobDeadlineAtMs(1_000, 300_000, 50_000, Number.NaN)).toBe(0); + }); it('counts time spent waiting in BullMQ against JOB_TIMEOUT', () => { expect(jobDeadlineAtMs(1_000, 300_000, 50_000)).toBe(301_000); }); diff --git a/service/src/config.ts b/service/src/config.ts index d025831e..55570dfe 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -95,10 +95,17 @@ export function jobDeadlineAtMs( enqueuedAtMs: number | undefined, timeoutMs: number, nowMs: number = Date.now(), + producerDeadlineAtMs?: number, ): number { - return Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 + const localDeadline = Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 ? (enqueuedAtMs as number) + timeoutMs : nowMs + timeoutMs; + if (producerDeadlineAtMs === undefined) return localDeadline; + // A worker with a larger JOB_TIMEOUT must not outlive the admission fence + // retained by its API producer. Malformed explicit deadlines fail closed. + return Number.isFinite(producerDeadlineAtMs) + ? Math.min(localDeadline, producerDeadlineAtMs) + : 0; } /** The worker stops user work at JOB_TIMEOUT, then may still need to terminate diff --git a/service/src/job-cancellation-commit.test.ts b/service/src/job-cancellation-commit.test.ts new file mode 100644 index 00000000..8ded29f2 --- /dev/null +++ b/service/src/job-cancellation-commit.test.ts @@ -0,0 +1,431 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test'; +import { startTestRedis } from './test/redis'; +import { RedisBridgeStore } from './bridge/store'; +import { + commitJobResult, + readCommittedJobResult, + requestJobCancellation, + JobCancellationRegistry, + jobCancellationInternals, + fenceJobCancellation, + waitForJobWithCancellation, + jobCancellationRetentionSeconds, + claimJobExecution, +} from './job-cancellation'; + +let redis: Awaited>; +beforeEach(async () => { + redis = await startTestRedis(); +}); +afterEach(async () => { + await redis.closeTestServer(); +}); +const target = { queueName: 'other', jobId: 'commit-race' }; + +for (const outcome of ['commit', 'stop', 'duplicate']) + test(`native mutation handoff commits or quarantines before root release (${outcome})`, async () => { + const store = new RedisBridgeStore(redis); + const workerId = 'handoff-worker'; + const incarnationId = 'incarnation-handoff-01'; + await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const controller = new AbortController(); + const dispatchArgs = { + workerId, + workspaceId: 'primary', + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'handoff-session', + files: [{ name: 'main.sh', content: 'echo mutation' }], + }, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }; + const completion = store.dispatch({ + ...dispatchArgs, + finalize: async settlement => { + if (outcome === 'stop') await requestJobCancellation(redis, target, 60); + if (outcome === 'duplicate') + await commitJobResult( + redis, + target, + { stdout: 'first mutation' }, + 60, + ); + if ( + (await commitJobResult( + redis, + target, + { stdout: 'mutation settled' }, + 60, + )) !== 'committed' + ) + throw new Error('handoff did not win'); + // This represents Stop during post-handoff egress cleanup. It must no + // longer turn the applied mutation into an acknowledged cancellation. + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + return settlement; + }, + }); + void completion.catch(() => undefined); + const assignment = await store.lease(workerId, incarnationId, 1_000); + if (assignment == null) throw new Error('Missing assignment'); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled', + result: { + session_id: 'handoff-session', + language: 'bash', + version: '5.2', + files: [], + }, + }); + if (outcome !== 'commit') { + await expect(completion).rejects.toThrow('handoff did not win'); + await expect(store.dispatch(dispatchArgs)).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + } else { + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + }); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'mutation settled' }, + }); + } + }); + +test('concurrent stalled-job redelivery claims at most one sandbox execution', async () => { + let executions = 0; + const attempt = async () => { + const claim = await claimJobExecution(redis, target, 60); + if (claim.status === 'claimed') executions += 1; + return claim; + }; + const results = await Promise.allSettled([attempt(), attempt()]); + expect(executions).toBe(1); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength( + 1, + ); + expect(results.filter(result => result.status === 'rejected')).toHaveLength( + 1, + ); + await expect(attempt()).rejects.toThrow('already claimed'); + expect(executions).toBe(1); + await commitJobResult(redis, target, { stdout: 'first result' }, 60); + expect(await attempt()).toEqual({ + status: 'completed', + result: { stdout: 'first result' }, + }); + expect(executions).toBe(1); + expect( + await commitJobResult(redis, target, { stdout: 'different result' }, 60), + ).toBe('already_completed'); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'first result' }, + }); +}); + +test('a lost execution-claim reply never authorizes a second attempt', async () => { + const lostReply = { + eval: async (...args: Parameters) => { + await redis.eval(...args); + throw new Error('claim reply lost'); + }, + } as unknown as typeof redis; + await expect(claimJobExecution(lostReply, target, 60)).rejects.toThrow( + 'claim reply lost', + ); + await expect(claimJobExecution(redis, target, 60)).rejects.toThrow( + 'already claimed', + ); +}); + +test('cancel-before-claim and missing completion payload fail closed', async () => { + await requestJobCancellation(redis, target, 60); + await expect(claimJobExecution(redis, target, 60)).rejects.toThrow( + 'cancelled', + ); + const completedTarget = { ...target, jobId: 'missing-payload-claim' }; + await commitJobResult(redis, completedTarget, { stdout: 'done' }, 60); + await redis.del( + `${jobCancellationInternals.cancellationKey(completedTarget)}:result`, + ); + await expect(claimJobExecution(redis, completedTarget, 60)).rejects.toThrow( + 'refusing re-execution', + ); +}); + +for (const corrupt of [false, true]) + test(`invalid committed result fails immediately without Redis retries (corrupt=${corrupt})`, async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = jobCancellationInternals.cancellationKey(target); + if (corrupt) await redis.set(`${key}:result`, '{invalid'); + else await redis.del(`${key}:result`); + let calls = 0; + const commands = { + eval: (...args: Parameters) => { + calls += 1; + return redis.eval(...args); + }, + } as unknown as typeof redis; + await expect( + fenceJobCancellation({ + commands, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 30_000, + }), + ).rejects.toThrow(); + expect(calls).toBe(1); + }); + +test('completion retention includes the API producer across timeout configuration drift', async () => { + const ttl = jobCancellationRetentionSeconds(30_000, 430); + expect(ttl).toBe(430); + expect(jobCancellationRetentionSeconds(300_000, 430)).toBe(780); + await commitJobResult(redis, target, { stdout: 'done' }, ttl); + const key = jobCancellationInternals.cancellationKey(target); + expect(await redis.ttl(key)).toBeGreaterThanOrEqual(429); + expect(await redis.ttl(`${key}:result`)).toBeGreaterThanOrEqual(429); +}); + +test('a late Stop renews completion evidence along with its request tombstone', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 1); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + const key = jobCancellationInternals.cancellationKey(target); + expect(await redis.ttl(key)).toBeGreaterThanOrEqual(59); + expect(await redis.ttl(`${key}:result`)).toBeGreaterThanOrEqual(59); +}); + +test('retention renewal does not lose subsecond time to rounded TTL readings', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = jobCancellationInternals.cancellationKey(target); + await redis.pexpire(key, 59_900); + const expiration = async () => + Number( + await redis.eval( + ` + local now = redis.call('TIME') + return tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + redis.call('PTTL', KEYS[1]) + `, + 1, + key, + ), + ); + const before = await expiration(); + await requestJobCancellation(redis, target, 60); + expect(await expiration()).toBeGreaterThan(before); +}); + +test('fencing returns the committed result without a vulnerable second Redis read', async () => { + const result = { stdout: 'one committed effect' }; + await commitJobResult(redis, target, result, 60); + let calls = 0; + const connectionDropsAfterDecision = { + eval: async (...args: Parameters) => { + calls += 1; + return redis.eval(...args); + }, + get: async () => { + throw new Error('connection lost after decision'); + }, + mget: async () => { + throw new Error('connection lost after decision'); + }, + } as unknown as typeof redis; + expect( + await fenceJobCancellation({ + commands: connectionDropsAfterDecision, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 1_000, + }), + ).toEqual({ status: 'completed', result }); + expect(calls).toBe(1); +}); + +test('disconnect returns a known completed result without waiting for a lost queue event', async () => { + const result = { stdout: 'done' }; + await commitJobResult(redis, target, result, 60); + const registry = new JobCancellationRegistry(redis); + const controller = new AbortController(); + controller.abort(); + const job = { + id: target.jobId, + queueName: target.queueName, + waitUntilFinished: () => new Promise(() => {}), + } as unknown as Parameters[0]['job']; + try { + expect( + await waitForJobWithCancellation({ + commands: redis, + registry, + job, + events: {} as Parameters< + typeof waitForJobWithCancellation + >[0]['events'], + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + signal: controller.signal, + }), + ).toEqual(result); + } finally { + await registry.close(); + } +}); + +test('durable cancellation wins even before its subscriber notification arrives', async () => { + expect(await requestJobCancellation(redis, target, 60)).toBe(true); + expect(await commitJobResult(redis, target, { stdout: 'late' }, 60)).toBe( + 'cancelled', + ); + expect(await readCommittedJobResult(redis, target)).toBeUndefined(); +}); + +test('committed results reject late Stop and survive a lost BullMQ completion reply', async () => { + const result = { stdout: 'one mutation', files: [] }; + expect(await commitJobResult(redis, target, result, 60)).toBe('committed'); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + expect(await readCommittedJobResult(redis, target)).toEqual({ result }); + expect( + await redis.get(jobCancellationInternals.cancellationKey(target)), + ).toBe('completed'); + const registry = new JobCancellationRegistry(redis); + const controller = new AbortController(); + try { + await registry.register(target, controller); + expect(controller.signal.aborted).toBe(false); + } finally { + await registry.close(); + } +}); + +test('concurrent cancellation and completion have exactly one winner', async () => { + const [cancelled, committed] = await Promise.all([ + requestJobCancellation(redis, target, 60), + commitJobResult(redis, target, { stdout: 'result' }, 60), + ]); + expect(Number(cancelled) + Number(committed === 'committed')).toBe(1); +}); + +test('a missing committed result fails closed instead of re-executing', async () => { + await commitJobResult(redis, target, { stdout: 'already applied' }, 60); + await redis.del(`${jobCancellationInternals.cancellationKey(target)}:result`); + await expect(readCommittedJobResult(redis, target)).rejects.toThrow( + 'refusing re-execution', + ); +}); + +test('an enqueue failure can recover a result that won cancellation fencing', async () => { + const result = { stdout: 'effect already applied' }; + await commitJobResult(redis, target, result, 60); + expect( + await fenceJobCancellation({ + commands: redis, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 5_000, + }), + ).toEqual({ status: 'completed', result }); + expect(await readCommittedJobResult(redis, target)).toEqual({ result }); +}); + +test('enqueue fencing still recovers completion after the original deadline', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + expect( + await fenceJobCancellation({ + commands: redis, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() - 1_000, + }), + ).toEqual({ status: 'completed', result: { stdout: 'done' } }); +}); + +test('Redis rejects commitment when recovery happens after the producer deadline', async () => { + const delayed = { + eval: async (...args: Parameters) => { + await new Promise(resolve => setTimeout(resolve, 150)); + return redis.eval(...args); + }, + } as unknown as typeof redis; + await expect( + commitJobResult(delayed, target, { stdout: 'late' }, 60, Date.now() + 100), + ).rejects.toThrow('exceeded its deadline'); + expect(await readCommittedJobResult(redis, target)).toBeUndefined(); +}); + +test('a timely durable commit remains successful when only its acknowledgement is late', async () => { + const delayedReply = { + eval: async (...args: Parameters) => { + const value = await redis.eval(...args); + await new Promise(resolve => setTimeout(resolve, 150)); + return value; + }, + } as unknown as typeof redis; + expect( + await commitJobResult( + delayedReply, + target, + { stdout: 'committed' }, + 60, + Date.now() + 100, + ), + ).toBe('committed'); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'committed' }, + }); +}); + +for (const failedStage of ['subscription', 'completion'] as const) { + test(`a lost ${failedStage} reply recovers the committed result instead of reporting failure`, async () => { + const result = { stdout: 'already applied once' }; + await commitJobResult(redis, target, result, 60); + const registry = new JobCancellationRegistry(redis); + if (failedStage === 'subscription') + registry.register = async () => { + throw new Error('lost reply'); + }; + const job = { + id: target.jobId, + queueName: target.queueName, + waitUntilFinished: () => Promise.reject(new Error('lost result event')), + } as unknown as Parameters[0]['job']; + try { + expect( + await waitForJobWithCancellation({ + commands: redis, + registry, + job, + events: {} as Parameters< + typeof waitForJobWithCancellation + >[0]['events'], + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + }), + ).toEqual(result); + } finally { + await registry.close(); + } + }); +} diff --git a/service/src/job-cancellation.test.ts b/service/src/job-cancellation.test.ts new file mode 100644 index 00000000..e23186e4 --- /dev/null +++ b/service/src/job-cancellation.test.ts @@ -0,0 +1,599 @@ +import { expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import type IORedis from 'ioredis'; +import type { Job, QueueEvents } from 'bullmq'; +import { + CLIENT_DISCONNECT_REASON, + JobCancellationRegistry, + jobResultCommitFailure, + jobCancellationInternals, + removeJobIfWaiting, + requestJobCancellation, + throwIfJobAborted, + waitForJobWithCancellation, + fenceJobCancellation, +} from './job-cancellation'; + +class FakeSubscriber extends EventEmitter { + subscribed?: string; + closed = false; + subscribeFailures = 0; + + async subscribe(channel: string): Promise { + if (this.subscribeFailures > 0) { + this.subscribeFailures -= 1; + throw new Error('subscriber unavailable'); + } + this.subscribed = channel; + return 1; + } + + async quit(): Promise<'OK'> { + this.closed = true; + return 'OK'; + } + + disconnect(): void { + this.closed = true; + } +} + +class FakeTransaction { + readonly operations: unknown[][] = []; + + set(...args: unknown[]): this { + this.operations.push(['set', ...args]); + return this; + } + + publish(...args: unknown[]): this { + this.operations.push(['publish', ...args]); + return this; + } + + async exec(): Promise> { + return this.operations.map(() => [null, 'OK']); + } +} + +class FakeRedis { + readonly subscriber = new FakeSubscriber(); + duplicateCalls = 0; + readonly existing = new Set(); + readonly deleted: string[] = []; + readonly transactions: FakeTransaction[] = []; + mgetFailures = 0; + cancellationFailures = 0; + cancellationAttempts = 0; + + duplicate(): FakeSubscriber { + this.duplicateCalls += 1; + return this.subscriber; + } + + async get(key: string): Promise { + return this.existing.has(key) ? '1' : null; + } + + async eval( + _script: string, + _keys: number, + key: string, + _resultKey: string, + ttl: number, + channel: string, + payload: string, + ): Promise { + this.cancellationAttempts += 1; + if (this.cancellationFailures-- > 0) throw new Error('Redis unavailable'); + const transaction = this.multi(); + transaction.set(key, '1', 'EX', ttl); + transaction.publish(channel, payload); + await transaction.exec(); + return [1]; + } + + async mget(...keys: string[]): Promise> { + if (this.mgetFailures > 0) { + this.mgetFailures -= 1; + throw new Error('command connection unavailable'); + } + return keys.map(key => (this.existing.has(key) ? '1' : null)); + } + + async del(key: string): Promise { + this.deleted.push(key); + this.existing.delete(key); + return 1; + } + + multi(): FakeTransaction { + const transaction = new FakeTransaction(); + this.transactions.push(transaction); + return transaction; + } +} + +function redis(fake: FakeRedis): IORedis { + return fake as unknown as IORedis; +} + +test('idle registries allocate no subscriber connection', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + + await registry.close(); + + expect(fake.duplicateCalls).toBe(0); +}); + +test('shutdown disconnects a subscriber whose startup is still waiting for Redis', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribe = async () => new Promise(() => {}); + const registry = new JobCancellationRegistry(redis(fake)); + void registry + .register( + { queueName: 'other', jobId: 'shutdown-startup' }, + new AbortController(), + ) + .catch(() => undefined); + await registry.close(); + expect(fake.subscriber.closed).toBe(true); + expect(fake.subscriber.listenerCount('message')).toBe(0); +}, 1_000); + +test('failed subscription startup removes handlers before a bounded retry', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + const registry = new JobCancellationRegistry(redis(fake)); + const first = new AbortController(); + + await expect( + registry.register({ queueName: 'other', jobId: 'job-failed-start' }, first), + ).rejects.toThrow('subscriber unavailable'); + expect(fake.subscriber.listenerCount('message')).toBe(0); + expect(fake.subscriber.listenerCount('ready')).toBe(0); + expect(fake.subscriber.listenerCount('error')).toBe(0); + + const second = new AbortController(); + await registry.register({ queueName: 'other', jobId: 'job-retry' }, second); + expect(fake.duplicateCalls).toBe(2); + expect(fake.subscriber.listenerCount('message')).toBe(1); + await registry.close(); +}); + +test('registry catches durable cancellation before subscriber registration', async () => { + const fake = new FakeRedis(); + const target = { queueName: 'other', jobId: 'job-1' }; + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + const registry = new JobCancellationRegistry(redis(fake)); + const controller = new AbortController(); + + await registry.register(target, controller); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + expect(fake.subscriber.subscribed).toBe(jobCancellationInternals.channel); + await registry.unregister(target); + await registry.close(); + expect(fake.subscriber.closed).toBe(true); +}); + +test('one pubsub listener cancels only the matching active job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const first = new AbortController(); + const second = new AbortController(); + await registry.register({ queueName: 'other', jobId: 'job-1' }, first); + await registry.register({ queueName: 'other', jobId: 'job-2' }, second); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-2' }), + ); + + expect(first.signal.aborted).toBe(false); + expect(second.signal.aborted).toBe(true); + await registry.close(); +}); + +test('one pubsub listener wakes every local waiter for the same job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-shared' }; + const first = new AbortController(); + const second = new AbortController(); + await registry.register(target, first); + await registry.register(target, second); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify(target), + ); + + expect(first.signal.aborted).toBe(true); + expect(second.signal.aborted).toBe(true); + expect(fake.duplicateCalls).toBe(1); + await registry.close(); +}); + +test('unregistering one local waiter preserves other waiters for the job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-shared-unregister' }; + const first = new AbortController(); + const second = new AbortController(); + await registry.register(target, first); + await registry.register(target, second); + await registry.unregister(target, first); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify(target), + ); + + expect(first.signal.aborted).toBe(false); + expect(second.signal.aborted).toBe(true); + await registry.close(); +}); + +test('subscriber reconnect reconciles active jobs against durable markers', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-reconnect' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + + fake.subscriber.emit('ready'); + await new Promise(resolve => setTimeout(resolve, 10)); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('subscriber reconnect retries durable-marker reconciliation', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-retry-reconcile' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + fake.mgetFailures = 1; + + fake.subscriber.emit('ready'); + await new Promise(resolve => setTimeout(resolve, 150)); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('terminal subscriber disconnect rebuilds the subscription and reconciles markers', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-terminal-reconnect' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + + fake.subscriber.emit('end'); + await new Promise(resolve => setTimeout(resolve, 10)); + + expect(fake.duplicateCalls).toBe(2); + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('cancellation writes a durable marker before publishing', async () => { + const fake = new FakeRedis(); + const target = { queueName: 'other', jobId: 'job-3' }; + + await requestJobCancellation(redis(fake), target, 42); + + expect(fake.transactions).toHaveLength(1); + expect(fake.transactions[0]?.operations).toEqual([ + ['set', jobCancellationInternals.cancellationKey(target), '1', 'EX', 42], + ['publish', jobCancellationInternals.channel, JSON.stringify(target)], + ]); +}); + +test('result commit barrier rejects cancellation observed after execution', () => { + const controller = new AbortController(); + expect(() => throwIfJobAborted(controller.signal)).not.toThrow(); + controller.abort(CLIENT_DISCONNECT_REASON); + expect(() => throwIfJobAborted(controller.signal)).toThrow( + CLIENT_DISCONNECT_REASON, + ); +}); + +test('result cleanup maps late cancellation to stable worker failures', () => { + const disconnected = new AbortController(); + disconnected.abort(CLIENT_DISCONNECT_REASON); + expect(jobResultCommitFailure(disconnected.signal, 30_000)?.message).toBe( + 'Job cancelled after client disconnected', + ); + + const deadline = new AbortController(); + deadline.abort('deadline'); + expect(jobResultCommitFailure(deadline.signal, 30_000)?.message).toBe( + 'Job timed out after 30000ms', + ); + expect( + jobResultCommitFailure(new AbortController().signal, 30_000), + ).toBeUndefined(); +}); + +test('disconnect frees a waiting job and rejects promptly', async () => { + const fake = new FakeRedis(); + const controller = new AbortController(); + let removed = false; + const never = new Promise(() => {}); + const job = { + id: 'job-4', + queueName: 'other', + waitUntilFinished: () => never, + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + + const registry = new JobCancellationRegistry(redis(fake)); + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + signal: controller.signal, + }); + controller.abort(CLIENT_DISCONNECT_REASON); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(removed).toBe(true); + expect(fake.cancellationAttempts).toBe(1); + expect(fake.transactions[0]?.operations[0]).toEqual([ + 'set', + jobCancellationInternals.cancellationKey({ + queueName: 'other', + jobId: 'job-4', + }), + '1', + 'EX', + 120, + ]); + await registry.close(); +}); + +test('registration failure fences and removes the already-enqueued job', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + let removed = false; + const job = { + id: 'job-register-failure', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + const registry = new JobCancellationRegistry(redis(fake)); + + await expect( + waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }), + ).rejects.toThrow('subscriber unavailable'); + + expect(removed).toBe(true); + expect(fake.transactions[0]?.operations[0]).toEqual([ + 'set', + jobCancellationInternals.cancellationKey({ + queueName: 'other', + jobId: 'job-register-failure', + }), + '1', + 'EX', + 120, + ]); + await registry.close(); +}); + +test('a result rejection is owned while subscription registration is pending', async () => { + const fake = new FakeRedis(); + let release!: () => void; + fake.subscriber.subscribe = async () => { + await new Promise(resolve => { + release = resolve; + }); + return 1; + }; + const registry = new JobCancellationRegistry(redis(fake)); + const job = { + id: 'pending-registration', + queueName: 'other', + waitUntilFinished: () => Promise.reject(new Error('completion timeout')), + getState: async () => 'active', + remove: async () => {}, + } as unknown as Job; + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + }); + const rejection = waiting.catch((error: Error) => error); + // An unowned rejection fails the test runner on this event-loop turn. + await new Promise(resolve => setImmediate(resolve)); + release(); + expect(await rejection).toMatchObject({ message: 'completion timeout' }); + await registry.close(); +}); + +test('external cancellation frees a waiting job before rejecting its waiter', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + let removed = false; + const job = { + id: 'job-external-waiting', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }); + await new Promise(resolve => setImmediate(resolve)); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-external-waiting' }), + ); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(removed).toBe(true); + await registry.close(); +}); + +test('a separate cancellation request wakes the original job waiter', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const never = new Promise(() => {}); + const job = { + id: 'job-external-cancel', + queueName: 'other', + waitUntilFinished: () => never, + getState: async () => 'active', + remove: async () => undefined, + } as unknown as Job; + + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }); + await new Promise(resolve => setImmediate(resolve)); + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-external-cancel' }), + ); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(fake.deleted).toEqual([]); + await registry.close(); +}); + +test('queued removal never removes an active job', async () => { + let removed = false; + const job = { + getState: async () => 'active' as const, + remove: async () => { + removed = true; + }, + }; + + expect(await removeJobIfWaiting(job)).toBe(false); + expect(removed).toBe(false); +}); + +test('a failed cancellation write retains ownership until a durable retry succeeds', async () => { + const fake = new FakeRedis(); + fake.cancellationFailures = 2; + const target = { queueName: 'other', jobId: 'ambiguous-enqueue' }; + let released = false; + const fencing = fenceJobCancellation({ + commands: redis(fake), + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 500, + }).then(() => { + released = true; + }); + await new Promise(resolve => setTimeout(resolve, 10)); + expect(released).toBe(false); + await fencing; + expect(fake.cancellationAttempts).toBe(3); + expect(released).toBe(true); +}); + +test('an unavailable Redis cannot release ownership before the fixed job deadline', async () => { + const fake = new FakeRedis(); + fake.cancellationFailures = 1_000; + const startedAt = Date.now(); + await fenceJobCancellation({ + commands: redis(fake), + target: { queueName: 'other', jobId: 'offline' }, + ttlSeconds: 60, + deadlineAtMs: startedAt + 80, + }); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(80); + expect(fake.cancellationAttempts).toBeLessThanOrEqual(4); +}); + +test('a pending Redis write allocates no retry backlog and waits until the deadline', async () => { + const fake = new FakeRedis(); + let calls = 0; + fake.eval = async () => { + calls += 1; + return new Promise(() => {}); + }; + const startedAt = Date.now(); + await fenceJobCancellation({ + commands: redis(fake), + target: { queueName: 'other', jobId: 'pending-write' }, + ttlSeconds: 60, + deadlineAtMs: startedAt + 40, + }); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(39); + expect(calls).toBe(1); +}); + +test('queued removal frees waiting capacity and tolerates an activation race', async () => { + let removals = 0; + expect( + await removeJobIfWaiting({ + getState: async () => 'waiting', + remove: async () => { + removals += 1; + }, + }), + ).toBe(true); + expect( + await removeJobIfWaiting({ + getState: async () => 'waiting', + remove: async () => { + removals += 1; + throw new Error('job is active'); + }, + }), + ).toBe(false); + expect(removals).toBe(2); +}); diff --git a/service/src/job-cancellation.ts b/service/src/job-cancellation.ts new file mode 100644 index 00000000..4da3253d --- /dev/null +++ b/service/src/job-cancellation.ts @@ -0,0 +1,664 @@ +import type IORedis from 'ioredis'; +import type { Job, QueueEvents } from 'bullmq'; + +const JOB_CANCELLATION_PREFIX = 'codeapi:job-cancellation:v1'; +const JOB_CANCELLATION_CHANNEL = `${JOB_CANCELLATION_PREFIX}:events`; +export const CLIENT_DISCONNECT_REASON = 'client_disconnected'; +export const JOB_CANCELLED_MESSAGE = 'Job cancelled after client disconnected'; + +interface JobTarget { + queueName: string; + jobId: string; +} + +function targetKey(target: JobTarget): string { + return `${target.queueName}:${target.jobId}`; +} + +function cancellationKey(target: JobTarget): string { + return `${JOB_CANCELLATION_PREFIX}:${encodeURIComponent( + target.queueName, + )}:${encodeURIComponent(target.jobId)}`; +} + +function parseTarget(raw: string): JobTarget | undefined { + try { + const parsed = JSON.parse(raw) as Partial; + if ( + typeof parsed.queueName !== 'string' || + parsed.queueName.length === 0 || + parsed.queueName.length > 256 || + typeof parsed.jobId !== 'string' || + parsed.jobId.length === 0 || + parsed.jobId.length > 256 + ) { + return undefined; + } + return { queueName: parsed.queueName, jobId: parsed.jobId }; + } catch { + return undefined; + } +} + +/** + * Cross-process cancellation for BullMQ work. + * + * The durable marker closes the publish-before-subscribe race while one + * process-wide pub/sub connection makes active cancellation O(events), not + * O(active jobs) Redis polling. Only explicitly cancellable replay jobs use + * this path, so ordinary queue traffic pays no extra Redis round trips. + */ +export class JobCancellationRegistry { + private subscriber?: IORedis; + private readonly controllers = new Map< + string, + { target: JobTarget; controllers: Set } + >(); + private startPromise?: Promise; + private readonly subscriberEndHandlers = new WeakMap void>(); + private reconcileTimer?: ReturnType; + private subscriberRestartTimer?: ReturnType; + private reconcileRetryMs = 100; + private closed = false; + + constructor(private readonly commands: IORedis) {} + + private readonly onSubscriberError = (): void => { + // ioredis reconnects using the shared policy. The listener prevents a + // transient subscriber outage from becoming an uncaught process error. + }; + + private readonly onSubscriberReady = (): void => { + this.scheduleReconcile(0); + }; + + private readonly onSubscriberMessage = ( + channel: string, + raw: string, + ): void => { + if (channel !== JOB_CANCELLATION_CHANNEL) return; + const target = parseTarget(raw); + if (target == null) return; + for (const controller of this.controllers.get(targetKey(target)) + ?.controllers ?? []) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + }; + + private detachSubscriber(subscriber: IORedis): void { + subscriber.removeListener('error', this.onSubscriberError); + subscriber.removeListener('ready', this.onSubscriberReady); + subscriber.removeListener('message', this.onSubscriberMessage); + const onEnd = this.subscriberEndHandlers.get(subscriber); + if (onEnd != null) subscriber.removeListener('end', onEnd); + this.subscriberEndHandlers.delete(subscriber); + } + + private restartAfterTerminalDisconnect(subscriber: IORedis): void { + if (this.closed || this.subscriber !== subscriber) return; + this.detachSubscriber(subscriber); + this.subscriber = undefined; + this.startPromise = undefined; + if (this.controllers.size === 0) return; + void this.start().then( + () => this.scheduleReconcile(0), + () => this.scheduleSubscriberRestart(), + ); + } + + private scheduleSubscriberRestart(): void { + if ( + this.closed || + this.controllers.size === 0 || + this.startPromise != null || + this.subscriberRestartTimer != null + ) + return; + const retryMs = this.reconcileRetryMs; + this.reconcileRetryMs = Math.min(2_000, retryMs * 2); + this.subscriberRestartTimer = setTimeout(() => { + this.subscriberRestartTimer = undefined; + if ( + this.closed || + this.controllers.size === 0 || + this.startPromise != null + ) + return; + void this.start().then( + () => { + this.reconcileRetryMs = 100; + this.scheduleReconcile(0); + }, + () => this.scheduleSubscriberRestart(), + ); + }, retryMs); + } + + private async reconcile(): Promise { + const entries = [...this.controllers.values()]; + if (entries.length === 0) return; + const cancelled = await this.commands.mget( + ...entries.map(({ target }) => cancellationKey(target)), + ); + cancelled.forEach((value, index) => { + if (value === '1') { + for (const controller of entries[index]?.controllers ?? []) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + } + }); + } + + private scheduleReconcile(delayMs: number): void { + if ( + this.closed || + this.controllers.size === 0 || + this.reconcileTimer != null + ) { + return; + } + this.reconcileTimer = setTimeout(() => { + this.reconcileTimer = undefined; + void this.reconcile().then( + () => { + this.reconcileRetryMs = 100; + }, + () => { + const retryMs = this.reconcileRetryMs; + this.reconcileRetryMs = Math.min(2_000, retryMs * 2); + this.scheduleReconcile(retryMs); + }, + ); + }, delayMs); + } + + private start(): Promise { + if (this.closed) { + return Promise.reject(new Error('Job cancellation registry is closed')); + } + if (this.startPromise != null) return this.startPromise; + const starting = (async (): Promise => { + const subscriber = this.commands.duplicate(); + this.subscriber = subscriber; + subscriber.on('error', this.onSubscriberError); + subscriber.on('ready', this.onSubscriberReady); + subscriber.on('message', this.onSubscriberMessage); + const onEnd = (): void => this.restartAfterTerminalDisconnect(subscriber); + this.subscriberEndHandlers.set(subscriber, onEnd); + subscriber.on('end', onEnd); + try { + await subscriber.subscribe(JOB_CANCELLATION_CHANNEL); + } catch (error) { + this.detachSubscriber(subscriber); + if (this.subscriber === subscriber) this.subscriber = undefined; + subscriber.disconnect(false); + throw error; + } + })(); + this.startPromise = starting; + void starting.catch(() => { + if (this.startPromise === starting) this.startPromise = undefined; + }); + return starting; + } + + async register( + target: JobTarget, + controller: AbortController, + ): Promise { + const key = targetKey(target); + const entry = this.controllers.get(key) ?? { + target, + controllers: new Set(), + }; + entry.controllers.add(controller); + this.controllers.set(key, entry); + try { + await this.start(); + if ((await this.commands.get(cancellationKey(target))) === '1') { + controller.abort(CLIENT_DISCONNECT_REASON); + } + } catch (error) { + entry.controllers.delete(controller); + if (entry.controllers.size === 0) this.controllers.delete(key); + throw error; + } + } + + async unregister( + target: JobTarget, + controller?: AbortController, + ): Promise { + const key = targetKey(target); + const entry = this.controllers.get(key); + if (controller == null) { + this.controllers.delete(key); + } else if (entry != null) { + entry.controllers.delete(controller); + if (entry.controllers.size === 0) this.controllers.delete(key); + } + // Markers expire by TTL. Deleting one here can erase the only evidence + // needed by another replica whose subscriber was reconnecting. + } + + async close(): Promise { + this.closed = true; + this.controllers.clear(); + if (this.reconcileTimer != null) clearTimeout(this.reconcileTimer); + this.reconcileTimer = undefined; + if (this.subscriberRestartTimer != null) + clearTimeout(this.subscriberRestartTimer); + this.subscriberRestartTimer = undefined; + const subscriber = this.subscriber; + this.subscriber = undefined; + this.startPromise = undefined; + if (subscriber == null) return; + this.detachSubscriber(subscriber); + // This socket only carries notifications. Disconnect it before awaiting + // anything: subscribe() may be queued through an indefinite Redis outage. + subscriber.disconnect(false); + } +} + +async function cancelJobInRedis( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, + includeResult: boolean, +): Promise { + // Cancellation and result publication have ONE durable winner. Pub/sub is + // only a notification; it must not decide whether Stop was accepted. + return commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == 'completed' then + -- Keep completion evidence at least as long as the requesting process + -- requires, even across API/worker config differences. Attached request + -- tombstones never renew independently of this decision. + local requestedTtlMs = tonumber(ARGV[1]) * 1000 + for i = 1, 2 do + if redis.call('PTTL', KEYS[i]) < requestedTtlMs then + redis.call('PEXPIRE', KEYS[i], requestedTtlMs) + end + end + if ARGV[4] == '1' then return {0, redis.call('GET', KEYS[2])} end + return {0} + end + if state and state ~= '1' then return {-1} end + redis.call('SET', KEYS[1], '1', 'EX', ARGV[1]) + redis.call('PUBLISH', ARGV[2], ARGV[3]) + return {1} + `, + 2, + cancellationKey(target), + `${cancellationKey(target)}:result`, + Math.max(1, ttlSeconds), + JOB_CANCELLATION_CHANNEL, + JSON.stringify(target), + includeResult ? '1' : '0', + ); +} + +export async function requestJobCancellation( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, +): Promise { + const decision = await cancelJobInRedis(commands, target, ttlSeconds, false); + if (!Array.isArray(decision) || ![0, 1].includes(decision[0])) { + throw new Error('Invalid durable cancellation decision'); + } + return decision[0] === 1; +} + +export type JobFenceOutcome = + | { status: 'cancelled' | 'expired' } + | { status: 'completed'; result: T }; + +function decodeCommittedResult(value: unknown): { result: T } { + if (typeof value !== 'string') { + throw new Error( + 'Committed programmatic result expired; refusing re-execution', + ); + } + const decoded: unknown = JSON.parse(value); + if ( + decoded == null || + typeof decoded !== 'object' || + !Object.prototype.hasOwnProperty.call(decoded, 'result') + ) { + throw new Error( + 'Invalid committed programmatic result; refusing re-execution', + ); + } + return decoded as { result: T }; +} + +export function jobCancellationRetentionSeconds( + localTimeoutMs: number, + producerTtlSeconds = 0, +): number { + return Math.max( + Math.ceil(localTimeoutMs / 1_000) * 2 + 180, + Number.isFinite(producerTtlSeconds) ? producerTtlSeconds : 0, + ); +} + +/** Retain the actual result so a BullMQ retry after a lost completion reply + * cannot repeat sandbox mutations. Keep the status small: reconnect MGETs must + * never load every active job's output into each API/worker replica. */ +export async function commitJobResult( + commands: IORedis, + target: JobTarget, + result: T, + ttlSeconds: number, + deadlineAtMs = Number.MAX_SAFE_INTEGER, +): Promise<'committed' | 'cancelled' | 'already_completed'> { + const serialized = JSON.stringify({ result }); + if (Buffer.byteLength(serialized) > 16 * 1024 * 1024) { + throw new Error('Programmatic completion exceeds the 16 MiB result limit'); + } + const decision = await commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == '1' then return 0 end + if state == 'completed' then return 2 end + if state then return -2 end + if not state then + local now = redis.call('TIME') + if tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) >= tonumber(ARGV[3]) then + return -1 + end + -- One write command, so an OOM cannot publish just half the decision. + redis.call('MSET', KEYS[1], 'completed', KEYS[2], ARGV[1]) + redis.call('EXPIRE', KEYS[1], ARGV[2]) + redis.call('EXPIRE', KEYS[2], ARGV[2]) + end + return 1 + `, + 2, + cancellationKey(target), + `${cancellationKey(target)}:result`, + serialized, + Math.max(1, ttlSeconds), + deadlineAtMs, + ); + if (decision === -1) + throw new Error('Job result commitment exceeded its deadline'); + if (decision === 1) return 'committed'; + if (decision === 0) return 'cancelled'; + if (decision === 2) return 'already_completed'; + throw new Error('Invalid durable result commitment'); +} + +/** BullMQ lock loss can redeliver a job while its first processor still runs. + * Claim once before any sandbox work, retaining the claim through the job's + * recovery horizon. An ambiguous/stalled attempt is never permission to rerun. + * Completion lookup and claim are atomic, so there is no read-then-start gap. */ +export async function claimJobExecution( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, +): Promise<{ status: 'claimed' } | { status: 'completed'; result: T }> { + const key = cancellationKey(target); + const decision = await commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == 'completed' then return {0, redis.call('GET', KEYS[2])} end + if state == '1' then return {-1} end + if state then return {-3} end + if redis.call('SET', KEYS[3], '1', 'NX', 'EX', ARGV[1]) then return {1} end + return {-2} + `, + 3, + key, + `${key}:result`, + `${key}:execution`, + Math.max(1, ttlSeconds), + ); + if (!Array.isArray(decision)) throw new Error('Invalid execution claim'); + if (decision[0] === 1) return { status: 'claimed' }; + if (decision[0] === 0) + return { + status: 'completed', + result: decodeCommittedResult(decision[1]).result, + }; + if (decision[0] === -1) throw new Error(JOB_CANCELLED_MESSAGE); + if (decision[0] === -2) + throw new Error( + 'Programmatic job already claimed; refusing duplicate execution', + ); + throw new Error('Invalid durable execution claim'); +} + +export async function readCommittedJobResult( + commands: IORedis, + target: JobTarget, +): Promise<{ result: T } | undefined> { + const [state, value] = await commands.mget( + cancellationKey(target), + `${cancellationKey(target)}:result`, + ); + if (state !== 'completed') return undefined; + return decodeCommittedResult(value); +} + +/** Do not release replay ownership on an ambiguous Redis failure. Keep one + * outstanding marker write, retry rejected writes with bounded backoff, and + * retain ownership until it succeeds or the job's ORIGINAL deadline expires. + * A delayed queue.add must carry that same timestamp into the worker. */ +export async function fenceJobCancellation(args: { + commands: IORedis; + target: JobTarget; + ttlSeconds: number; + deadlineAtMs: number; +}): Promise> { + let retryMs = 25; + let firstAttempt = true; + while (firstAttempt || Date.now() < args.deadlineAtMs) { + firstAttempt = false; + // If a lost enqueue reply arrives after the execution deadline, still + // give a healthy Redis one bounded opportunity to return completion's + // winning decision. Never translate a known committed effect to failure. + const remainingMs = args.deadlineAtMs - Date.now(); + let timer: ReturnType | undefined; + let decision: unknown; + try { + decision = await Promise.race([ + cancelJobInRedis(args.commands, args.target, args.ttlSeconds, true), + new Promise(resolve => { + timer = setTimeout( + () => resolve(undefined), + remainingMs > 0 ? remainingMs : 1_000, + ); + }), + ]); + } catch { + await new Promise(resolve => + setTimeout( + resolve, + Math.min(retryMs, Math.max(0, args.deadlineAtMs - Date.now())), + ), + ); + retryMs = Math.min(1_000, retryMs * 2); + continue; + } finally { + if (timer != null) clearTimeout(timer); + } + // Only transport failures retry. Corrupt/missing durable results are + // deterministic invariant failures, not an invitation to extend their TTL. + if (decision === undefined) return { status: 'expired' }; + if (!Array.isArray(decision)) + throw new Error('Invalid durable cancellation decision'); + if (decision[0] === 1) return { status: 'cancelled' }; + if (decision[0] === 0) + return { + status: 'completed', + result: decodeCommittedResult(decision[1]).result, + }; + throw new Error('Invalid durable cancellation decision'); + } + return { status: 'expired' }; +} + +const REMOVABLE_JOB_STATES = new Set([ + 'waiting', + 'delayed', + 'prioritized', + 'waiting-children', +]); + +/** Frees queued capacity without ever removing an active or settled job. */ +export async function removeJobIfWaiting( + job: Pick, +): Promise { + if (!REMOVABLE_JOB_STATES.has(await job.getState())) return false; + try { + await job.remove(); + return true; + } catch { + // A worker may have activated the job between getState() and remove(). + // The durable marker remains authoritative for that race. + return false; + } +} + +export function programmaticCancellationError(): Error { + return new DOMException( + 'Programmatic execution request disconnected', + 'AbortError', + ); +} + +/** Commit barrier for result-processing stages that may yield after execution. */ +export function throwIfJobAborted(signal: AbortSignal): void { + if (!signal.aborted) return; + if (signal.reason instanceof Error) throw signal.reason; + throw new DOMException( + typeof signal.reason === 'string' ? signal.reason : 'Job aborted', + 'AbortError', + ); +} + +/** Maps cancellation observed during asynchronous result cleanup to the same + * stable worker failure used by the main execution catch path. */ +export function jobResultCommitFailure( + signal: AbortSignal, + jobTimeoutMs: number, +): Error | undefined { + if (!signal.aborted) return undefined; + return new Error( + signal.reason === CLIENT_DISCONNECT_REASON + ? JOB_CANCELLED_MESSAGE + : `Job timed out after ${jobTimeoutMs}ms`, + ); +} + +export async function waitForJobWithCancellation(args: { + commands: IORedis; + registry: JobCancellationRegistry; + job: Job; + events: QueueEvents; + timeoutMs: number; + cancellationTtlSeconds: number; + deadlineAtMs?: number; + signal?: AbortSignal; +}): Promise { + const { + commands, + registry, + job, + events, + timeoutMs, + cancellationTtlSeconds, + signal, + } = args; + const completion = job.waitUntilFinished(events, timeoutMs); + // Subscription startup can itself wait for Redis recovery. Own the losing + // promise immediately, before any await, rather than after registration. + void completion.catch(() => undefined); + const target = { queueName: job.queueName, jobId: String(job.id) }; + const deadlineAtMs = args.deadlineAtMs ?? Date.now() + timeoutMs; + let fencing: Promise> | undefined; + const fence = (): Promise> => + (fencing ??= fenceJobCancellation({ + commands, + target, + ttlSeconds: cancellationTtlSeconds, + deadlineAtMs, + })); + const externalController = new AbortController(); + try { + await registry.register(target, externalController); + } catch (error) { + void completion.catch(() => undefined); + const outcome = await fence(); + if (outcome.status === 'completed') return outcome.result; + await removeJobIfWaiting(job).catch(() => false); + throw error; + } + + let removeAbortListener = (): void => {}; + const disconnected = new Promise((resolve, reject) => { + let cancelling = false; + const cancel = (): void => { + if (cancelling) return; + cancelling = true; + void fence() + .then(async outcome => { + if (outcome.status === 'completed') { + resolve(outcome.result); + return; + } + // Removing a waiting job immediately frees queue capacity. An active + // job cannot be removed; its worker observes the durable marker or + // pub/sub event and aborts the sandbox transport instead. + await removeJobIfWaiting(job).catch(() => false); + reject(programmaticCancellationError()); + }) + .catch(reject); + }; + if (signal != null) { + removeAbortListener = (): void => + signal.removeEventListener('abort', cancel); + signal.addEventListener('abort', cancel, { once: true }); + if (signal.aborted) cancel(); + } + }); + const cancelled = new Promise((_, reject) => { + const cancel = (): void => { + void removeJobIfWaiting(job).then( + () => reject(programmaticCancellationError()), + () => reject(programmaticCancellationError()), + ); + }; + externalController.signal.addEventListener('abort', cancel, { + once: true, + }); + if (externalController.signal.aborted) cancel(); + }); + + // A cancelled request stops awaiting the BullMQ result, so attach a sink to + // the losing promise before racing it to avoid an unhandled late rejection. + void completion.catch(() => undefined); + try { + return await Promise.race([completion, disconnected, cancelled]); + } catch (error) { + // Includes waitUntilFinished timeouts and registration/transport errors, + // not only explicit Stop. Replay cleanup is unsafe until this barrier. + const outcome = await fence(); + if (outcome.status === 'completed') return outcome.result; + throw error; + } finally { + removeAbortListener(); + await registry + .unregister(target, externalController) + .catch(() => undefined); + } +} + +export const jobCancellationInternals = { + channel: JOB_CANCELLATION_CHANNEL, + cancellationKey, + parseTarget, +}; diff --git a/service/src/metrics.ts b/service/src/metrics.ts index adfd9872..42c5536f 100644 --- a/service/src/metrics.ts +++ b/service/src/metrics.ts @@ -117,6 +117,12 @@ export const jobsFailed = new Counter({ labelNames: ['language'] as const, }); +export const jobsCancelled = new Counter({ + name: 'codeapi_jobs_cancelled_total', + help: 'Total number of jobs cancelled after the calling client disconnected', + labelNames: ['language'] as const, +}); + export const activeJobs = new Gauge({ name: 'codeapi_active_jobs', help: 'Number of jobs currently being processed', diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index 099261a1..9ffaeb9d 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -182,6 +182,19 @@ export const executionLimiter = createRateLimiter( } ); +/** Keep Stop available when execution admission is full, while independently + * bounding request-id churn in the cancellation registry. */ +export const cancellationLimiter = createRateLimiter( + 'exec-cancel', + env.EXEC_LIMIT_WINDOW, + Math.max(80, env.EXEC_MAX_REQUESTS * 4), + { + message: 'Too many CodeAPI cancellation requests.', + structuredBody: true, + logRejections: true, + } +); + export const uploadLimiter = createRateLimiter( 'upload', env.UPLOAD_LIMIT_WINDOW, diff --git a/service/src/programmatic-cancellation.test.ts b/service/src/programmatic-cancellation.test.ts new file mode 100644 index 00000000..53a4340e --- /dev/null +++ b/service/src/programmatic-cancellation.test.ts @@ -0,0 +1,234 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test'; +import type IORedis from 'ioredis'; +import { startTestRedis } from './test/redis'; +import { commitJobResult, requestJobCancellation } from './job-cancellation'; +import { + attachProgrammaticCancellationTarget, + cancelProgrammaticRequest, + normalizeProgrammaticRequestId, + programmaticCancellationInternals, + releaseProgrammaticCancellation, + reserveProgrammaticCancellation, +} from './programmatic-cancellation'; + +let redis: IORedis & { closeTestServer(): Promise }; + +beforeEach(async () => { + redis = await startTestRedis(); +}); + +afterEach(async () => { + await redis.closeTestServer(); +}); + +test('normalizes only bounded opaque request IDs', () => { + expect(normalizeProgrammaticRequestId('request_123456789')).toBe( + 'request_123456789', + ); + expect(normalizeProgrammaticRequestId(' short ')).toBeUndefined(); + expect( + normalizeProgrammaticRequestId('../request_123456789'), + ).toBeUndefined(); + expect(normalizeProgrammaticRequestId('a'.repeat(129))).toBeUndefined(); +}); + +test('Stop cannot extend an attached tombstone before the outcome command succeeds', async () => { + const requestId = 'request_no_split_renewal'; + const owner = 'owner-a'; + const target = { queueName: 'other', jobId: 'split-renewal' }; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }); + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target, + ttlSeconds: 60, + }); + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = programmaticCancellationInternals.requestKey(requestId); + await redis.pexpire(key, 5_000); + const before = await redis.pttl(key); + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 600, + }), + ).toEqual({ status: 'accepted', target }); + // Simulate losing Redis before requestJobCancellation: no second command. + expect(await redis.pttl(key)).toBeLessThanOrEqual(before); + expect(await requestJobCancellation(redis, target, 600)).toBe(false); +}); + +test('cancellation before queue attachment is retained atomically', async () => { + const requestId = 'request_early_cancel_123'; + const owner = 'owner-a'; + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toEqual({ status: 'accepted' }); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('cancelled'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target: { queueName: 'other', jobId: '42' }, + ttlSeconds: 60, + }), + ).toBe('cancelled'); +}); + +test('cancellation after attachment returns the exact queue target', async () => { + const requestId = 'request_attached_cancel_1'; + const owner = 'owner-a'; + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('active'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target: { queueName: 'other', jobId: '43' }, + ttlSeconds: 60, + }), + ).toBe('active'); + + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toEqual({ + status: 'accepted', + target: { queueName: 'other', jobId: '43' }, + }); +}); + +test('overlapping requests from the same owner cannot share cancellation state', async () => { + const requestId = 'request_duplicate_owner_1'; + const owner = 'owner-a'; + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('active'); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('duplicate'); +}); + +test('a different principal cannot reserve, attach, cancel, or release a request', async () => { + const requestId = 'request_owned_cancel_123'; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-b', + ttlSeconds: 60, + }), + ).toBe('forbidden'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner: 'owner-b', + target: { queueName: 'other', jobId: '44' }, + ttlSeconds: 60, + }), + ).toBe('forbidden'); + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner: 'owner-b', + ttlSeconds: 60, + }), + ).toEqual({ status: 'forbidden' }); + await releaseProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-b', + }); + expect( + await redis.exists(programmaticCancellationInternals.requestKey(requestId)), + ).toBe(1); +}); + +test('settlement retains a bounded target tombstone for late Stop classification', async () => { + const requestId = 'request_release_cancel_1'; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + const target = { queueName: 'other', jobId: 'settled-job' }; + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner: 'owner-a', + target, + ttlSeconds: 60, + }); + await commitJobResult(redis, target, { stdout: 'done' }, 60); + await releaseProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + }); + const key = programmaticCancellationInternals.requestKey(requestId); + expect(await redis.exists(key)).toBe(1); + expect(await redis.ttl(key)).toBeGreaterThan(0); + expect(await redis.ttl(key)).toBeLessThanOrEqual(60); + const cancelled = await cancelProgrammaticRequest({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + expect(cancelled).toEqual({ status: 'accepted', target }); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); +}); diff --git a/service/src/programmatic-cancellation.ts b/service/src/programmatic-cancellation.ts new file mode 100644 index 00000000..12a6694d --- /dev/null +++ b/service/src/programmatic-cancellation.ts @@ -0,0 +1,182 @@ +import { createHash } from 'node:crypto'; +import type IORedis from 'ioredis'; +import type { AuthenticatedRequest } from './types'; +import { getCredentialId } from './auth/principal'; +import { getExecutionIdentity } from './execution-identity'; + +export const CODEAPI_PROGRAMMATIC_REQUEST_HEADER = + 'X-LibreChat-Code-Request-ID'; +const REQUEST_PREFIX = 'codeapi:programmatic-cancellation:v1'; +const REQUEST_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; + +interface CancellationTarget { + queueName: string; + jobId: string; +} + +export type CancellationRequestResult = + | { status: 'accepted'; target?: CancellationTarget } + | { status: 'forbidden' }; + +function requestKey(requestId: string): string { + return `${REQUEST_PREFIX}:${requestId}`; +} + +export function normalizeProgrammaticRequestId( + value: unknown, +): string | undefined { + if (typeof value !== 'string') return undefined; + const trimmed = value.trim(); + return REQUEST_ID_PATTERN.test(trimmed) ? trimmed : undefined; +} + +export function programmaticCancellationOwner( + req: AuthenticatedRequest, + userId: string, +): string { + const identity = getExecutionIdentity(req, userId); + return createHash('sha256') + .update( + JSON.stringify([ + identity.storageNamespace, + identity.canonicalUserId, + getCredentialId(req), + identity.authContextHash ?? '', + ]), + ) + .digest('hex'); +} + +const RESERVE_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local ttl = tonumber(ARGV[2]) +local existing = redis.call('HGET', key, 'owner') +if existing and existing ~= owner then return -1 end +if redis.call('HGET', key, 'reserved') == '1' then return -2 end +if not existing then + redis.call('HSET', key, 'owner', owner, 'cancelled', '0') +end +redis.call('HSET', key, 'reserved', '1') +redis.call('EXPIRE', key, ttl) +return tonumber(redis.call('HGET', key, 'cancelled') or '0') +`; + +const ATTACH_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local queueName = ARGV[2] +local jobId = ARGV[3] +local ttl = tonumber(ARGV[4]) +if redis.call('HGET', key, 'owner') ~= owner then return -1 end +redis.call('HSET', key, 'queueName', queueName, 'jobId', jobId) +redis.call('EXPIRE', key, ttl) +return tonumber(redis.call('HGET', key, 'cancelled') or '0') +`; + +const CANCEL_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local ttl = tonumber(ARGV[2]) +local existing = redis.call('HGET', key, 'owner') +if existing and existing ~= owner then return {-1} end +if not existing then redis.call('HSET', key, 'owner', owner) end +redis.call('HSET', key, 'cancelled', '1') +local queueName = redis.call('HGET', key, 'queueName') +local jobId = redis.call('HGET', key, 'jobId') +-- Once attached, never extend this mapping independently of the job decision. +-- Its original admission TTL already covers execution and late cancellation. +if queueName and jobId then return {1, queueName, jobId} end +redis.call('EXPIRE', key, ttl) +return {1} +`; + +const RELEASE_SCRIPT = ` +if redis.call('HGET', KEYS[1], 'owner') == ARGV[1] then + -- Keep the owner/target tombstone through its existing bounded TTL. A Stop + -- racing response delivery must still reach the job's completion decision. + return redis.call('HSET', KEYS[1], 'finished', '1') +end +return 0 +`; + +export async function reserveProgrammaticCancellation(args: { + redis: IORedis; + requestId: string; + owner: string; + ttlSeconds: number; +}): Promise<'active' | 'cancelled' | 'duplicate' | 'forbidden'> { + const result = Number( + await args.redis.eval( + RESERVE_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + Math.max(1, args.ttlSeconds), + ), + ); + if (result === -1) return 'forbidden'; + if (result === -2) return 'duplicate'; + return result === 1 ? 'cancelled' : 'active'; +} + +export async function attachProgrammaticCancellationTarget(args: { + redis: IORedis; + requestId: string; + owner: string; + target: CancellationTarget; + ttlSeconds: number; +}): Promise<'active' | 'cancelled' | 'forbidden'> { + const result = Number( + await args.redis.eval( + ATTACH_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + args.target.queueName, + args.target.jobId, + Math.max(1, args.ttlSeconds), + ), + ); + if (result < 0) return 'forbidden'; + return result === 1 ? 'cancelled' : 'active'; +} + +export async function cancelProgrammaticRequest(args: { + redis: IORedis; + requestId: string; + owner: string; + ttlSeconds: number; +}): Promise { + const raw = await args.redis.eval( + CANCEL_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + Math.max(1, args.ttlSeconds), + ); + const result = Array.isArray(raw) ? raw.map(String) : []; + if (result[0] === '-1') return { status: 'forbidden' }; + if (result.length >= 3) { + return { + status: 'accepted', + target: { queueName: result[1]!, jobId: result[2]! }, + }; + } + return { status: 'accepted' }; +} + +export async function releaseProgrammaticCancellation(args: { + redis: IORedis; + requestId: string; + owner: string; +}): Promise { + await args.redis.eval( + RELEASE_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + ); +} + +export const programmaticCancellationInternals = { requestKey }; diff --git a/service/src/queue.ts b/service/src/queue.ts index 54fea308..72c2fbb1 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -1,6 +1,7 @@ // src/queue.ts import IORedis from 'ioredis'; import { Queue, QueueEvents } from 'bullmq'; +import type { Job } from 'bullmq'; import { setMaxListeners } from 'events'; import type { CommonRedisOptions } from 'ioredis'; import type * as tls from 'tls'; @@ -17,19 +18,13 @@ import type { SandboxBackendName, } from './execution-profile'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; +import { redisKeepAliveOptions, redisReconnectDelay } from './redis-options'; import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; - -const MAX_RECONNECT_ATTEMPTS = 5; -const RECONNECT_DELAY = 2000; +import { JobCancellationRegistry } from './job-cancellation'; const retryStrategy: CommonRedisOptions['retryStrategy'] = (times) => { - if (times > MAX_RECONNECT_ATTEMPTS) { - logger.error(`Failed to connect to Redis after ${times} attempts`); - return null; - } logger.warn(`Retrying Redis connection attempt ${times}`); - return RECONNECT_DELAY; + return redisReconnectDelay(times); }; const reconnectOnError: CommonRedisOptions['reconnectOnError'] = (err) => { @@ -60,6 +55,7 @@ const connection = new IORedis({ ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } : {}) }); +const jobCancellationRegistry = new JobCancellationRegistry(connection); // Global queues - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job @@ -110,6 +106,19 @@ export function getExecutionQueueBinding( return { ...getQueueResources(name), language }; } +/** + * Resolve a job only from this deployment's already-open queue set. Every + * homogeneous API replica opens both execution queues at startup, so this + * supports cross-replica cancellation without allocating attacker-shaped + * QueueEvents connections for arbitrary names recovered from Redis. + */ +export async function getExistingExecutionJob( + queueName: string, + jobId: string, +): Promise | undefined> { + return queueResources.get(queueName)?.queue.getJob(jobId); +} + const { queue: pyQueue, events: pyQueueEvents } = getQueueResources(queueNames.python); const { queue: otherQueue, events: otherQueueEvents } = getQueueResources(queueNames.other); @@ -163,8 +172,16 @@ export async function closeQueueConnections(): Promise { [...queueResources.values()].flatMap(({ queue, events }) => [ queue.close(), events.close(), - ]), + ]).concat(jobCancellationRegistry.close()), ); } -export { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection }; +export { + pyQueue, + otherQueue, + pyQueueEvents, + otherQueueEvents, + queueNames, + connection, + jobCancellationRegistry, +}; diff --git a/service/src/redis-options.test.ts b/service/src/redis-options.test.ts index 1bb942e6..9795e279 100644 --- a/service/src/redis-options.test.ts +++ b/service/src/redis-options.test.ts @@ -1,5 +1,15 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { redisKeepAliveMs, redisKeepAliveOptions } from './redis-options'; +import { + redisKeepAliveMs, + redisKeepAliveOptions, + redisReconnectDelay, +} from './redis-options'; + +test('long-lived command connections keep recovering with a bounded retry delay', () => { + expect(redisReconnectDelay(1)).toBe(100); + expect(redisReconnectDelay(6)).toBe(600); + expect(redisReconnectDelay(1_000)).toBe(2_000); +}); describe('Redis keepalive options', () => { afterEach(() => { diff --git a/service/src/redis-options.ts b/service/src/redis-options.ts index 98d3f5b1..1f2c8e42 100644 --- a/service/src/redis-options.ts +++ b/service/src/redis-options.ts @@ -1,5 +1,11 @@ import type { CommonRedisOptions } from 'ioredis'; +/** Long-lived queue/cancellation command and subscriber connections must both + * recover after an outage. Never leave a live process with a terminal client. */ +export function redisReconnectDelay(attempt: number): number { + return Math.min(2_000, 100 * Math.max(1, attempt)); +} + export function redisKeepAliveMs(): number { const raw = process.env.REDIS_KEEP_ALIVE_MS; const trimmed = raw?.trim(); diff --git a/service/src/request-disconnect.test.ts b/service/src/request-disconnect.test.ts new file mode 100644 index 00000000..15660aca --- /dev/null +++ b/service/src/request-disconnect.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import type { Response } from 'express'; +import type { AuthenticatedRequest } from './types'; +import { CLIENT_DISCONNECT_REASON } from './job-cancellation'; +import { observeRequestDisconnect } from './request-disconnect'; + +function requestAndResponse(options: { + requestAborted?: boolean; + requestDestroyed?: boolean; + responseDestroyed?: boolean; +} = {}): { + req: AuthenticatedRequest & EventEmitter; + res: Response & EventEmitter; +} { + const req = Object.assign(new EventEmitter(), { + aborted: options.requestAborted ?? false, + destroyed: options.requestDestroyed ?? false, + }) as AuthenticatedRequest & EventEmitter; + const res = Object.assign(new EventEmitter(), { + destroyed: options.responseDestroyed ?? false, + writableFinished: false, + }) as Response & EventEmitter; + return { req, res }; +} + +test('a consumed Bun request stream is not mistaken for a disconnect', () => { + const { req, res } = requestAndResponse({ requestDestroyed: true }); + const observer = observeRequestDisconnect(req, res); + + expect(observer.isDisconnected()).toBe(false); + expect(observer.signal.aborted).toBe(false); + observer.dispose(); +}); + +test('current and future transport abandonment abort exactly once', () => { + const current = requestAndResponse({ requestAborted: true }); + const currentObserver = observeRequestDisconnect(current.req, current.res); + expect(currentObserver.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + + const future = requestAndResponse(); + const futureObserver = observeRequestDisconnect(future.req, future.res); + future.res.emit('close'); + future.req.emit('aborted'); + expect(futureObserver.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + expect(future.req.listenerCount('aborted')).toBe(0); + expect(future.res.listenerCount('close')).toBe(0); +}); + +test('a completed response disposes listeners without aborting', () => { + const { req, res } = requestAndResponse(); + const observer = observeRequestDisconnect(req, res); + (res as unknown as { writableFinished: boolean }).writableFinished = true; + res.emit('finish'); + res.emit('close'); + + expect(observer.isDisconnected()).toBe(false); + expect(req.listenerCount('aborted')).toBe(0); + expect(res.listenerCount('close')).toBe(0); +}); diff --git a/service/src/request-disconnect.ts b/service/src/request-disconnect.ts new file mode 100644 index 00000000..2a4d7a9d --- /dev/null +++ b/service/src/request-disconnect.ts @@ -0,0 +1,49 @@ +import type { Response } from 'express'; +import type { AuthenticatedRequest } from './types'; +import { CLIENT_DISCONNECT_REASON } from './job-cancellation'; + +export interface RequestDisconnectObserver { + signal: AbortSignal; + isDisconnected(): boolean; + dispose(): void; +} + +/** + * Observe a genuinely abandoned HTTP response across Node and Bun. + * + * Bun may mark the consumed IncomingMessage stream as `destroyed` while the + * response remains healthy, so request stream destruction is deliberately not + * treated as a disconnect. Express' `aborted` event and ServerResponse's + * pre-finish `close` event are the portable abandonment signals. + */ +export function observeRequestDisconnect( + req: AuthenticatedRequest, + res: Response, +): RequestDisconnectObserver { + const controller = new AbortController(); + let disposed = false; + const dispose = (): void => { + if (disposed) return; + disposed = true; + req.removeListener('aborted', disconnect); + res.removeListener('close', disconnect); + res.removeListener('finish', dispose); + }; + const disconnect = (): void => { + if (!res.writableFinished && !controller.signal.aborted) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + dispose(); + }; + + req.once('aborted', disconnect); + res.once('close', disconnect); + res.once('finish', dispose); + if (req.aborted || res.destroyed) disconnect(); + + return { + signal: controller.signal, + isDisconnected: () => controller.signal.aborted, + dispose, + }; +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 93bbc0c5..bb140013 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -4,13 +4,32 @@ import { Router } from 'express'; import type { Response } from 'express'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; -import { executionLimiter } from '../middleware/limits'; +import { cancellationLimiter, executionLimiter } from '../middleware/limits'; import { pyQueue, pyQueueEvents, connection, + jobCancellationRegistry, getExecutionQueueBinding, + getExistingExecutionJob, } from '../queue'; +import { + JOB_CANCELLED_MESSAGE, + programmaticCancellationError, + removeJobIfWaiting, + requestJobCancellation, + fenceJobCancellation, + waitForJobWithCancellation, +} from '../job-cancellation'; +import { + CODEAPI_PROGRAMMATIC_REQUEST_HEADER, + attachProgrammaticCancellationTarget, + cancelProgrammaticRequest, + normalizeProgrammaticRequestId, + programmaticCancellationOwner, + releaseProgrammaticCancellation, + reserveProgrammaticCancellation, +} from '../programmatic-cancellation'; import { createProgrammaticPayload, extractPendingFromControlPayload, @@ -38,6 +57,7 @@ import { Jobs } from '../enum'; import { env, jobCompletionWaitTimeoutMs } from '../config'; import { resolveQueuedSandboxBackend } from '../execution-profile'; import { publicExecutionFailure } from '../utils'; +import { observeRequestDisconnect } from '../request-disconnect'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, @@ -107,6 +127,18 @@ const JOB_COMPLETION_WAIT_TIMEOUT_MS = jobCompletionWaitTimeoutMs( env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS, env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, ); +const PROGRAMMATIC_CANCELLATION_TTL_SECONDS = + Math.ceil(JOB_COMPLETION_WAIT_TIMEOUT_MS / 1000) + 60; + +interface ReplayRequestCancellation { + signal: AbortSignal; + isDisconnected(): boolean; + request?: { + requestId: string; + owner: string; + cancelledBeforeStart: boolean; + }; +} const router = Router(); @@ -322,7 +354,10 @@ async function runReplayIteration( state: ExecutionState, apiKeyId: string, userId: string, + signal?: AbortSignal, + cancellation?: { requestId: string; owner: string }, ): Promise { + if (signal?.aborted) throw programmaticCancellationError(); const history = await loadToolHistory(state.execution_id); const rawPayload = buildReplayPayload(req, state, history); const sessionKey = state.sessionKey ?? state.userId; @@ -369,41 +404,83 @@ async function runReplayIteration( state.executionProfile ?? env.EXECUTION_PROFILE, state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, ); - const job = await queue.add( - Jobs.execute, - { - code: state.userCode ?? '', - userId, - payload: sandboxSecurity.payload, - apiKeyId, - isPyPlot: state.isPyPlot ?? false, - principalSource: state.principalSource, - executionId: state.execution_id, - tenantId: state.tenantId, - canonicalUserId: state.canonicalUserId, - executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, - sandboxBackend: replayBackend, - ...(state.bridgeWorkerId != null - ? { bridgeWorkerId: state.bridgeWorkerId } - : {}), - ...(state.workspaceId != null - ? { workspaceId: state.workspaceId } - : {}), - runtimeSessionMode: 'stateless', - runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, - executionManifestClaims: sandboxSecurity.executionManifestClaims, - egressGrantClaims: sandboxSecurity.egressGrantClaims, - egressGrantToken: sandboxSecurity.egressGrantToken, - }, - { - removeOnComplete: { age: 60, count: 1 }, - removeOnFail: { age: 180, count: 1 }, - attempts: 1, - }, + if (signal?.aborted) throw programmaticCancellationError(); + const cancellationTarget = { queueName: queue.name, jobId: nanoid() }; + if (cancellation != null) { + const attachment = await attachProgrammaticCancellationTarget({ + redis: connection, + requestId: cancellation.requestId, + owner: cancellation.owner, + target: cancellationTarget, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + if (attachment === 'forbidden') { + throw new Error('Programmatic cancellation request ownership changed'); + } + if (attachment === 'cancelled') { + throw programmaticCancellationError(); + } + } + const submittedAtMs = Date.now(); + const deadlineAtMs = submittedAtMs + env.JOB_TIMEOUT; + let job: Awaited>; + try { + job = await queue.add( + Jobs.execute, + { + code: state.userCode ?? '', + userId, + payload: sandboxSecurity.payload, + apiKeyId, + isPyPlot: state.isPyPlot ?? false, + principalSource: state.principalSource, + executionId: state.execution_id, + tenantId: state.tenantId, + canonicalUserId: state.canonicalUserId, + executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, + sandboxBackend: replayBackend, + ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), + ...(state.workspaceId != null ? { workspaceId: state.workspaceId } : {}), + cancellable: true, + deadlineAtMs, + cancellationTtlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + runtimeSessionMode: 'stateless', + runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, + executionManifestClaims: sandboxSecurity.executionManifestClaims, + egressGrantClaims: sandboxSecurity.egressGrantClaims, + egressGrantToken: sandboxSecurity.egressGrantToken, + }, + { + removeOnComplete: { age: 60, count: 1 }, + removeOnFail: { age: 180, count: 1 }, + attempts: 1, + jobId: cancellationTarget.jobId, + timestamp: submittedAtMs, + }, ); + } catch (error) { + // Redis may have enqueued the job even though its reply was lost. + // Preserve replay ownership until cancellation is durable or the job's + // fixed worker deadline prevents a late admission from executing. + const outcome = await fenceJobCancellation({ + commands: connection, target: cancellationTarget, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, deadlineAtMs, + }); + if (outcome.status === 'completed') return outcome.result; + throw error; + } jobsSubmitted.inc({ language }); - return job.waitUntilFinished(events, JOB_COMPLETION_WAIT_TIMEOUT_MS); + return waitForJobWithCancellation({ + commands: connection, + registry: jobCancellationRegistry, + job, + events, + timeoutMs: JOB_COMPLETION_WAIT_TIMEOUT_MS, + cancellationTtlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + deadlineAtMs, + signal, + }); } function isSandboxRunSuccess(result: t.ExecuteResult): boolean { @@ -425,6 +502,7 @@ async function handleReplayInitial( bridgeWorkerId?: string; workspaceId?: string; }, + cancellation: ReplayRequestCancellation, ): Promise { const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; const { code, tools, user_id, files } = @@ -543,6 +621,19 @@ async function handleReplayInitial( throw error; } + if ( + cancellation.signal.aborted || + cancellation.request?.cancelledBeforeStart === true + ) { + if (!cancellation.isDisconnected()) { + res.status(200).json({ + status: 'error', + error: 'Programmatic execution request cancelled', + }); + } + return; + } + const session_id = nanoid(); const execution_id = nanoid(); const authContext = req.codeApiAuthContext; @@ -622,7 +713,7 @@ async function handleReplayInitial( timeout, }); - await runAndRespond(req, res, state, apiKeyId, userId); + await runAndRespond(req, res, state, apiKeyId, userId, cancellation); } async function handleReplayContinuation( @@ -634,6 +725,7 @@ async function handleReplayContinuation( decoded: { execution_id: string }; tool_results: NonNullable; }, + cancellation: ReplayRequestCancellation, ): Promise { const { apiKeyId, userId, decoded, tool_results } = params; @@ -692,6 +784,20 @@ async function handleReplayContinuation( res.status(404).json({ error: 'Execution not found or expired' }); return; } + if ( + cancellation.signal.aborted || + cancellation.request?.cancelledBeforeStart === true + ) { + await cleanupExecution(state.execution_id, 'replay'); + if (!cancellation.isDisconnected()) { + res.status(200).json({ + status: 'error', + error: 'Programmatic execution request cancelled', + session_id: state.session_id, + }); + } + return; + } /** Compute the delta against already-persisted history first so the * cap checks see the real impact of this batch (new call_ids only * advance `callCount`; overwrites may shrink or grow `historyBytes` @@ -845,7 +951,14 @@ async function handleReplayContinuation( }); } - await runAndRespond(req, res, state, apiKeyId, userId); + await runAndRespond( + req, + res, + state, + apiKeyId, + userId, + cancellation, + ); } finally { await releaseExecutionLock(decoded.execution_id, lockToken); } @@ -857,29 +970,29 @@ async function runAndRespond( state: ExecutionState, apiKeyId: string, userId: string, + cancellation: ReplayRequestCancellation, ): Promise { - /** Read disconnect state through `isDisconnected()` rather than a - * direct boolean. The `req.on('close', ...)` handler flips the flag - * during awaits, but `@typescript-eslint/no-unnecessary-condition` - * (correctly per TS semantics) narrows a directly-mutated `let`/object - * member to its literal value after an early-return `if (...) return`, - * even across awaits. A function call is opaque to that narrowing. */ - let disconnected = false; - const isDisconnected = (): boolean => disconnected; - req.on('close', () => { - if (!res.writableEnded) disconnected = true; - }); - let result: t.ExecuteResult; try { - result = await runReplayIteration(req, state, apiKeyId, userId); + result = await runReplayIteration( + req, + state, + apiKeyId, + userId, + cancellation.signal, + cancellation.request, + ); } catch (err) { - logger.error('Replay iteration failed', { - execution_id: state.execution_id, - err, - }); + const cancelled = + (err as Error).name === 'AbortError' || + (err as Error).message === JOB_CANCELLED_MESSAGE; + logger.log(cancelled ? 'info' : 'error', 'Replay iteration failed', { + execution_id: state.execution_id, + cancelled, + err, + }); await cleanupExecution(state.execution_id, 'replay'); - if (!isDisconnected()) { + if (!cancellation.isDisconnected()) { const publicFailure = publicExecutionFailure(err); const message = publicFailure?.body.message ?? (err as Error).message; @@ -892,7 +1005,7 @@ async function runAndRespond( return; } - if (isDisconnected()) { + if (cancellation.isDisconnected()) { logger.info('Client disconnected during replay; cleaning up', { execution_id: state.execution_id, }); @@ -1002,7 +1115,7 @@ async function runAndRespond( await cleanupExecution(state.execution_id, 'replay').catch( () => {}, ); - if (!isDisconnected()) { + if (!cancellation.isDisconnected()) { if (err instanceof ExecutionStateTooLargeError) { /** A continuation that pushes `emittedCallIds` past the * `MAX_EXECUTION_STATE_BYTES` cap is a client-input sizing @@ -1076,6 +1189,67 @@ async function runAndRespond( // Request entrypoint // --------------------------------------------------------------------------- +router.post( + '/exec/programmatic/cancel', + cancellationLimiter, + async (req: t.AuthenticatedRequest, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + const requestId = normalizeProgrammaticRequestId( + (req.body as Record)?.request_id, + ); + if (requestId == null) { + res.status(400).json({ error: 'Invalid or missing request_id' }); + return; + } + try { + const owner = programmaticCancellationOwner(req, principal.userId); + const cancellation = await cancelProgrammaticRequest({ + redis: connection, + requestId, + owner, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + if (cancellation.status === 'forbidden') { + res.status(403).json({ error: 'Programmatic request belongs to another principal' }); + return; + } + if (cancellation.target != null) { + const accepted = await requestJobCancellation( + connection, + cancellation.target, + PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + ); + if (!accepted) { + res.status(200).json({ status: 'already_completed' }); + return; + } + try { + const queuedJob = await getExistingExecutionJob( + cancellation.target.queueName, + cancellation.target.jobId, + ); + if (queuedJob != null) await removeJobIfWaiting(queuedJob); + } catch (error) { + logger.warn('Failed to remove cancelled waiting execution', { + requestId, + queueName: cancellation.target?.queueName, + jobId: cancellation.target?.jobId, + error: (error as Error).message, + }); + } + } + res.status(202).json({ status: 'cancellation_requested' }); + } catch (error) { + logger.error('Failed to request programmatic execution cancellation', { + requestId, + error: (error as Error).message, + }); + res.status(503).json({ error: 'Cancellation service unavailable' }); + } + }, +); + router.post( '/exec/programmatic', executionLimiter, @@ -1095,6 +1269,11 @@ router.post( const { continuation_token, tool_results } = req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; + const rawRequestId = req.header(CODEAPI_PROGRAMMATIC_REQUEST_HEADER); + const requestId = normalizeProgrammaticRequestId(rawRequestId); + if (rawRequestId != null && requestId == null) { + return res.status(400).json({ error: 'Invalid programmatic request ID' }); + } const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; @@ -1151,7 +1330,51 @@ router.post( }); } + const disconnectObserver = observeRequestDisconnect(req, res); + + const cancellation: ReplayRequestCancellation = { + signal: disconnectObserver.signal, + isDisconnected: disconnectObserver.isDisconnected, + }; + let reservedCancellation: { requestId: string; owner: string } | undefined; + try { + if (requestId != null) { + const owner = programmaticCancellationOwner(req, userId); + let reservation: Awaited>; + try { + reservation = await reserveProgrammaticCancellation({ + redis: connection, + requestId, + owner, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + } catch (error) { + logger.error('Failed to reserve programmatic cancellation request', { + requestId, + error: (error as Error).message, + }); + if (!cancellation.isDisconnected()) { + return res.status(503).json({ error: 'Cancellation service unavailable' }); + } + return; + } + if (reservation === 'forbidden' || reservation === 'duplicate') { + if (!cancellation.isDisconnected()) { + return res.status(409).json({ + error: 'Programmatic request ID is already in use', + }); + } + return; + } + reservedCancellation = { requestId, owner }; + cancellation.request = { + requestId, + owner, + cancelledBeforeStart: reservation === 'cancelled', + }; + } + /** For continuations, peek at the stored execution to route by the * mode it was started in rather than the current process default. * Without this, a replay-mode execution resumed via an instance @@ -1184,7 +1407,7 @@ router.post( userId, decoded, tool_results, - }); + }, cancellation); } return await handleBlocking(req, res, { apiKeyId, userId }); } @@ -1203,7 +1426,7 @@ router.post( userId, bridgeWorkerId, workspaceId, - }); + }, cancellation); } if (workspaceId != null) { return res.status(400).json({ @@ -1221,6 +1444,20 @@ router.post( return res.status(500).json({ error: 'Internal server error' }); } return; + } finally { + disconnectObserver.dispose(); + if (reservedCancellation != null) { + await releaseProgrammaticCancellation({ + redis: connection, + requestId: reservedCancellation.requestId, + owner: reservedCancellation.owner, + }).catch(error => { + logger.warn('Failed to release programmatic cancellation request', { + requestId: reservedCancellation?.requestId, + error: (error as Error).message, + }); + }); + } } }, ); diff --git a/service/src/types/service.ts b/service/src/types/service.ts index d17b5a99..a0c78486 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -297,6 +297,12 @@ export type JobData = { bridgeWorkerId?: string; /** Trusted selected workspace for native replay-mode PTC. */ workspaceId?: string; + /** Opts replay jobs into durable client-disconnect cancellation. */ + cancellable?: boolean; + /** Absolute producer budget; queue-worker configuration may only tighten it. */ + deadlineAtMs?: number; + /** Producer request tombstones must never outlive the completion decision. */ + cancellationTtlSeconds?: number; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; /** Required sandbox transport. Optional only for jobs queued before fencing. */ diff --git a/service/src/workers.ts b/service/src/workers.ts index f11a5420..a7153b53 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -1,69 +1,148 @@ import axios from 'axios'; import { Worker } from 'bullmq'; import type * as t from './types'; -import { filterSystemLogs, applySystemReplacements, getAxiosErrorDetails, sandboxErrorMessageFromAxios } from './utils'; -import { jobProcessingDuration, jobsCompleted, jobsFailed, activeJobs, workerRunning } from './metrics'; -import { connection, queueNames } from './queue'; +import { + filterSystemLogs, + applySystemReplacements, + getAxiosErrorDetails, + sandboxErrorMessageFromAxios, +} from './utils'; +import { + jobProcessingDuration, + jobsCancelled, + jobsCompleted, + jobsFailed, + activeJobs, + workerRunning, +} from './metrics'; +import { connection, jobCancellationRegistry, queueNames } from './queue'; import { env, jobDeadlineAtMs } from './config'; import { summarizeSandboxResponse, summarizeText } from './execution-log'; -import { createGatewayEgressGrant, restoreGatewaySandboxResult, revokeGatewayEgressGrant } from './egress-gateway-client'; +import { + createGatewayEgressGrant, + restoreGatewaySandboxResult, + revokeGatewayEgressGrant, +} from './egress-gateway-client'; import { refreshEgressGrantClaims } from './sandbox-egress'; import { buildSandboxExecuteRequest } from './sandbox-dispatch'; import { prepareInputDelivery } from './runtime-session/input-delivery'; import { SessionFilesError } from './runtime-session/files'; import { resolveRuntimeSessionForJob } from './runtime-session/job-policy'; -import { getSandboxBackend, SandboxBackendError, type SandboxRawResponse } from './sandbox-backend'; +import { + getSandboxBackend, + SandboxBackendError, + type SandboxRawResponse, +} from './sandbox-backend'; import { isSyntheticPrincipalSource } from './auth/synthetic'; import { withSpan, withTraceContext } from './telemetry'; import { workerDeadlineFailure } from './worker-error'; +import { + CLIENT_DISCONNECT_REASON, + JOB_CANCELLED_MESSAGE, + jobResultCommitFailure, + commitJobResult, + claimJobExecution, + jobCancellationRetentionSeconds, + throwIfJobAborted, +} from './job-cancellation'; import logger from './logger'; import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; -import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs } from '../../packages/code/src/protocol'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + programmaticTransferReserveMs, +} from '../../packages/code/src/protocol'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; function isAbortError(error: unknown): boolean { - return axios.isAxiosError(error) && (error.name === 'AbortError' || error.code === 'ERR_CANCELED'); + return ( + axios.isAxiosError(error) && + (error.name === 'AbortError' || error.code === 'ERR_CANCELED') + ); } async function processJob(job: t.ExecuteJob): Promise { - return withTraceContext(job.data._otel, () => withSpan('codeapi.job.process', { - 'messaging.system': 'bullmq', - 'messaging.operation.name': 'process', - 'messaging.message.id': typeof job.id === 'string' ? job.id : String(job.id ?? ''), - 'codeapi.language': job.data.payload?.language ?? 'unknown', - 'codeapi.execution_profile': job.data.executionProfile ?? 'legacy', - 'codeapi.worker_execution_profile': env.EXECUTION_PROFILE, - }, () => processJobInner(job), 'CONSUMER')); + return withTraceContext(job.data._otel, () => + withSpan( + 'codeapi.job.process', + { + 'messaging.system': 'bullmq', + 'messaging.operation.name': 'process', + 'messaging.message.id': + typeof job.id === 'string' ? job.id : String(job.id ?? ''), + 'codeapi.language': job.data.payload?.language ?? 'unknown', + 'codeapi.execution_profile': job.data.executionProfile ?? 'legacy', + 'codeapi.worker_execution_profile': env.EXECUTION_PROFILE, + }, + () => processJobInner(job), + 'CONSUMER', + ), + ); } async function processJobInner(job: t.ExecuteJob): Promise { const { payload, isPyPlot } = job.data; - const isSyntheticJob = job.data.isSynthetic === true || isSyntheticPrincipalSource(job.data.principalSource); + const isSyntheticJob = + job.data.isSynthetic === true || + isSyntheticPrincipalSource(job.data.principalSource); const language = payload?.language ?? 'unknown'; const endTimer = jobProcessingDuration.startTimer({ language }); activeJobs.inc({ language }); const controller = new AbortController(); - const deadlineAtMs = jobDeadlineAtMs(job.timestamp, env.JOB_TIMEOUT); + const cancellationTarget = + job.data.cancellable === true && job.id != null + ? { queueName: job.queueName, jobId: String(job.id) } + : undefined; + let cancellationRegistered = false; + const deadlineAtMs = jobDeadlineAtMs( + job.timestamp, + env.JOB_TIMEOUT, + Date.now(), + job.data.deadlineAtMs, + ); const remainingBudgetMs = Math.max(0, deadlineAtMs - Date.now()); - const timer = remainingBudgetMs > 0 - ? setTimeout(() => controller.abort(), remainingBudgetMs) - : undefined; - if (remainingBudgetMs === 0) controller.abort(); + const timer = + remainingBudgetMs > 0 + ? setTimeout(() => controller.abort('deadline'), remainingBudgetMs) + : undefined; + if (remainingBudgetMs === 0) controller.abort('deadline'); let egressGrantId: string | undefined; let egressGrantTokenForRestore: string | undefined; let revokeReason = 'completed'; + let completedResult = false; + let resultToCommit: t.ExecuteResult | undefined; + let resultCommittedAtHandoff = false; + const commitAtHandoff = + cancellationTarget != null && + job.data.workspaceId != null && + env.SANDBOX_BACKEND === 'remote-bridge'; try { + if (cancellationTarget != null) { + await jobCancellationRegistry.register(cancellationTarget, controller); + cancellationRegistered = true; + const claim = await claimJobExecution( + connection, + cancellationTarget, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + ); + if (claim.status === 'completed') return claim.result; + } if (controller.signal.aborted) { throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); } - validateQueuedExecutionProfile(job.data.executionProfile, env.EXECUTION_PROFILE); + validateQueuedExecutionProfile( + job.data.executionProfile, + env.EXECUTION_PROFILE, + ); validateQueuedSandboxBackend( job.data.sandboxBackend, env.SANDBOX_BACKEND, @@ -77,7 +156,10 @@ async function processJobInner(job: t.ExecuteJob): Promise { const nowSeconds = Math.floor(Date.now() / 1000); const prepared = await createGatewayEgressGrant({ payload, - claims: refreshEgressGrantClaims(job.data.egressGrantClaims, nowSeconds), + claims: refreshEgressGrantClaims( + job.data.egressGrantClaims, + nowSeconds, + ), isSynthetic: isSyntheticJob, signal: controller.signal, }); @@ -85,19 +167,27 @@ async function processJobInner(job: t.ExecuteJob): Promise { sandboxPayload = prepared.payload; egressGrantToken = prepared.egressGrantToken; egressGrantTokenForRestore = prepared.egressGrantToken; - executionManifestClaims = (env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET) - ? prepared.executionManifestClaims - : undefined; + executionManifestClaims = + env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET + ? prepared.executionManifestClaims + : undefined; } const delivery = prepareInputDelivery(payload, sandboxPayload); const sandboxRequest = buildSandboxExecuteRequest({ - ...(job.data.workspaceId == null ? {} : { programmaticTransferReserveMs: programmaticTransferReserveMs(env.JOB_TIMEOUT) }), + ...(job.data.workspaceId == null + ? {} + : { + programmaticTransferReserveMs: programmaticTransferReserveMs( + env.JOB_TIMEOUT, + ), + }), payload: delivery.payload, egressGrantToken, executionManifestClaims, maxOutputFileBytes: Math.min( - executionManifestClaims?.max_upload_bytes ?? env.EGRESS_GATEWAY_MAX_FILE_BYTES, + executionManifestClaims?.max_upload_bytes ?? + env.EGRESS_GATEWAY_MAX_FILE_BYTES, env.EGRESS_GATEWAY_MAX_FILE_BYTES, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, ), @@ -121,21 +211,41 @@ async function processJobInner(job: t.ExecuteJob): Promise { * the transformed object makes that second call an idempotent no-op. */ const resultRestoreToken = egressGrantTokenForRestore; const finalizedSandboxResults = new WeakSet(); - const finalizeSandboxResult = async (result: SandboxRawResponse): Promise => { - if ( - resultRestoreToken === undefined || - resultRestoreToken.length === 0 || - finalizedSandboxResults.has(result) - ) { - return result; + const finalizeSandboxResult = async ( + result: SandboxRawResponse, + ): Promise => { + if (finalizedSandboxResults.has(result)) return result; + const restored = + resultRestoreToken == null || resultRestoreToken.length === 0 + ? result + : await restoreGatewaySandboxResult({ + grantId: egressGrantId, + egressGrantToken: resultRestoreToken, + result, + isSynthetic: isSyntheticJob, + signal: controller.signal, + }); + if (commitAtHandoff && cancellationTarget != null) { + // The bridge still owns its mutation fence here. A failed/ambiguous + // commit quarantines that root before it can serve a caller retry. + throwIfJobAborted(controller.signal); + const mapped = mapSandboxResult(restored); + const committed = await commitJobResult( + connection, + cancellationTarget, + mapped, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + deadlineAtMs, + ); + if (committed === 'cancelled') throw new Error(JOB_CANCELLED_MESSAGE); + if (committed === 'already_completed') + throw new Error('Duplicate mutation handoff; quarantining workspace'); + resultToCommit = mapped; + resultCommittedAtHandoff = true; } - const restored = await restoreGatewaySandboxResult({ - grantId: egressGrantId, - egressGrantToken: resultRestoreToken, - result, - isSynthetic: isSyntheticJob, - signal: controller.signal, - }); finalizedSandboxResults.add(restored); return restored; }; @@ -162,76 +272,115 @@ async function processJobInner(job: t.ExecuteJob): Promise { /* Stateful backends run this as a commit barrier after user code but * before checkpointing/reusing the mutated workspace. Stateless/HTTP * paths retain the worker-owned fallback immediately below. */ - sessionResultFinalizer: resultRestoreToken !== undefined && resultRestoreToken.length > 0 - ? finalizeSandboxResult - : undefined, + sessionResultFinalizer: + commitAtHandoff || + (resultRestoreToken !== undefined && resultRestoreToken.length > 0) + ? finalizeSandboxResult + : undefined, }, ); const responseData = await finalizeSandboxResult(responseRaw); + // Cancellation can arrive after sandbox exit while artifact restoration + // yields. Do not let BullMQ commit a success after Stop was acknowledged. + if (!resultCommittedAtHandoff) throwIfJobAborted(controller.signal); - if (!isSyntheticJob) { - logger.info('Sandbox response', summarizeSandboxResponse(responseData)); - } + function mapSandboxResult( + responseData: SandboxRawResponse, + ): t.ExecuteResult { + if (!isSyntheticJob) { + logger.info('Sandbox response', summarizeSandboxResponse(responseData)); + } - const { files } = responseData; - const run = responseData.run; - const stdout = applySystemReplacements(run?.stdout ?? ''); - const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); + const { files } = responseData; + const run = responseData.run; + const stdout = applySystemReplacements(run?.stdout ?? ''); + const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); - const result: t.ExecuteResult = { - session_id: responseData.session_id, - /* `files` is optional on the sandbox response (e.g. dry-run - * execute with no outputs); the public `ExecuteResult.files` is - * required and downstream callers always iterate it. Default to - * `[]` so the strictened response type from Phase B doesn't - * surface a regression that wasn't there before. */ - files: files ?? [], - ...(responseData.artifact_delivery != null - ? { artifact_delivery: responseData.artifact_delivery } - : {}), - ...(responseData.artifact_truncation != null - ? { artifact_truncation: responseData.artifact_truncation } - : {}), - stdout, - stderr, - ...(responseData.pending_tool_calls_payload != null - ? { pending_tool_calls_payload: responseData.pending_tool_calls_payload } - : {}), - }; + const result: t.ExecuteResult = { + session_id: responseData.session_id, + /* `files` is optional on the sandbox response (e.g. dry-run + * execute with no outputs); the public `ExecuteResult.files` is + * required and downstream callers always iterate it. Default to + * `[]` so the strictened response type from Phase B doesn't + * surface a regression that wasn't there before. */ + files: files ?? [], + ...(responseData.artifact_delivery != null + ? { artifact_delivery: responseData.artifact_delivery } + : {}), + ...(responseData.artifact_truncation != null + ? { artifact_truncation: responseData.artifact_truncation } + : {}), + stdout, + stderr, + ...(responseData.pending_tool_calls_payload != null + ? { + pending_tool_calls_payload: + responseData.pending_tool_calls_payload, + } + : {}), + }; - if (run) { - result.code = run.code ?? null; - result.signal = run.signal != null ? String(run.signal) : null; - result.message = run.message ?? null; - result.status = run.status ?? null; - result.wall_time = (run as Record).wall_time as number | null ?? null; - } + if (run) { + result.code = run.code ?? null; + result.signal = run.signal != null ? String(run.signal) : null; + result.message = run.message ?? null; + result.status = run.status ?? null; + result.wall_time = + ((run as Record).wall_time as number | null) ?? null; + } - if (result.message || result.signal) { - logger.warn('Sandbox execution error metadata', { - session_id: responseData.session_id, - code: result.code, - signal: result.signal, - message: summarizeText(result.message), - status: result.status, - wall_time: result.wall_time, - }); + if (result.message || result.signal) { + logger.warn('Sandbox execution error metadata', { + session_id: responseData.session_id, + code: result.code, + signal: result.signal, + message: summarizeText(result.message), + status: result.status, + wall_time: result.wall_time, + }); + } + + return result; } + const result = resultToCommit ?? mapSandboxResult(responseData); + completedResult = true; + resultToCommit = result; return result; } catch (error) { - revokeReason = controller.signal.aborted || isAbortError(error) ? 'timeout' : 'failed'; + // Bridge fence cleanup can fail after the outcome was durably committed. + // Preserve the winning result; the bridge retains/quarantines its fence. + if (resultCommittedAtHandoff && resultToCommit != null) + return resultToCommit; + const clientDisconnected = + controller.signal.aborted && + controller.signal.reason === CLIENT_DISCONNECT_REASON; + revokeReason = clientDisconnected + ? 'cancelled' + : controller.signal.aborted || isAbortError(error) + ? 'timeout' + : 'failed'; const errorDetails = getAxiosErrorDetails(error); - logger.error('Error processing job', errorDetails); + if (clientDisconnected) { + logger.info('Job cancelled after client disconnected', { + queueName: job.queueName, + jobId: job.id, + executionId: job.data.executionId, + }); + } else { + logger.error('Error processing job', errorDetails); + } const deadlineFailure = workerDeadlineFailure( error, - controller.signal.aborted, + controller.signal.aborted && !clientDisconnected, env.JOB_TIMEOUT, ); if (deadlineFailure) { throw deadlineFailure; + } else if (clientDisconnected) { + throw new Error(JOB_CANCELLED_MESSAGE); } else if (error instanceof SandboxBackendError) { throw new Error(`${error.code}: ${error.message}`); } else if (error instanceof SessionFilesError) { @@ -251,17 +400,67 @@ async function processJobInner(job: t.ExecuteJob): Promise { if (egressGrantId || egressGrantTokenForRestore) { await revokeGatewayEgressGrant({ grantId: egressGrantId, - egressGrantToken: egressGrantId ? undefined : egressGrantTokenForRestore, + egressGrantToken: egressGrantId + ? undefined + : egressGrantTokenForRestore, isSynthetic: isSyntheticJob, reason: revokeReason, timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, }).catch(error => { - logger.error('Failed to revoke egress grant', { grantId: egressGrantId, error: getAxiosErrorDetails(error) }); + logger.error('Failed to revoke egress grant', { + grantId: egressGrantId, + error: getAxiosErrorDetails(error), + }); }); } + let lateCommitFailure = + completedResult && !resultCommittedAtHandoff + ? jobResultCommitFailure(controller.signal, env.JOB_TIMEOUT) + : undefined; + if ( + completedResult && + !resultCommittedAtHandoff && + cancellationTarget != null && + lateCommitFailure == null + ) { + try { + const committed = await commitJobResult( + connection, + cancellationTarget, + resultToCommit, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + deadlineAtMs, + ); + if (committed === 'cancelled') { + lateCommitFailure = new Error(JOB_CANCELLED_MESSAGE); + } else if (committed === 'already_completed') { + lateCommitFailure = new Error( + 'Duplicate result handoff; refusing replacement', + ); + } + } catch (error) { + lateCommitFailure = + error instanceof Error ? error : new Error('Result commit failed'); + } + } if (timer) clearTimeout(timer); + if (cancellationTarget != null && cancellationRegistered) { + await jobCancellationRegistry + .unregister(cancellationTarget, controller) + .catch(error => { + logger.warn('Failed to clear queued execution cancellation state', { + queueName: cancellationTarget.queueName, + jobId: cancellationTarget.jobId, + error: getAxiosErrorDetails(error), + }); + }); + } endTimer(); activeJobs.dec({ language }); + if (lateCommitFailure != null) throw lateCommitFailure; } } @@ -304,21 +503,31 @@ otherWorker.on('completed', job => { }); pyWorker.on('failed', (job, err) => { + if (err.message === JOB_CANCELLED_MESSAGE) { + logger.info(`[${WORKER_ID}] Python job ${job?.id} cancelled`); + jobsCancelled.inc({ language: 'python' }); + return; + } logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); jobsFailed.inc({ language: 'python' }); }); otherWorker.on('failed', (job, err) => { + if (err.message === JOB_CANCELLED_MESSAGE) { + logger.info(`[${WORKER_ID}] Other job ${job?.id} cancelled`); + jobsCancelled.inc({ language: 'other' }); + return; + } logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); jobsFailed.inc({ language: 'other' }); }); -pyWorker.on('error', (err) => { +pyWorker.on('error', err => { logger.error(`[${WORKER_ID}] Python worker error`, err); workerRunning.set({ worker_type: 'python' }, 0); }); -otherWorker.on('error', (err) => { +otherWorker.on('error', err => { logger.error(`[${WORKER_ID}] Other worker error`, err); workerRunning.set({ worker_type: 'other' }, 0); }); From 3e7b107c74960ab42197fe01d2b05a1709e4b762 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:43:49 -0400 Subject: [PATCH 096/116] fix: classify capped artifact probe candidates (#206) --- api/src/job.ts | 56 +++++++++++++++++++++++++++++++++--------- api/src/walker.test.ts | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/api/src/job.ts b/api/src/job.ts index 27604d29..a5b7ded2 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -2208,7 +2208,7 @@ export class Job { state = this.truncationProbeState, probeDepth = 0, rootPath = path.relative(this.submissionDir, dir) || '.', - respectSessionSuppression = false, + isOutputCapProbe = false, ): Promise { /* The state is shared by every probe in this job. Once exhausted, return * conservatively before opening yet another capped sibling directory. */ @@ -2246,17 +2246,51 @@ export class Job { if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; const existingFile = inputByName.get(relativePath); const inputFileInfo = this.inputFileHashes.get(relativePath); + let capProbeStat: fs.Stats | undefined; + if (isOutputCapProbe) { + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + relativePath, + ); + continue; + } + try { + capProbeStat = await fsp.lstat(fullPath); + if (!capProbeStat.isFile()) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during cap-probe stat'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + if (capProbeStat.size > this.runtime.max_file_size) { + /* Match handleRegularFile's one exception: an unchanged inline + * entrypoint is request input rather than an oversized output. */ + if (!inputFileInfo || existingFile?.id != null || relativePath !== this.entryPointName) { + this.recordArtifactTruncation('size', relativePath); + continue; + } + if (capProbeStat.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat.size; + try { + if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during oversized entrypoint cap probe'); + } + this.recordArtifactTruncation('size', relativePath); + continue; + } + } if ( - respectSessionSuppression + isOutputCapProbe && relativePath === this.entryPointName && existingFile?.id == null && inputFileInfo ) { try { - const st = await fsp.lstat(fullPath); - if (!st.isFile()) continue; - if (st.size > state.remainingHashBytes) return rootPath; - state.remainingHashBytes -= st.size; + if (capProbeStat!.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat!.size; if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: failed during entrypoint cap probe'); @@ -2270,13 +2304,11 @@ export class Job { * so the bounded cap probe must do the same or it reports a false * max_files warning. Current-request inputs remain reportable: they * would otherwise have been echoed into this response. */ - if (respectSessionSuppression && this.session && !existingFile) { + if (isOutputCapProbe && this.session && !existingFile) { if (this.session.isPrimedReadOnly(relativePath)) continue; try { - const st = await fsp.lstat(fullPath); - if (!st.isFile()) continue; - if (st.size > state.remainingHashBytes) return rootPath; - state.remainingHashBytes -= st.size; + if (capProbeStat!.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat!.size; const hash = await this.computeFileHash(fullPath, true); if (this.session.isSurfaced(relativePath, hash)) continue; if ( @@ -2303,7 +2335,7 @@ export class Job { state, probeDepth + 1, rootPath, - respectSessionSuppression, + isOutputCapProbe, ); if (nested) return nested; } diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index 543505a9..ba6f4af3 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -1021,6 +1021,49 @@ describe('walkDir / artifact truncation details', () => { }); }); + it('classifies an oversized supported file by size when the output cap is full', async () => { + await fsp.writeFile(path.join(tmpDir, 'oversized.txt'), 'too large'); + const internals = asInternals(makeJob({ maxFileSize: 3 })); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { size: 1 }, + skipped: ['oversized.txt'], + skipped_count: 1, + }); + }); + + it('classifies an overlong supported path by path when the output cap is full', async () => { + const directory = 'a'.repeat(200); + const filename = path.join(directory, `${'b'.repeat(60)}.txt`); + await fsp.mkdir(path.join(tmpDir, directory)); + await fsp.writeFile(path.join(tmpDir, filename), 'output'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [filename], + skipped_count: 1, + }); + }); + it('does not hash ordinary oversized files in session mode', async () => { await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); const session = new SessionWorkspace({ runtimeSessionId: 'rt_large' }); From 6ca38b23fd0b40eab66d82b0d8783e9168a092f9 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:44:01 -0400 Subject: [PATCH 097/116] feat: Report Deleted Code Session Files (#200) * fix: report deleted persisted files * fix: reconcile deletions across code runtimes * fix: preserve protected session inputs * fix: classify reserved runtime paths --- api/README.md | 7 + api/src/job.ts | 57 ++++- api/src/session-workspace.test.ts | 5 + api/src/session-workspace.ts | 6 + api/src/walker.test.ts | 196 ++++++++++++++++++ packages/code/src/native-programmatic.test.ts | 112 ++++++++++ packages/code/src/native-programmatic.ts | 44 +++- service/src/service/blocking-poll.test.ts | 2 + service/src/service/blocking-poll.ts | 2 + service/src/service/programmatic-router.ts | 4 + service/src/types/service.ts | 3 + service/src/workers.ts | 3 + 12 files changed, 424 insertions(+), 17 deletions(-) diff --git a/api/README.md b/api/README.md index 665a2931..38e4cd98 100644 --- a/api/README.md +++ b/api/README.md @@ -94,6 +94,13 @@ Other package-format-compatible runtimes (Go, Rust, Java, GCC) can be installed Execute code in a sandboxed environment. +When a persisted input file is removed during execution, a complete artifact +scan reports its relative path in `deleted_files`. Callers can use this +explicit list to remove stale file references from their next session request. +The field is omitted when no persisted inputs were removed or when artifact +scanning is incomplete, so truncation or unreadable paths cannot be mistaken +for deletions. + When supported output files are omitted because the response reaches its file count limit, nesting or path limits, file-size limit, or a filesystem entry cannot be read, the response includes `artifact_truncation`. Its `reasons` diff --git a/api/src/job.ts b/api/src/job.ts index a5b7ded2..0939de08 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -655,6 +655,8 @@ interface ExecuteResult { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRef[]; + /** Persisted input paths that no longer exist after this execution. */ + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; } @@ -714,6 +716,8 @@ export class Job { private pendingSurfaced = new Map(); private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; + private presentInputFiles = new Set(); + private deletedFiles: string[] = []; private artifactTruncation: ArtifactTruncation | undefined; private truncationProbeState: TruncationProbeState = { remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, @@ -1699,6 +1703,9 @@ export class Job { version: this.runtime.version.raw, session_id: this.outputSessionId, files: this.sessionFiles, + ...(this.deletedFiles.length > 0 + ? { deleted_files: this.deletedFiles } + : {}), ...(this.artifactTruncation ? { artifact_truncation: this.artifactTruncation } : {}), }; } @@ -1707,6 +1714,8 @@ export class Job { this.generatedFiles = []; this.sessionFiles = []; this.inheritedRefs = []; + this.presentInputFiles.clear(); + this.deletedFiles = []; this.artifactTruncation = undefined; this.truncationProbeState = { remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, @@ -1720,6 +1729,26 @@ export class Job { await this.walkDir(this.submissionDir, 0, inputByName); } catch (error) { this.log.error({ err: error }, 'Error scanning submission directory'); + this.recordArtifactTruncation('unreadable', '.'); + } + + if (this.artifactTruncation == null) { + const returnedNames = new Set([ + ...this.sessionFiles.map(file => file.name), + ...this.inheritedRefs.map(file => file.name), + ]); + for (const file of this.files) { + if ( + file.id != null && + file.storage_session_id != null && + this.inputFileHashes.get(file.name)?.readOnly !== true && + !this.presentInputFiles.has(file.name) && + !returnedNames.has(file.name) + ) { + this.deletedFiles.push(file.name); + this.session?.forgetPrimed(file.name); + } + } } /* Generated files get priority in sessionFiles; fill remaining slots up @@ -2227,13 +2256,18 @@ export class Job { let sawVisibleNonHiddenEntry = false; try { for await (const entry of directory) { - if (entry.name === PTC_HISTORY_FILENAME) continue; - sawVisibleEntry = true; - state.remainingEntries--; - if (state.remainingEntries < 0) return rootPath; const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); const kind = await this.classifyDirent(entry, fullPath, relativePath); + if (kind === 'file' && entry.name === PTC_HISTORY_FILENAME) { + if (inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } + continue; + } + sawVisibleEntry = true; + state.remainingEntries--; + if (state.remainingEntries < 0) return rootPath; if (kind === 'skip') { /* Ordinary walking counts symlinks/special entries as non-empty even * though it does not surface them, so the probe must not invent a @@ -2243,6 +2277,9 @@ export class Job { } if (kind === 'file') { sawVisibleNonHiddenEntry = true; + if (inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; const existingFile = inputByName.get(relativePath); const inputFileInfo = this.inputFileHashes.get(relativePath); @@ -2423,13 +2460,21 @@ export class Job { let skippedHiddenDirs = 0; for (const entry of entries) { - if (isPtcReserved(entry.name)) continue; - const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); const kind = await this.classifyDirent(entry, fullPath, relativePath); if (kind === 'skip') continue; + if (kind === 'file' && inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } + + /* A by-reference input may legitimately use the reserved replay-history + * basename on the ordinary execution endpoint. It remains hidden from + * output collection, but must be observed before the runtime fixture is + * skipped so an untouched input is not reported as deleted. */ + if (kind === 'file' && isPtcReserved(entry.name)) continue; + if (kind === 'dir') { /* Skip hidden directories (basename starts with `.`) unless the user * explicitly primed something under them. Matplotlib, pip, and other diff --git a/api/src/session-workspace.test.ts b/api/src/session-workspace.test.ts index 87ca71df..1a5320c6 100644 --- a/api/src/session-workspace.test.ts +++ b/api/src/session-workspace.test.ts @@ -113,6 +113,11 @@ describe('SessionWorkspace state', () => { expect(ws.isPrimedInput('in.csv')).toBe(false); ws.markPrimed('in.csv', 'file_abc'); expect(ws.primedInputId('in.csv')).toBe('file_abc'); + ws.markSurfaced('in.csv', 'old-output'); + ws.forgetPrimed('in.csv'); + expect(ws.primedInputId('in.csv')).toBeUndefined(); + expect(ws.isSurfaced('in.csv', 'old-output')).toBe(false); + ws.markPrimed('in.csv', 'file_abc'); /* read-only primes report as not-primed so the caller re-downloads them * (a reused on-disk copy could have been tampered via the writable dir). */ diff --git a/api/src/session-workspace.ts b/api/src/session-workspace.ts index 3f2105bc..82ed6c89 100644 --- a/api/src/session-workspace.ts +++ b/api/src/session-workspace.ts @@ -202,6 +202,12 @@ export class SessionWorkspace { this.primed.set(relPath, { id: storageFileId, readOnly, hash }); } + /** Clears input lineage after execution proves that the path was deleted. */ + forgetPrimed(relPath: string): void { + this.primed.delete(relPath); + this.forget(relPath); + } + markDirty(reason: string): void { this.dirty = reason; logger.error( diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index ba6f4af3..44e252a6 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -26,6 +26,8 @@ interface WalkerInternals { generatedFiles: Array<{ id: string; name: string; path: string }>; sessionFiles: Array<{ id: string; name: string; storage_session_id: string; modified_from?: { id: string; storage_session_id: string }; inherited?: true; entity_id?: string }>; inheritedRefs: Array<{ id: string; name: string; storage_session_id: string; inherited?: true; entity_id?: string }>; + presentInputFiles: Set; + deletedFiles: string[]; artifactTruncation?: { code: 'artifact_truncated'; reasons: Partial>; @@ -39,6 +41,14 @@ interface WalkerInternals { reusePrimedInput: (file: TFile) => Promise; writeFile: (file: TFile) => Promise; computeFileHash: (filePath: string, noFollow?: boolean) => Promise; + findTruncatedArtifact: ( + dir: string, + inputByName: Map, + state?: { remainingEntries: number; remainingHashBytes: number }, + probeDepth?: number, + rootPath?: string, + respectSessionSuppression?: boolean, + ) => Promise; walkDir: (dir: string, depth: number, inputByName: Map) => Promise<'collected' | 'empty' | 'skipped'>; handleSessionFiles: () => Promise; } @@ -1300,6 +1310,192 @@ describe('handleSessionFiles / priority-fill composition', () => { }); }); +describe('handleSessionFiles / persisted input deletion', () => { + it('reports a persisted input that no longer exists', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual(['removed.txt']); + }); + + it('retains a read-only persisted input when sandbox code removes its local copy', async () => { + const inherited: TFile = { + id: 'skill-id', + storage_session_id: 'skill-session', + name: path.join('skills', 'review', 'SKILL.md'), + }; + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + internals.inputFileHashes.set(inherited.name, { + hash: sha256('trusted-skill'), + path: path.join(tmpDir, inherited.name), + originalId: inherited.id, + originalSessionId: inherited.storage_session_id, + readOnly: true, + }); + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + }); + + it('tracks a persisted input using the reserved PTC history basename', async () => { + const inherited: TFile = { + id: 'history-id', + storage_session_id: 'prior-session', + name: path.join('fixtures', '_ptc_history.json'), + }; + await fsp.mkdir(path.join(tmpDir, 'fixtures')); + await fsp.writeFile(path.join(tmpDir, inherited.name), '{}'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect(internals.generatedFiles.map(file => file.name)).not.toContain(inherited.name); + }); + + it('traverses a directory that uses the reserved PTC history basename', async () => { + const inherited: TFile = { + id: 'nested-id', + storage_session_id: 'prior-session', + name: path.join('_ptc_history.json', 'data.csv'), + }; + await fsp.mkdir(path.join(tmpDir, '_ptc_history.json')); + await fsp.writeFile(path.join(tmpDir, inherited.name), 'persisted'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); + + it('tracks a reserved persisted input during a capped subtree probe', async () => { + const inherited: TFile = { + id: 'history-id', + storage_session_id: 'prior-session', + name: path.join('fixtures', '_ptc_history.json'), + }; + const fixtures = path.join(tmpDir, 'fixtures'); + await fsp.mkdir(fixtures); + await fsp.writeFile(path.join(tmpDir, inherited.name), '{}'); + await fsp.writeFile(path.join(fixtures, 'unsupported.bin'), 'binary'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + const skipped = await internals.findTruncatedArtifact( + fixtures, + new Map([[inherited.name, inherited]]) + ); + + expect(skipped).toBeUndefined(); + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); + + it('does not report a surviving input that is unsupported as an output artifact', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'archive.bin', + }; + await fsp.writeFile(path.join(tmpDir, inherited.name), 'binary-placeholder'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.deletedFiles).toEqual([]); + }); + + it('suppresses deletion reporting when the artifact scan is incomplete', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + await fsp.writeFile(path.join(tmpDir, 'too-large.txt'), 'too large'); + const internals = asInternals(makeJob({ files: [inherited], maxFileSize: 3 })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.artifactTruncation?.reasons).toEqual({ size: 1 }); + expect(internals.deletedFiles).toEqual([]); + }); + + it('does not report an inherited marker that is returned for an empty directory', async () => { + const name = path.join('empty', DIRKEEP); + const inherited: TFile = { + id: 'marker-id', + storage_session_id: 'prior-session', + name, + }; + await fsp.mkdir(path.join(tmpDir, 'empty')); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect([ + ...internals.sessionFiles, + ...internals.inheritedRefs, + ].map(file => file.name)).toContain(name); + }); + + it('clears stateful priming lineage when a persisted input is deleted', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + const session = new SessionWorkspace({ runtimeSessionId: 'rt_deleted' }); + session.markPrimed(inherited.name, inherited.id!, true, 'old-hash'); + session.markSurfaced(inherited.name, 'old-output-hash'); + const internals = asInternals(makeJob({ files: [inherited], session })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([inherited.name]); + expect(session.isPrimedInput(inherited.name)).toBe(false); + expect(session.isSurfaced(inherited.name, 'old-output-hash')).toBe(false); + }); + + it('tracks surviving persisted inputs during capped subtree probes', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: path.join('assets', 'model.bin'), + }; + await fsp.mkdir(path.join(tmpDir, 'assets')); + await fsp.writeFile( + path.join(tmpDir, inherited.name), + 'unsupported-but-persisted', + ); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.findTruncatedArtifact( + tmpDir, + new Map([[inherited.name, inherited]]), + ); + + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); +}); + describe('walkDir / dirent classification', () => { it('ignores symlinks (never classifies them as file or dir)', async () => { await fsp.writeFile(path.join(tmpDir, 'real.py'), 'print(1)'); diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts index 82650f95..902c707a 100644 --- a/packages/code/src/native-programmatic.test.ts +++ b/packages/code/src/native-programmatic.test.ts @@ -121,6 +121,118 @@ test('stages skill files privately and returns generated artifacts', async () => } }); +test('reports persisted inputs deleted by selected-workspace execution', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-delete-test-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const server = createServer((_req, res) => res.end('persisted input')); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(() => resolve()))); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await rm(join(dataDirectory, 'input.txt')); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + + const result = await executor.execute({ + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'rm input.txt' }, + { + name: 'input.txt', + id: 'input-id', + storage_session_id: 'input-session', + }, + ], + }, + }, 'primary'); + + assert.deepEqual(result.files, []); + assert.deepEqual(result.deleted_files, ['input.txt']); +}); + +test('retains read-only persisted inputs removed by selected-workspace execution', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-readonly-delete-test-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + let downloads = 0; + const server = createServer((_req, res) => { + downloads++; + res.setHeader('X-Read-Only', 'true'); + res.end('trusted skill'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(() => resolve()))); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await rm(join(dataDirectory, 'skills', 'review', 'SKILL.md')); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const request = { + headers: {}, + body: { + language: 'bash' as const, + version: '5.2.0', + execution_id: 'readonly-execution', + session_id: 'execution-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'rm skills/review/SKILL.md' }, + { + name: 'skills/review/SKILL.md', + id: 'skill-id', + storage_session_id: 'skill-session', + input_cache_key: 'a'.repeat(64), + }, + ], + }, + }; + + const result = await executor.execute(request, 'primary'); + const replay = await executor.execute(request, 'primary'); + + assert.equal(downloads, 1); + assert.equal(result.deleted_files, undefined); + assert.equal(replay.deleted_files, undefined); +}); + test('reports unsupported and rejected artifacts without invalidating a completed command', async () => { const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-test-')); const uploads = new Map(); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index 42974b79..bd2700f2 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -50,6 +50,7 @@ type ProgrammaticResult = { version: string; session_id: string; files: ProgrammaticFileResult[]; + deleted_files?: string[]; artifact_delivery?: { code: 'artifact_delivery_failed'; status: 'partial' | 'failed'; @@ -75,8 +76,11 @@ type ProgrammaticResult = { type InputBaseline = { sha256: string; source?: { id: string; storage_session_id: string }; + readOnly?: boolean; }; +type CachedInput = { bytes: Buffer; readOnly: boolean }; + function sha256(value: Uint8Array): string { return createHash('sha256').update(value).digest('hex'); } @@ -227,7 +231,7 @@ export class NativeWorkspaceProgrammaticExecutor { /** Parent-process cache: sandboxed children cannot inspect this memory. */ private readonly inputCache = new Map< string, - { bytes: Buffer; lastUsed: number } + CachedInput & { lastUsed: number } >(); private inputCacheBytes = 0; @@ -272,14 +276,15 @@ export class NativeWorkspaceProgrammaticExecutor { : undefined; } - private cachedInput(key: string): Buffer | undefined { + private cachedInput(key: string): CachedInput | undefined { const cached = this.inputCache.get(key); if (!cached) return undefined; cached.lastUsed = Date.now(); - return cached.bytes; + return { bytes: cached.bytes, readOnly: cached.readOnly }; } - private cacheInput(key: string, bytes: Buffer): void { + private cacheInput(key: string, input: CachedInput): void { + const { bytes } = input; if (bytes.byteLength > INPUT_CACHE_MAX_BYTES) return; const existing = this.inputCache.get(key); if (existing) this.inputCacheBytes -= existing.bytes.byteLength; @@ -300,7 +305,7 @@ export class NativeWorkspaceProgrammaticExecutor { this.inputCache.get(oldestKey)!.bytes.byteLength; this.inputCache.delete(oldestKey); } - this.inputCache.set(key, { bytes, lastUsed: Date.now() }); + this.inputCache.set(key, { ...input, lastUsed: Date.now() }); this.inputCacheBytes += bytes.byteLength; } @@ -310,7 +315,7 @@ export class NativeWorkspaceProgrammaticExecutor { executionId: string | undefined, signal?: AbortSignal, transferTimeoutMs = TRANSFER_TIMEOUT_MS, - ): Promise { + ): Promise { const key = this.cacheKey(executionId, file); const cached = key ? this.cachedInput(key) : undefined; if (cached) return cached; @@ -341,8 +346,12 @@ export class NativeWorkspaceProgrammaticExecutor { response, controller.signal, ); - if (key) this.cacheInput(key, bytes); - return bytes; + const input = { + bytes, + readOnly: response.headers.get('x-read-only')?.toLowerCase() === 'true', + }; + if (key) this.cacheInput(key, input); + return input; } finally { clearTimeout(timer); signal?.removeEventListener('abort', abort); @@ -393,9 +402,9 @@ export class NativeWorkspaceProgrammaticExecutor { request.body.files, TRANSFER_CONCURRENCY, async (file): Promise => { - const bytes = + const input = 'content' in file - ? Buffer.from(file.content) + ? { bytes: Buffer.from(file.content), readOnly: false } : await this.downloadInput( file, grant!, @@ -403,6 +412,7 @@ export class NativeWorkspaceProgrammaticExecutor { signal, request.body.transfer_timeout_ms, ); + const { bytes } = input; totalInputBytes += bytes.byteLength; if ( totalInputBytes > @@ -421,6 +431,7 @@ export class NativeWorkspaceProgrammaticExecutor { await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); baselines.set(file.name, { sha256: sha256(bytes), + ...(input.readOnly ? { readOnly: true } : {}), ...('id' in file ? { source: { @@ -578,7 +589,15 @@ export class NativeWorkspaceProgrammaticExecutor { } const outputSessionId = request.body.output_session_id; - const outputNames = (await listRegularFiles(dataDirectory)).filter( + const survivingNames = new Set(await listRegularFiles(dataDirectory)); + const deletedFiles = refFiles + .filter( + file => + baselines.get(file.name)?.readOnly !== true && + !survivingNames.has(file.name), + ) + .map(file => file.name); + const outputNames = [...survivingNames].filter( name => name !== EXECUTION_MAIN_FILE && name !== EXECUTION_HISTORY_FILE && @@ -731,6 +750,7 @@ export class NativeWorkspaceProgrammaticExecutor { performance.now() - startedAt, undefined, artifactDelivery, + deletedFiles, ); } catch (error) { if (!commandDispatched) { @@ -785,6 +805,7 @@ export class NativeWorkspaceProgrammaticExecutor { elapsedMs: number, pendingToolCallsPayload?: string, artifactDelivery?: ProgrammaticResult['artifact_delivery'], + deletedFiles: string[] = [], ): ProgrammaticResult { return { language: 'bash', @@ -795,6 +816,7 @@ export class NativeWorkspaceProgrammaticExecutor { session_id: request.body.output_session_id ?? request.body.session_id, files, + ...(deletedFiles.length > 0 ? { deleted_files: deletedFiles } : {}), ...(artifactDelivery ? { artifact_delivery: artifactDelivery } : {}), ...(pendingToolCallsPayload ? { pending_tool_calls_payload: pendingToolCallsPayload } diff --git a/service/src/service/blocking-poll.test.ts b/service/src/service/blocking-poll.test.ts index 35190157..96903d1f 100644 --- a/service/src/service/blocking-poll.test.ts +++ b/service/src/service/blocking-poll.test.ts @@ -4,6 +4,7 @@ import type * as t from '../types'; const result: t.ExecuteResult = { session_id: 'session', stdout: 'successful code', stderr: '', files: [], + deleted_files: ['removed.txt'], artifact_delivery: { code: 'artifact_delivery_failed', status: 'failed', attempted: 1, delivered: 0, failed: 1, }, @@ -29,6 +30,7 @@ describe('blocking worker settlement', () => { const deps = fixture(); expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ status: 'completed', stdout: result.stdout, stderr: '', files: [], + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, }); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts index 03e386d4..1505f818 100644 --- a/service/src/service/blocking-poll.ts +++ b/service/src/service/blocking-poll.ts @@ -33,6 +33,7 @@ export async function pollBlockingExecution( stdout?: string; stderr?: string; files?: t.FileRefs; + deleted_files?: string[]; artifact_delivery?: t.ArtifactDeliveryFailure; artifact_truncation?: t.ArtifactTruncation; }> { @@ -48,6 +49,7 @@ export async function pollBlockingExecution( stdout: result.stdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, }; diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index bb140013..1063fbfe 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -1165,6 +1165,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, session_id: state.session_id, @@ -1179,6 +1180,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, session_id: state.session_id, @@ -1571,6 +1573,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + deleted_files: state.deleted_files, artifact_delivery: state.artifact_delivery, artifact_truncation: state.artifact_truncation, session_id: execution.session_id, @@ -1874,6 +1877,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + deleted_files: state.deleted_files, artifact_delivery: state.artifact_delivery, artifact_truncation: state.artifact_truncation, session_id, diff --git a/service/src/types/service.ts b/service/src/types/service.ts index a0c78486..0404ad16 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -141,6 +141,7 @@ export type ExecuteResponse = { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; }; @@ -259,6 +260,7 @@ export type ExecuteResult = { stdout: string; stderr: string; files: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; code?: number | null; @@ -408,6 +410,7 @@ export interface ProgrammaticResponse { stdout?: string; stderr?: string; files?: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; /** Top-level execution session id (one sandbox PTC invocation). */ diff --git a/service/src/workers.ts b/service/src/workers.ts index a7153b53..dbfd544b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -305,6 +305,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { * `[]` so the strictened response type from Phase B doesn't * surface a regression that wasn't there before. */ files: files ?? [], + ...(responseData.deleted_files != null + ? { deleted_files: responseData.deleted_files } + : {}), ...(responseData.artifact_delivery != null ? { artifact_delivery: responseData.artifact_delivery } : {}), From 926569e38975acc39ff202ef5800c83e8e0bc464 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:44:10 -0400 Subject: [PATCH 098/116] fix: preserve trusted jq path for PTC (#207) --- packages/code/src/native-process.test.ts | 14 ++++++++++-- packages/code/src/native-process.ts | 11 +++++----- packages/code/src/native-sandbox.ts | 5 +++++ service/src/preamble-bash.test.ts | 8 +++++++ service/src/preamble-bash.ts | 27 ++++++++++++------------ 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index e96f1e65..2f38d642 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'node:events'; import test from 'node:test'; import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import { NativeProcessWorkspaceCommandSandbox, @@ -199,6 +199,7 @@ test('programmatic executor resolves and scopes credentials to its command', asy { workspaceRoot: tmpdir(), programmaticFileUpstream: 'http://127.0.0.1:3190', + environment: { PATH: '/sandbox-only' }, maskedEnvironment: { variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], async resolve() { @@ -225,9 +226,18 @@ test('programmatic executor resolves and scopes credentials to its command', asy JSON.stringify(fake.options).includes('per-programmatic-secret'), false, ); - const message = fake.messages.find( + const message = fake.messages.find( candidate => candidate.type === 'programmatic', )!; + const prepareMessage = fake.messages.find( + candidate => candidate.type === 'prepare', + )!; + assert.equal(typeof prepareMessage.options.jqPath, 'string'); + assert.equal(prepareMessage.options.jqPath.startsWith('/'), true); + assert.equal( + '/sandbox-only'.split(':').includes(dirname(prepareMessage.options.jqPath)), + false, + ); assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); assert.equal( message.wrappedCommand, diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index d2ea0d72..4ed3ffec 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -61,7 +61,7 @@ export async function trustedProgrammaticExecutable(candidate: string, workspace async function resolveProgrammaticShell( options: NativeProcessSandboxOptions, -): Promise { +): Promise<{ shellPath: string; jqPath: string }> { const environment = options.environment ?? process.env; const shellPath = options.shellPath != null @@ -99,7 +99,7 @@ async function resolveProgrammaticShell( 'COMMAND_UNAVAILABLE', ); } - return shellPath; + return { shellPath, jqPath }; } /** Only OS discovery and conventional proxy settings cross into the executor. @@ -211,9 +211,9 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan } private async start(): Promise { - const programmaticShellPath = this.options.programmaticFileUpstream + const programmaticExecutables = this.options.programmaticFileUpstream ? await resolveProgrammaticShell(this.options) - : this.options.shellPath; + : undefined; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], @@ -299,7 +299,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan protectedPaths, allowedDomains, homeDirectory, - shellPath: programmaticShellPath ?? shellPath, + shellPath: programmaticExecutables?.shellPath ?? shellPath, + jqPath: programmaticExecutables?.jqPath, programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 8d150d9f..91e9832d 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -169,6 +169,8 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { platform?: NodeJS.Platform; /** Trusted shell path used by SRT on POSIX hosts. */ shellPath?: string; + /** Trusted jq path used by generated programmatic scripts. */ + jqPath?: string; /** Host-owned credentials exposed only as SRT sentinels inside the sandbox. */ maskedEnvironment?: { variables: Array<{ @@ -702,6 +704,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox '_ptc_pending_result.json', ), LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', + ...(this.options.jqPath + ? { LIBRECHAT_CODE_JQ_PATH: this.options.jqPath } + : {}), PTC_HISTORY_PATH: join( canonicalDataDirectory, '_ptc_history.json', diff --git a/service/src/preamble-bash.test.ts b/service/src/preamble-bash.test.ts index e35a77f5..e110abdc 100644 --- a/service/src/preamble-bash.test.ts +++ b/service/src/preamble-bash.test.ts @@ -186,6 +186,14 @@ printf '%s\\n' "$_PTC_PENDING_FILE" "$_PTC_ERROR_FILE" "$_PTC_COUNTER_FILE" rmSync(dir, { recursive: true, force: true }); } }); + + test('uses the trusted jq path instead of resolving jq through PATH', () => { + const preamble = generateBashReplayPreamble({ executionId, tools }); + expect(preamble).toContain( + '_PTC_JQ_PATH="${LIBRECHAT_CODE_JQ_PATH:-jq}"', + ); + expect(preamble).not.toMatch(/(^|[|;(]\s*)jq\s/m); + }); }); describe('generateBashReplayPreamble - command substitution pending emission', () => { diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index c442b495..31ed4c6b 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -150,6 +150,7 @@ _PTC_SENTINEL_START="${scopedStart}" _PTC_SENTINEL_END="${scopedEnd}" _PTC_HISTORY_PATH="\${PTC_HISTORY_PATH:-${PTC_HISTORY_SANDBOX_PATH}}" _PTC_CONTROL_PATH="\${LIBRECHAT_CODE_CONTROL_PATH:-}" +_PTC_JQ_PATH="\${LIBRECHAT_CODE_JQ_PATH:-jq}" _PTC_RUNTIME_DIR="\${TMPDIR:-/tmp}" _ptc_mktemp() { mktemp "\${_PTC_RUNTIME_DIR%/}/$1.XXXXXX" @@ -222,7 +223,7 @@ _ptc_sha256() { _ptc_hash_input() { local _ptc_canonical - _ptc_canonical=$(printf '%s' "$1" | jq -cS . 2>/dev/null) || return 1 + _ptc_canonical=$(printf '%s' "$1" | "$_PTC_JQ_PATH" -cS . 2>/dev/null) || return 1 printf '%s' "$_ptc_canonical" | _ptc_sha256 } @@ -369,7 +370,7 @@ _ptc_maybe_emit_pending() { return 0 fi local _ptc_payload - if ! _ptc_payload=$(jq -c -s '{pending:.}' "$_PTC_PENDING_FILE" 2>/dev/null); then + if ! _ptc_payload=$("$_PTC_JQ_PATH" -c -s '{pending:.}' "$_PTC_PENDING_FILE" 2>/dev/null); then printf 'failed to serialize pending PTC tool calls\\n' >&2 _ptc_cleanup_tempfiles trap - DEBUG EXIT @@ -480,7 +481,7 @@ _ptc_history_matches_by_signature() { return 0 fi # Path, not inline: large input can exceed ARG_MAX via --argjson. - jq -c \\ + "$_PTC_JQ_PATH" -c \\ --arg nm "$_ptc_name" \\ --arg site "$_ptc_call_site" \\ --arg hash "$_ptc_input_hash" \\ @@ -504,7 +505,7 @@ _ptc_first_unconsumed_history_match() { local _ptc_key while IFS= read -r _ptc_match; do [ -n "$_ptc_match" ] || continue - _ptc_key=$(printf '%s' "$_ptc_match" | jq -r '.key // empty' 2>/dev/null) + _ptc_key=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -r '.key // empty' 2>/dev/null) if [ -n "$_ptc_key" ] && ! grep -Fxq "$_ptc_key" "$_PTC_CONSUMED_FILE" 2>/dev/null; then printf '%s' "$_ptc_match" return 0 @@ -516,15 +517,15 @@ _ptc_first_unconsumed_history_match() { _ptc_print_history_entry() { local _ptc_entry="$1" local _ptc_is_err - _ptc_is_err=$(printf '%s' "$_ptc_entry" | jq -r 'if type == "object" then (.is_error // false) else false end' 2>/dev/null) + _ptc_is_err=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -r 'if type == "object" then (.is_error // false) else false end' 2>/dev/null) if [ "$_ptc_is_err" = "true" ]; then local _ptc_msg - _ptc_msg=$(printf '%s' "$_ptc_entry" | jq -r '.error_message // "tool execution failed"' 2>/dev/null) + _ptc_msg=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -r '.error_message // "tool execution failed"' 2>/dev/null) _ptc_write_error "$_ptc_msg" exit 1 fi local _ptc_result - _ptc_result=$(printf '%s' "$_ptc_entry" | jq -c 'if type == "object" and has("result") then .result else . end' 2>/dev/null || printf 'null') + _ptc_result=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -c 'if type == "object" and has("result") then .result else . end' 2>/dev/null || printf 'null') printf '%s' "$_ptc_result" return 0 } @@ -535,7 +536,7 @@ _ptc_history_entry_matches_current_call() { local _ptc_input_file="$3" local _ptc_input_hash="$4" # Path, same ARG_MAX reason as above. - printf '%s' "$_ptc_entry" | jq -e \\ + printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -e \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ --slurpfile inp_arr "$_ptc_input_file" \\ @@ -555,7 +556,7 @@ _ptc_call_tool() { local _ptc_call_site="\${BASH_LINENO[1]:-\${BASH_LINENO[0]:-0}}" # Reject extra trailing JSON values instead of silently dropping them. - if ! printf '%s' "$_ptc_input" | jq -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then + if ! printf '%s' "$_ptc_input" | "$_PTC_JQ_PATH" -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then _ptc_write_error "tool input for $_ptc_name must be a single JSON object, got: $_ptc_input" exit 1 fi @@ -583,8 +584,8 @@ _ptc_call_tool() { if [ -n "$_ptc_match" ] && [ "$_ptc_match" != "null" ]; then local _ptc_matched_call_id local _ptc_matched_entry - _ptc_matched_call_id=$(printf '%s' "$_ptc_match" | jq -r '.key' 2>/dev/null) - _ptc_matched_entry=$(printf '%s' "$_ptc_match" | jq -c '.value' 2>/dev/null) + _ptc_matched_call_id=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -r '.key' 2>/dev/null) + _ptc_matched_entry=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -c '.value' 2>/dev/null) printf '%s\\n' "$_ptc_matched_call_id" >> "$_PTC_CONSUMED_FILE" _ptc_mark_counter_at_least "$_ptc_matched_call_id" _ptc_release_lock @@ -598,7 +599,7 @@ _ptc_call_tool() { while :; do _ptc_call_id=$(_ptc_next_call_id) if [ -r "$_PTC_HISTORY_PATH" ]; then - _ptc_entry=$(jq -c --arg id "$_ptc_call_id" '.[$id] // empty' "$_PTC_HISTORY_PATH" 2>/dev/null || printf '') + _ptc_entry=$("$_PTC_JQ_PATH" -c --arg id "$_ptc_call_id" '.[$id] // empty' "$_PTC_HISTORY_PATH" 2>/dev/null || printf '') else _ptc_entry="" fi @@ -614,7 +615,7 @@ _ptc_call_tool() { fi done - if ! printf '%s' "$_ptc_input" | jq -c -n \\ + if ! printf '%s' "$_ptc_input" | "$_PTC_JQ_PATH" -c -n \\ --arg cid "$_ptc_call_id" \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ From 10ac19b74f87ec2ff4c9ff7e81198cc1f6b5e830 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:51:48 -0400 Subject: [PATCH 099/116] fix: isolate native PTC readiness and watchdog phases (#208) --- packages/code/src/native-process-child.ts | 89 ++++++-- packages/code/src/native-process.test.ts | 234 +++++++++++++++++++++- packages/code/src/native-process.ts | 140 ++++++++++--- packages/code/src/native-programmatic.ts | 15 +- packages/code/src/native-sandbox.ts | 17 +- 5 files changed, 444 insertions(+), 51 deletions(-) diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 2e8beb38..c3b9614b 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -11,7 +11,12 @@ import type { // argv credentials, bridge token, or persisted pairing material is required. let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; let programmaticExecutor: NativeWorkspaceProgrammaticExecutor | undefined; +let programmaticReady: Promise | undefined; +let programmaticFileUpstream: string | undefined; let active: { id: string; controller: AbortController } | undefined; +let commitAcknowledgement: + | { id: string; acknowledge(): void } + | undefined; let busy = false; let credentials: Record = {}; let wrappedCommand: string | undefined; @@ -25,6 +30,33 @@ function reply(message: object): void { /* Parent was lost. */ } } +async function awaitCommitAcknowledgement( + id: string, + signal: AbortSignal, +): Promise { + await new Promise((resolve, reject) => { + const abort = () => { + commitAcknowledgement = undefined; + reject( + new WorkspaceToolError( + 'Programmatic execution aborted before commit', + 'EXECUTION_ABORTED', + ), + ); + }; + commitAcknowledgement = { + id, + acknowledge() { + signal.removeEventListener('abort', abort); + commitAcknowledgement = undefined; + resolve(); + }, + }; + signal.addEventListener('abort', abort, { once: true }); + reply({ id, phase: 'commit' }); + if (signal.aborted) abort(); + }); +} let shuttingDown = false; const shutdown = () => { if (shuttingDown) return; @@ -59,19 +91,29 @@ process.on('message', async (raw: unknown) => { workspaceId?: string; credentials?: Record; wrappedCommand?: string; + programmaticShellPath?: string; + programmaticJqPath?: string; }; if (!message || typeof message.id !== 'string') return; if (message.type === 'cancel') { if (active?.id === message.id) active.controller.abort(); return; } + if (message.type === 'commit-ack') { + if (commitAcknowledgement?.id === message.id) { + commitAcknowledgement.acknowledge(); + } + return; + } if (busy) return; busy = true; + let mutationStarted = false; try { let result: unknown; if (message.type === 'prepare' && !sandbox) { - const { variables, programmaticFileUpstream, ...options } = + const { variables, programmaticFileUpstream: upstream, ...options } = message.options; + programmaticFileUpstream = upstream; sandbox = new NativeSrtWorkspaceCommandSandbox({ ...options, ...(variables @@ -89,32 +131,55 @@ process.on('message', async (raw: unknown) => { : {}), }); await sandbox.prepare(); - programmaticExecutor = programmaticFileUpstream - ? new NativeWorkspaceProgrammaticExecutor({ - sandbox, - upstreamUrl: programmaticFileUpstream, - }) - : undefined; - await programmaticExecutor?.prepare(); } else if (message.type === 'execute' && sandbox) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; + mutationStarted = true; result = await sandbox.execute(message.request, active.controller.signal); } else if ( message.type === 'programmatic' && sandbox && - programmaticExecutor && + programmaticFileUpstream && message.programmaticRequest && - typeof message.workspaceId === 'string' + typeof message.workspaceId === 'string' && + typeof message.programmaticShellPath === 'string' && + typeof message.programmaticJqPath === 'string' ) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; + if (!programmaticExecutor) { + programmaticExecutor = new NativeWorkspaceProgrammaticExecutor({ + sandbox, + upstreamUrl: programmaticFileUpstream, + shellPath: message.programmaticShellPath, + jqPath: message.programmaticJqPath, + }); + programmaticReady = programmaticExecutor.prepare( + active.controller.signal, + ); + } + try { + await programmaticReady; + } catch (error) { + programmaticExecutor = undefined; + programmaticReady = undefined; + throw error; + } result = await programmaticExecutor.execute( message.programmaticRequest, message.workspaceId, active.controller.signal, + { + async beforeCommit() { + await awaitCommitAcknowledgement( + message.id, + active!.controller.signal, + ); + mutationStarted = true; + }, + }, ); } else if (message.type === 'close' && sandbox) { await sandbox.close(); @@ -134,11 +199,11 @@ process.on('message', async (raw: unknown) => { mutation: error instanceof WorkspaceToolError ? error.mutationMayHaveCommitted - : true, + : mutationStarted, requiresQuarantine: error instanceof WorkspaceToolError ? error.requiresQuarantine - : true, + : mutationStarted, }); } finally { active = undefined; diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 2f38d642..89651845 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -11,6 +11,7 @@ import { trustedProgrammaticExecutable, } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS } from './protocol.js'; test('preflight rejects relative and workspace-controlled executables including symlinks', async t => { const root = await mkdtemp(join(tmpdir(), 'native-ptc-path-')); @@ -99,6 +100,24 @@ function fixture( }; } +class ObservedWatchdogSandbox extends NativeProcessWorkspaceCommandSandbox { + readonly watchdogTimeouts: number[] = []; + readonly watchdogCallbacks: Array<() => void> = []; + + protected override scheduleRpcTimeout( + callback: () => void, + timeoutMs: number, + ): ReturnType { + this.watchdogTimeouts.push(timeoutMs); + this.watchdogCallbacks.push(callback); + return super.scheduleRpcTimeout(callback, timeoutMs); + } + + fireLatestWatchdog(): void { + this.watchdogCallbacks.at(-1)?.(); + } +} + test('executor bootstrap excludes bridge credentials and Node injection variables', async () => { assert.deepEqual( nativeExecutorEnvironment({ @@ -229,13 +248,12 @@ test('programmatic executor resolves and scopes credentials to its command', asy const message = fake.messages.find( candidate => candidate.type === 'programmatic', )!; - const prepareMessage = fake.messages.find( - candidate => candidate.type === 'prepare', - )!; - assert.equal(typeof prepareMessage.options.jqPath, 'string'); - assert.equal(prepareMessage.options.jqPath.startsWith('/'), true); + assert.equal(typeof message.programmaticShellPath, 'string'); + assert.equal(message.programmaticShellPath.startsWith('/'), true); + assert.equal(typeof message.programmaticJqPath, 'string'); + assert.equal(message.programmaticJqPath.startsWith('/'), true); assert.equal( - '/sandbox-only'.split(':').includes(dirname(prepareMessage.options.jqPath)), + '/sandbox-only'.split(':').includes(dirname(message.programmaticJqPath)), false, ); assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); @@ -246,8 +264,109 @@ test('programmatic executor resolves and scopes credentials to its command', asy await sandbox.close(); }); +test('omitted PTC timeout gives the commit watchdog the protocol execution default', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('message', { id: message.id, ok: true, result: {} }); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + replay_tool_count: 0, + max_output_files: 0, + files: [{ name: 'main.sh', content: 'sleep 45' }], + }, + }); + assert.ok( + sandbox.watchdogTimeouts.at(-1)! > + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ); + await sandbox.close(); +}); + +test('PTC watchdog budgets staging separately and resets when commit begins', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('message', { id: message.id, ok: true, result: {} }); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + run_timeout: 1_000, + replay_tool_count: 0, + max_output_files: 0, + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }); + assert.deepEqual(sandbox.watchdogTimeouts.slice(-2), [65_000, 6_000]); + assert.ok( + fake.messages.some(message => message.type === 'commit-ack'), + 'the child must not enter the mutating phase before the parent arms it', + ); + await sandbox.close(); +}); + +test('PTC-only preflight failures do not disable ordinary native commands', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + shellPath: '/definitely/missing/bash', + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + assert.deepEqual(await sandbox.execute(request), result); + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + !error.mutationMayHaveCommitted, + ); + assert.deepEqual(await sandbox.execute(request), result); + await sandbox.close(); +}); + test('programmatic executor preserves a child-reported pre-dispatch failure', async () => { - const fake = fixture((child, message) => + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') { + child.emit('message', { id: message.id, ok: true, result }); + return; + } child.emit('message', { id: message.id, ok: false, @@ -255,8 +374,39 @@ test('programmatic executor preserves a child-reported pre-dispatch failure', as errorMessage: 'Programmatic input download failed', mutation: false, requiresQuarantine: false, + }); + }); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, }), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, ); + assert.deepEqual(await sandbox.execute(request), result); + await sandbox.close(); +}); + +test('executor loss during programmatic staging is not an uncertain workspace mutation', async () => { + const fake = fixture((child, message) => { + if (message.type === 'programmatic') child.emit('exit', 1); + }); const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: tmpdir(), @@ -283,6 +433,76 @@ test('programmatic executor preserves a child-reported pre-dispatch failure', as await sandbox.close(); }); +test('programmatic staging watchdog expires without claiming a workspace mutation', async () => { + let staged!: () => void; + const staging = new Promise(resolve => { + staged = resolve; + }); + const fake = fixture((_child, message) => { + if (message.type === 'programmatic') staged(); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + const execution = sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }); + await staging; + sandbox.fireLatestWatchdog(); + + await assert.rejects( + execution, + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + assert.equal(fake.killCalls, 1); + await sandbox.close(); +}); + +test('executor loss after programmatic commit starts remains an uncertain mutation', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('exit', 1); + }); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted && + error.requiresQuarantine, + ); + await sandbox.close(); +}); + test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { const fake = fixture(child => child.emit('exit', 1)); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 4ed3ffec..81bbd9c4 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -7,6 +7,7 @@ import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, isWorkspaceToolRequest, isWorkspaceToolResult, @@ -29,6 +30,13 @@ export type NativeProcessSandboxOptions = Omit< }; const execFileAsync = promisify(execFile); +const PROGRAMMATIC_STAGING_TIMEOUT_MS = 60_000; +const PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; +const RPC_SETTLEMENT_SLACK_MS = 5_000; + +type RpcTimeoutBudget = + | number + | { stagingMs: number; commitMs: number }; async function systemProgrammaticExecutable( name: string, @@ -183,11 +191,16 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan private closing?: Promise; private failed = false; private terminationTimer?: ReturnType; + private programmaticExecutables?: Promise<{ + shellPath: string; + jqPath: string; + }>; private pending?: { id: string; resolve(value: unknown): void; reject(error: Error): void; mutation: boolean; + commit?(): void; }; constructor( @@ -199,6 +212,14 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ) => ChildProcess = fork, ) {} + /** Overridable only for deterministic watchdog tests. */ + protected scheduleRpcTimeout( + callback: () => void, + timeoutMs: number, + ): ReturnType { + return setTimeout(callback, timeoutMs); + } + async prepare(): Promise { if (this.failed || this.closing) throw this.unavailable(false); if (this.ready) return this.ready; @@ -210,10 +231,22 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return new NativeExecutorUnavailableError(mutation); } + private async resolveProgrammaticExecutables(): Promise<{ + shellPath: string; + jqPath: string; + }> { + this.programmaticExecutables ??= resolveProgrammaticShell(this.options); + try { + return await this.programmaticExecutables; + } catch (error) { + // An operator may install or repair this optional dependency while the + // worker stays online. Keep ordinary execution live and let PTC retry. + this.programmaticExecutables = undefined; + throw error; + } + } + private async start(): Promise { - const programmaticExecutables = this.options.programmaticFileUpstream - ? await resolveProgrammaticShell(this.options) - : undefined; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], @@ -237,6 +270,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan code?: unknown; errorMessage?: unknown; fatal?: unknown; + phase?: unknown; }; if ( !message || @@ -246,6 +280,25 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return; const pending = this.pending; if (!pending) return; + if (message.phase === 'commit') { + pending.mutation = true; + const commit = pending.commit; + pending.commit = undefined; + commit?.(); + try { + child.send({ type: 'commit-ack', id: pending.id }, error => { + if (!error) return; + this.failed = true; + this.terminate(); + pending.reject(this.unavailable(true)); + }); + } catch { + this.failed = true; + this.terminate(); + pending.reject(this.unavailable(true)); + } + return; + } if (message.fatal === true) this.failed = true; if (message.ok === true) pending.resolve(message.result); else { @@ -299,8 +352,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan protectedPaths, allowedDomains, homeDirectory, - shellPath: programmaticExecutables?.shellPath ?? shellPath, - jqPath: programmaticExecutables?.jqPath, + shellPath, programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, @@ -376,9 +428,12 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ); let credentials: Record | undefined; let wrappedCommand: string | undefined; + let programmaticExecutables: { shellPath: string; jqPath: string }; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); + programmaticExecutables = await this.resolveProgrammaticExecutables(); + if (signal?.aborted) throw new Error('aborted'); credentials = await this.options.maskedEnvironment?.resolve(signal); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( @@ -407,19 +462,11 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan workspaceId, credentials, wrappedCommand, + programmaticShellPath: programmaticExecutables.shellPath, + programmaticJqPath: programmaticExecutables.jqPath, }, - (request.body.run_timeout ?? 30_000) * - ((request.body.replay_tool_count ?? 0) > 0 ? 2 : 1) + - (Math.ceil( - request.body.files.filter(file => 'id' in file).length / 4, - ) + - Math.ceil( - (request.body.max_output_files ?? - BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, - )) * - (request.body.transfer_timeout_ms ?? 30_000) + - 5_000, - true, + this.programmaticWatchdogBudget(request), + false, signal, ); if (signal?.aborted) { @@ -437,6 +484,35 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return result; } + private programmaticWatchdogBudget( + request: BridgeWorkspaceProgrammaticRequest, + ): Exclude { + const runTimeoutMs = Math.min( + request.body.run_timeout ?? BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ); + const transferTimeoutMs = + request.body.transfer_timeout_ms ?? PROGRAMMATIC_TRANSFER_TIMEOUT_MS; + const inputBatches = Math.ceil( + request.body.files.filter(file => 'id' in file).length / 4, + ); + const outputBatches = Math.ceil( + (request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, + ); + return { + stagingMs: + PROGRAMMATIC_STAGING_TIMEOUT_MS + + inputBatches * transferTimeoutMs + + ((request.body.replay_tool_count ?? 0) > 0 ? runTimeoutMs : 0) + + RPC_SETTLEMENT_SLACK_MS, + commitMs: + runTimeoutMs + + outputBatches * transferTimeoutMs + + RPC_SETTLEMENT_SLACK_MS, + }; + } + private async executeOnce( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -498,7 +574,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan private async rpc( type: string, payload: object, - timeoutMs: number, + timeout: RpcTimeoutBudget, mutation: boolean, signal?: AbortSignal, ): Promise { @@ -506,7 +582,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan throw this.unavailable(false); const id = randomUUID(); const child = this.child; - let timer: ReturnType; + let timer: ReturnType | undefined; const abort = () => { try { if (child.connected) @@ -518,12 +594,24 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan }; try { return await new Promise((resolve, reject) => { - this.pending = { id, resolve, reject, mutation }; - timer = setTimeout(() => { - this.failed = true; - this.terminate(); - reject(this.unavailable(mutation)); - }, timeoutMs); + const schedule = (timeoutMs: number): void => { + if (timer) clearTimeout(timer); + timer = this.scheduleRpcTimeout(() => { + this.failed = true; + this.terminate(); + reject(this.unavailable(this.pending?.mutation ?? mutation)); + }, timeoutMs); + }; + this.pending = { + id, + resolve, + reject, + mutation, + ...(typeof timeout === 'number' + ? {} + : { commit: () => schedule(timeout.commitMs) }), + }; + schedule(typeof timeout === 'number' ? timeout : timeout.stagingMs); signal?.addEventListener('abort', abort, { once: true }); const sendFailed = () => { this.failed = true; @@ -540,7 +628,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (signal?.aborted) abort(); }); } finally { - clearTimeout(timer!); + if (timer) clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending = undefined; } diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index bd2700f2..9b6eea9e 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -216,6 +216,8 @@ export interface NativeWorkspaceProgrammaticOptions { Pick >; upstreamUrl: string; + shellPath?: string; + jqPath?: string; fetchImpl?: typeof fetch; } @@ -362,6 +364,7 @@ export class NativeWorkspaceProgrammaticExecutor { request: BridgeWorkspaceProgrammaticRequest, workspaceId: string, signal?: AbortSignal, + lifecycle?: { beforeCommit?(): Promise | void }, ): Promise { if (!isBridgeWorkspaceProgrammaticRequest(request)) { throw new WorkspaceToolError( @@ -456,7 +459,10 @@ export class NativeWorkspaceProgrammaticExecutor { errorOnExist: true, mode: constants.COPYFILE_FICLONE, }); - if (!probe) commandDispatched = true; + if (!probe) { + await lifecycle?.beforeCommit?.(); + commandDispatched = true; + } return await this.options.sandbox.executeProgrammatic( { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -473,7 +479,12 @@ export class NativeWorkspaceProgrammaticExecutor { }, directory, signal, - { probe, workspaceRoot }, + { + probe, + workspaceRoot, + shellPath: this.options.shellPath, + jqPath: this.options.jqPath, + }, ); }; diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 91e9832d..550d0d52 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -661,7 +661,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox request: WorkspaceExecuteCommandRequest, dataDirectory: string, signal?: AbortSignal, - options?: { probe?: boolean; workspaceRoot?: string }, + options?: { + probe?: boolean; + workspaceRoot?: string; + shellPath?: string; + jqPath?: string; + }, ): Promise { if (this.execution || this.closing) { throw new WorkspaceToolError( @@ -703,9 +708,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox canonicalDataDirectory, '_ptc_pending_result.json', ), - LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', - ...(this.options.jqPath - ? { LIBRECHAT_CODE_JQ_PATH: this.options.jqPath } + LIBRECHAT_CODE_BASH_PATH: + options?.shellPath ?? this.options.shellPath ?? '/bin/bash', + ...((options?.jqPath ?? this.options.jqPath) + ? { + LIBRECHAT_CODE_JQ_PATH: + options?.jqPath ?? this.options.jqPath, + } : {}), PTC_HISTORY_PATH: join( canonicalDataDirectory, From 3a2c2a0a974b3b01c1c509e40cc414f75b23faab Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 18:30:33 -0400 Subject: [PATCH 100/116] feat: Declare Named Worker Project Environments (#209) * feat: declare named worker project environments * fix: preserve environment trust and negotiated action boundaries * Harden environment loading and executor identity * Protect environment root traversal and exact config bytes * Reject self-controlled environment root aliases * Check filesystem identities at environment trust boundaries * Validate environment containment across Linux mount aliases * Handle stacked mounts conservatively without blocking unrelated paths --- packages/code/README.md | 52 +++ packages/code/package-lock.json | 18 +- packages/code/package.json | 3 +- packages/code/src/cli.ts | 249 +++++++++--- packages/code/src/environment-live.test.ts | 117 ++++++ packages/code/src/environment-mount.test.ts | 73 ++++ packages/code/src/environment-mount.ts | 123 ++++++ packages/code/src/environment.test.ts | 348 +++++++++++++++++ packages/code/src/environment.ts | 399 ++++++++++++++++++++ packages/code/src/private-storage.ts | 10 +- packages/code/src/protocol.ts | 239 ++++++++++-- packages/code/src/worker.ts | 21 +- packages/code/src/workspace-worker.test.ts | 50 +++ packages/code/src/workspace.ts | 3 + 14 files changed, 1617 insertions(+), 88 deletions(-) create mode 100644 packages/code/src/environment-live.test.ts create mode 100644 packages/code/src/environment-mount.test.ts create mode 100644 packages/code/src/environment-mount.ts create mode 100644 packages/code/src/environment.test.ts create mode 100644 packages/code/src/environment.ts diff --git a/packages/code/README.md b/packages/code/README.md index 87d9dded..c3bb16d3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -652,3 +652,55 @@ To recover a quarantined native root: The workspace selector in LibreChat must preserve these registered IDs. Adding roots here does not grant a principal access or change an agent's selected root. +# Named project environments + +An operator can keep a project definition outside the coding workspace and start +the worker with `librechat-code run --environment /operator/app.yaml +--allow-workspace-commands --allow-workspace-writes`. Existing pairing settings +still identify the machine and its principal. Repeat `--environment` for independent, +non-overlapping roots (up to 32). Do not combine definitions with workspace directory, +ID, or name flags or environment variables. + +```yaml +name: app-dev +root: /projects/app +repo: example/app +ref: main +setup: + command: npm ci + timeoutMs: 300000 +actions: + - name: typecheck + command: npm run typecheck + timeoutMs: 120000 +``` + +The root must already exist; relative roots resolve from the YAML file's directory. +Repository and ref are descriptive metadata, not a clone or checkout instruction. +No Git repository is required. Definitions are loaded once at startup, hashed into +the worker's policy identity, and protected from sandbox writes. All definition +files must be outside every registered root. Unknown fields are rejected. +On Linux, startup also verifies the mount namespace so bind mounts cannot expose +definitions or their controlling paths through a workspace. The mount table is +bounded to 4 MiB, with at most 256 exposed mount boundaries; stacked and hidden +mount mappings are considered conservatively. Operators must keep mount topology stable while the +worker runs. This inspection happens at startup, not on the command hot path. + +Setup is an operator-authorized startup command under the configured native sandbox +policy. It requires commands to be enabled, runs once per worker startup before +registration, and must be idempotent for restarts. Its timeout is bounded to five +minutes and captured output to 8 KiB. Setup failure prevents registration. A crash +or uncertain termination retains the existing workspace quarantine marker; inspect +the workspace before clearing quarantine. No setup output is sent to the model. + +Named actions are fixed commands without model-supplied substitution. The bridge +advertises only their names and the definition fingerprint, never their shell source +or host root. A command request can select `environmentAction: { name, fingerprint }`; +the worker resolves the command from its loaded definition and rejects stale revisions, +unknown names, other roots, or a changed working directory. Actions use ordinary +command authorization, queueing, cancellation and quarantine. They never override +deployment approval rules or expand the pairing's principal scope. + +Rollout: update Code API and the LibreChat environment-descriptor consumer before +enabling this opt-in flag on a worker. Older validators reject the additional metadata. +Existing workers without `--environment` continue to use their existing registration. diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index ac9affc8..426b950f 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", - "koffi": "3.2.1" + "koffi": "3.2.1", + "yaml": "2.9.1" }, "bin": { "librechat-code": "dist/cli.js" @@ -414,6 +415,21 @@ "dev": true, "license": "MIT" }, + "node_modules/yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/packages/code/package.json b/packages/code/package.json index 452a8be7..b00ca807 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -65,6 +65,7 @@ }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", - "koffi": "3.2.1" + "koffi": "3.2.1", + "yaml": "2.9.1" } } diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 289b0dc4..5498af63 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,11 @@ import { realpath, stat } from 'node:fs/promises'; import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, + EnvironmentWorkspaceTools, +} from './environment.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -61,7 +66,8 @@ function workspaceSecurityIdentity( configuredToken: string | undefined, ): string { return ( - pairedPublicKey ?? required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) + pairedPublicKey ?? + required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) ); } @@ -70,7 +76,8 @@ function workspaceQuarantinePath(options: { workerId: string; workspaceRoot?: string; }): string { - const override = process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); + const override = + process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); if (override) return override; return defaultWorkspaceQuarantinePath({ ...options, @@ -88,7 +95,7 @@ function list(value: string | undefined): string[] { return ( value ?.split(',') - .map((item) => item.trim()) + .map(item => item.trim()) .filter(Boolean) ?? [] ); } @@ -127,7 +134,7 @@ function option(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index >= 0) return args[index + 1]; return args - .find((value) => value.startsWith(`${name}=`)) + .find(value => value.startsWith(`${name}=`)) ?.slice(name.length + 1); } @@ -175,7 +182,9 @@ function githubCredentials(): { try { parsedApiUrl = new URL(apiUrl); } catch { - throw new Error('LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL'); + throw new Error( + 'LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL', + ); } apiHost = parsedApiUrl.hostname.toLowerCase() === 'api.github.com' @@ -288,7 +297,7 @@ async function relay(): Promise { process.stdout.write( `librechat-code: file relay listening at ${handle.url}\n`, ); - await new Promise((resolve) => { + await new Promise(resolve => { process.once('SIGINT', resolve); process.once('SIGTERM', resolve); }); @@ -299,6 +308,47 @@ async function run( runtimeSessionId?: string, args: string[] = [], ): Promise { + const environmentPaths: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--environment') { + const path = args[++i]; + if (!path || path.startsWith('--')) + throw new Error('--environment requires a YAML file'); + environmentPaths.push(path); + } else if (args[i].startsWith('--environment=')) { + const path = args[i].slice('--environment='.length); + if (!path) throw new Error('--environment requires a YAML file'); + environmentPaths.push(path); + } + } + if (environmentPaths.length > 32) + throw new Error('At most 32 environments may be registered'); + const environments = await Promise.all( + environmentPaths.map(loadCodeEnvironment), + ); + if ( + environments.length && + (runtimeSessionId != null || + args.some(arg => + [ + '--worker-dir', + '--default-workspace', + '--workspace', + '--workspace-id', + '--workspace-name', + ].some(flag => arg === flag || arg.startsWith(`${flag}=`)), + ) || + [ + process.env.LIBRECHAT_CODE_WORKER_DIR, + process.env.LIBRECHAT_CODE_WORKSPACE_ID, + process.env.LIBRECHAT_CODE_WORKSPACE_NAME, + ].some(value => value?.trim()) || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === 'true') + ) { + throw new Error( + '--environment cannot be combined with workspace directory, ID, or name settings', + ); + } const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); @@ -347,7 +397,8 @@ async function run( ); } const nsjailDockerMode = - runtimeMode === 'docker-nsjail' || runtimeMode === 'docker-macos-nsjail'; + runtimeMode === 'docker-nsjail' || + runtimeMode === 'docker-macos-nsjail'; const sandboxEndpoint = process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? 'http://127.0.0.1:2000/api/v2'; @@ -374,16 +425,18 @@ async function run( runtimeSessionId == null && (fileRelayUpstream?.length ?? 0) > 0; const workspaceId = + environments[0]?.definition.name ?? option(args, '--workspace-id') ?? process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; const explicitWorkerDirectory = - runtimeSessionId == null + environments[0]?.definition.root ?? + (runtimeSessionId == null ? nonEmpty( option(args, '--worker-dir') ?? process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), ) - : undefined; + : undefined); const useDefaultWorkspace = runtimeSessionId == null && (args.includes('--default-workspace') || @@ -403,11 +456,25 @@ async function run( option(args, '--command-sandbox') ?? process.env.LIBRECHAT_CODE_COMMAND_SANDBOX?.trim().toLowerCase() ?? (nsjailDockerMode ? 'runtime' : 'native-srt'); - if (commandSandboxMode !== 'native-srt' && commandSandboxMode !== 'runtime') { + if ( + commandSandboxMode !== 'native-srt' && + commandSandboxMode !== 'runtime' + ) { throw new Error( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + if (environments.length && commandSandboxMode !== 'native-srt') { + throw new Error('Environment definitions require native-srt'); + } + if ( + environments.some(environment => environment.definition.setup) && + !allowWorkspaceCommands + ) { + throw new Error( + 'Environment setup requires --allow-workspace-commands', + ); + } const nativeProgrammaticEnabled = allowWorkspaceCommands && commandSandboxMode === 'native-srt' && @@ -486,7 +553,8 @@ async function run( } } const mutationQuarantinePath = - (allowWorkspaceWrites || allowWorkspaceCommands) && canonicalWorkerDirectory + (allowWorkspaceWrites || allowWorkspaceCommands) && + canonicalWorkerDirectory ? workspaceQuarantinePath({ codeApiUrl, workerId, @@ -508,14 +576,27 @@ async function run( root: canonicalWorkerDirectory, writable: allowWorkspaceWrites, name: + environments[0]?.definition.name ?? option(args, '--workspace-name') ?? process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? (useDefaultWorkspace ? workspaceId - : defaultWorkspaceName(workerDirectory!, workspaceId)), + : defaultWorkspaceName( + workerDirectory!, + workspaceId, + )), }, ] : []; + for (const environment of environments.slice(1)) { + roots.push({ + id: environment.definition.name, + name: environment.definition.name, + root: environment.definition.root, + writable: allowWorkspaceWrites, + }); + } + await assertEnvironmentDefinitionsOutsideRoots(environments, roots); for (let i = 0; i < args.length; i++) { if ( args[i] === '--workspace' && @@ -551,16 +632,18 @@ async function run( if (roots.length > 32) throw new Error('At most 32 workspace roots may be registered'); const rootIdentities = await Promise.all( - roots.map((root) => stat(root.root)), + roots.map(root => stat(root.root)), ); - const normalized = roots.map((root) => root.root); + const normalized = roots.map(root => root.root); for (let i = 0; i < roots.length; i++) for (let j = 0; j < i; j++) { const inside = (a: string, b: string): boolean => { const path = relative(a, b); return ( path === '' || - (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + (path !== '..' && + !path.startsWith(`..${sep}`) && + !isAbsolute(path)) ); }; if ( @@ -578,7 +661,9 @@ async function run( workspaceLeaseSlots > 1 && (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') ) { - throw new Error('Concurrent workspace leases require native-srt commands'); + throw new Error( + 'Concurrent workspace leases require native-srt commands', + ); } if ( roots.length > 1 && @@ -589,7 +674,7 @@ async function run( ); } const rootQuarantinePaths = new Map( - roots.map((root) => [ + roots.map(root => [ root.id, workspaceQuarantinePath({ codeApiUrl, @@ -676,7 +761,10 @@ async function run( token: createHmac( 'sha256', pairedIdentity?.privateKey ?? - required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken), + required( + 'LIBRECHAT_CODE_WORKER_TOKEN', + configuredToken, + ), ) .update('librechat-code-file-relay-v1') .digest('hex'), @@ -712,8 +800,11 @@ async function run( image: runtimeImage, ...(nsjailDockerMode && runtimeSessionId == null ? (() => { - const { seccompProfile, packagesPath, profileRevision } = - nsjailLaunchProfile!; + const { + seccompProfile, + packagesPath, + profileRevision, + } = nsjailLaunchProfile!; return { capabilities: MACOS_NSJAIL_CAPABILITIES, securityOptions: [`seccomp=${seccompProfile}`], @@ -733,10 +824,12 @@ async function run( httpClient: 'bun' as const, environment: { SANDBOX_USE_CGROUPV2: 'false', - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: + 'false', ...(workspaceMount ? { - SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', + SANDBOX_EXTERNAL_WORKSPACE_ENABLED: + 'true', SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, SANDBOX_EXTERNAL_WORKSPACE_TOKEN: @@ -745,15 +838,21 @@ async function run( : {}), ...(fileRelayProfile ? { - EGRESS_GATEWAY_URL: fileRelayProfile.url, + EGRESS_GATEWAY_URL: + fileRelayProfile.url, SANDBOX_PRIME_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, + fileRelayLimits! + .maxConcurrentRequests, ), - SANDBOX_UPLOAD_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, + SANDBOX_UPLOAD_CONCURRENCY: + String( + fileRelayLimits! + .maxConcurrentRequests, ), - SANDBOX_FILE_RELAY_TOKEN: fileRelayProfile.token, - SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_FILE_RELAY_TOKEN: + fileRelayProfile.token, + SANDBOX_REQUIRE_EGRESS_MANIFEST: + 'true', SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: executionManifestPublicKey!, } @@ -766,8 +865,10 @@ async function run( bindMounts: [workspaceMount], environment: { SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', - SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, - SANDBOX_EXTERNAL_WORKSPACE_TOKEN: workspaceCommandToken!, + SANDBOX_EXTERNAL_WORKSPACE_ROOT: + workspaceMount.target, + SANDBOX_EXTERNAL_WORKSPACE_TOKEN: + workspaceCommandToken!, }, } : {}), @@ -781,6 +882,7 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, ].filter((path): path is string => path != null), @@ -814,7 +916,7 @@ async function run( ? roots.length > 1 || workspaceLeaseSlots > 1 ? new NativeWorkspaceCommandPool( new Map( - roots.map((root) => [ + roots.map(root => [ root.id, { ...nativeOptions, workspaceRoot: root.root }, ]), @@ -826,7 +928,7 @@ async function run( if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ workspaceTools, - commandWorkspaces: roots.map((root) => root.id), + commandWorkspaces: roots.map(root => root.id), ...(nativeProgrammaticEnabled ? { programmaticLanguages: ['bash'] } : {}), @@ -839,6 +941,12 @@ async function run( }), }); } + if (workspaceTools && environments.length) { + workspaceTools = new EnvironmentWorkspaceTools( + workspaceTools, + environments, + ); + } const capabilities = { statefulWorkspace, sandboxProfile: @@ -854,6 +962,11 @@ async function run( policyDigest: createHash('sha256') .update(policy) .update( + environments.length + ? `\0environments\0${environments.map(environment => environment.fingerprint).join('\0')}` + : '', + ) + .update( allowWorkspaceCommands && commandSandboxMode === 'native-srt' ? `\0native-srt\0${serializeNativeSrtCommandPolicy(commandPolicy)}\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` : '', @@ -863,7 +976,9 @@ async function run( ...(workspaceLeaseSlots > 1 ? { workspaceLeaseSlots, requiresReadyConfirmation: true } : {}), - ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), + ...(workspaceTools + ? { workspaceTools: workspaceTools.capabilities } + : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { await fileRelaySupervisor?.stop().catch(() => undefined); @@ -874,6 +989,39 @@ async function run( try { await github.provider?.getCredential(controller.signal); await nativeCommandSandbox?.prepare(); + for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { + const setup = environment.definition.setup; + if (!setup || !nativeCommandSandbox) continue; + const id = environment.definition.name; + const guard = workspaceMutationGuard( + rootQuarantinePaths.get(id)!, + workerId, + id, + incarnationId, + ); + await guard.assertAvailable(); + await guard.arm('Environment setup did not settle', 'setup'); + const result = await nativeCommandSandbox.execute( + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: id, + command: setup.command, + timeoutMs: setup.timeoutMs, + maxOutputBytes: 8192, + }, + controller.signal, + ); + await guard.clear('setup'); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `Environment ${id} setup failed; inspect the setup command before restarting`, + ); + } + process.stdout.write( + `librechat-code: environment ${id} prepared\n`, + ); + } } catch (error) { await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); @@ -895,7 +1043,7 @@ async function run( ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( - roots.map((root) => [ + roots.map(root => [ root.id, workspaceMutationGuard( rootQuarantinePaths.get(root.id)!, @@ -913,7 +1061,8 @@ async function run( roots.length === 1 ? { async assertAvailable() { - const record = await loadWorkspaceMutationQuarantine( + const record = + await loadWorkspaceMutationQuarantine( mutationQuarantinePath, ); if (record != null) { @@ -925,15 +1074,18 @@ async function run( } }, async arm(reason) { - await saveWorkspaceMutationQuarantine(mutationQuarantinePath, { + await saveWorkspaceMutationQuarantine( + mutationQuarantinePath, + { version: 1, workerId, workspaceId, ownerId: incarnationId, quarantinedAt: new Date().toISOString(), reason, - }); }, + ); + }, async clear() { await clearWorkspaceMutationQuarantine( mutationQuarantinePath, @@ -950,7 +1102,7 @@ async function run( : undefined, onIdentityChange: pairedIdentity && identityPath - ? async (identity) => { + ? async identity => { await saveBridgeIdentity(identityPath, { ...pairedIdentity, credential: identity.credential, @@ -959,10 +1111,12 @@ async function run( } : undefined, onRegistered: fileRelaySupervisor - ? async (registration) => { + ? async registration => { if ( registration.registrationGeneration == null || - !Number.isSafeInteger(registration.registrationGeneration) || + !Number.isSafeInteger( + registration.registrationGeneration, + ) || registration.registrationGeneration < 1 ) { throw new Error( @@ -975,10 +1129,14 @@ async function run( ); } : undefined, - onError: (error) => { + onError: error => { const message = - error instanceof Error ? error.message : 'unknown bridge error'; - process.stderr.write(`librechat-code: reconnecting after ${message}\n`); + error instanceof Error + ? error.message + : 'unknown bridge error'; + process.stderr.write( + `librechat-code: reconnecting after ${message}\n`, + ); }, }); if (runtimeSessionId !== undefined) { @@ -994,7 +1152,10 @@ async function run( if (resetNativeRoot != null) { await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); - await worker.resetNativeWorkspace(resetNativeRoot, controller.signal); + await worker.resetNativeWorkspace( + resetNativeRoot, + controller.signal, + ); process.stdout.write( `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, ); diff --git a/packages/code/src/environment-live.test.ts b/packages/code/src/environment-live.test.ts new file mode 100644 index 00000000..bb84c8b5 --- /dev/null +++ b/packages/code/src/environment-live.test.ts @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +for (const { succeeds, reset } of [ + { succeeds: true, reset: false }, + { succeeds: false, reset: false }, + { succeeds: true, reset: true }, +]) { + test( + `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`, + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 20_000, + }, + async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-live-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await mkdir(root); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + `name: project\nroot: project\nsetup:\n command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n timeoutMs: 5000\n`, + ); + let registrations = 0; + let receive: (() => void) | undefined; + const registered = new Promise(resolve => { + receive = resolve; + }); + const server = createServer(async (request, response) => { + request.resume(); + if (request.url?.endsWith('/register')) { + registrations++; + if (reset) + await assert.rejects( + readFile(join(root, 'prepared.txt')), + { code: 'ENOENT' }, + ); + else + assert.equal( + await readFile(join(root, 'prepared.txt'), 'utf8'), + 'prepared', + ); + receive?.(); + } + response.writeHead(503).end(); + }); + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const child = spawn( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--environment', + path, + '--allow-workspace-commands', + ...(reset + ? ['--reset-workspace-quarantine', 'project'] + : []), + ], + { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + LIBRECHAT_CODE_URL: `http://127.0.0.1:${address.port}/v1`, + LIBRECHAT_CODE_WORKER_ID: 'environment-test', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-only-token', + LIBRECHAT_CODE_DEFAULT_WORKSPACE: 'false', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: join( + directory, + 'quarantine.json', + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + const exited = once(child, 'exit'); + t.after(() => child.kill('SIGKILL')); + let stderr = ''; + child.stderr.on('data', chunk => { + stderr += chunk.toString(); + }); + if (succeeds) { + await Promise.race([ + registered, + exited.then(() => { + throw new Error(stderr); + }), + ]); + child.kill('SIGTERM'); + await exited; + assert.ok(registrations > 0); + } else { + const [code] = await exited; + assert.notEqual(code, 0); + assert.match(stderr, /Environment project setup failed/); + assert.equal(registrations, 0); + } + }, + ); +} diff --git a/packages/code/src/environment-mount.test.ts b/packages/code/src/environment-mount.test.ts new file mode 100644 index 00000000..fdd5b581 --- /dev/null +++ b/packages/code/src/environment-mount.test.ts @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { assertEnvironmentMountIsolation } from './environment-mount.js'; + +const base = '1 0 8:1 / / rw - ext4 /dev/root rw\n'; +test('mount coordinates reject definition aliases in both directions and mounted files', () => { + for (const entry of [ + '2 1 8:1 /workspace/config /operator rw - ext4 /dev/root rw', + '2 1 8:1 /operator /workspace/config rw - ext4 /dev/root rw', + '2 1 8:1 /operator/app.yaml /workspace/app.yaml rw - ext4 /dev/root rw', + '2 1 8:1 /workspace/app.yaml /operator/app.yaml rw - ext4 /dev/root rw', + ]) + assert.throws( + () => + assertEnvironmentMountIsolation( + base + entry, + ['/operator/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); +}); +test('mount coordinates retain safe separate filesystems and escaped paths', () => { + assertEnvironmentMountIsolation( + base + '2 1 9:1 / /workspace rw - ext4 /dev/other rw', + ['/operator/app.yaml'], + ['/workspace'], + ); + assertEnvironmentMountIsolation( + base, + ['/operator/app.yaml'], + ['/workspace'], + ); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + + '2 1 8:1 /workspace/my\\040config /operator rw - ext4 /dev/root rw', + ['/operator/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); + assert.throws(() => assertEnvironmentMountIsolation('invalid', [], [])); + assertEnvironmentMountIsolation( + base + '2 1 9:1 / / rw - ext4 /dev/other rw', + ['/operator/app.yaml'], + ['/workspace'], + ); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + '2 1 9:1 / / rw - ext4 /dev/other rw', + ['/workspace/config/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); + const many = Array.from( + { length: 257 }, + (_, index) => + `${index + 2} 1 9:1 / /workspace/m${index} rw - ext4 /dev/other rw`, + ).join('\n'); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + many, + ['/operator/app.yaml'], + ['/workspace'], + ), + /Too many/, + ); +}); diff --git a/packages/code/src/environment-mount.ts b/packages/code/src/environment-mount.ts new file mode 100644 index 00000000..bbc8483e --- /dev/null +++ b/packages/code/src/environment-mount.ts @@ -0,0 +1,123 @@ +import { open } from 'node:fs/promises'; +import { posix } from 'node:path'; + +interface Mount { + device: string; + root: string; + point: string; +} +const inside = (root: string, path: string): boolean => + path === root || path.startsWith(root === '/' ? '/' : `${root}/`); +const decode = (path: string): string => { + if (!path.startsWith('/') || /\\(?!040|011|012|134)/.test(path)) + throw new Error('Invalid environment mount table'); + return path.replace(/\\(040|011|012|134)/g, (_, octal: string) => + String.fromCharCode(parseInt(octal, 8)), + ); +}; + +/** Compare filesystem coordinates, not mount aliases. Include mounted descendants of each grant. */ +export function createEnvironmentMountIsolation( + table: string, +): (controls: readonly string[], roots: readonly string[]) => void { + if (Buffer.byteLength(table) > 4 * 1024 * 1024) + throw new Error('Environment mount table exceeds limit'); + const mounts: Mount[] = table + .trimEnd() + .split('\n') + .map(line => { + const fields = line.split(' '); + const separator = fields.indexOf('-', 6); + if ( + separator < 6 || + fields.length !== separator + 4 || + !/^\d+:\d+$/.test(fields[2] ?? '') + ) + throw new Error('Invalid environment mount table'); + return { + device: fields[2], + root: decode(fields[3] ?? ''), + point: decode(fields[4] ?? ''), + }; + }); + const cache = new Map(); + const coordinate = (path: string): { device: string; path: string }[] => { + const cached = cache.get(path); + if (cached) return cached; + // Include every possible backing mapping. Hidden/stacked mounts may cause + // conservative rejection but must never hide an accessible control path. + const result = mounts + .filter(mount => inside(mount.point, path)) + .map(mount => ({ + device: mount.device, + path: posix.join(mount.root, posix.relative(mount.point, path)), + })); + if (!result.length || result.length > 256) + throw new Error( + 'Environment path has an unsupported mount mapping', + ); + cache.set(path, result); + return result; + }; + return (controls, roots) => { + const points = new Set(roots); + for (const mount of mounts) + if (roots.some(root => inside(root, mount.point))) + points.add(mount.point); + if (points.size > 256) + throw new Error('Too many workspace mount boundaries'); + const exposed = [...points].flatMap(coordinate); + if (exposed.length > 1024) + throw new Error('Too many workspace mount mappings'); + for (const control of controls) { + const target = coordinate(control); + if ( + target.some(target => + exposed.some( + root => + root.device === target.device && + inside(root.path, target.path), + ), + ) + ) { + throw new Error( + 'Environment control path is writable through a workspace mount alias', + ); + } + } + }; +} + +export function assertEnvironmentMountIsolation( + table: string, + controls: readonly string[], + roots: readonly string[], +): void { + createEnvironmentMountIsolation(table)(controls, roots); +} + +export async function readEnvironmentMountTable(): Promise { + if (process.platform !== 'linux') return undefined; + const handle = await open('/proc/self/mountinfo', 'r'); + try { + const buffer = Buffer.alloc(4 * 1024 * 1024 + 1); + let length = 0; + while (length < buffer.length) { + const result = await handle.read( + buffer, + length, + buffer.length - length, + null, + ); + if (!result.bytesRead) break; + length += result.bytesRead; + } + if (length === buffer.length) + throw new Error('Environment mount table exceeds limit'); + return new TextDecoder('utf-8', { fatal: true }).decode( + buffer.subarray(0, length), + ); + } finally { + await handle.close(); + } +} diff --git a/packages/code/src/environment.test.ts b/packages/code/src/environment.test.ts new file mode 100644 index 00000000..9080f6a2 --- /dev/null +++ b/packages/code/src/environment.test.ts @@ -0,0 +1,348 @@ +import assert from 'node:assert/strict'; +import { + mkdtemp, + mkdir, + writeFile, + rm, + symlink, + link, + open, + realpath, +} from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + parseCodeEnvironment, + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, + EnvironmentWorkspaceTools, +} from './environment.js'; +import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; +import { isValidBridgeWorkspaceToolCapabilities } from './protocol.js'; +import type { WorkspaceExecuteCommandRequest } from './protocol.js'; + +test('environment YAML validates setup and rejects unsupported policy or action fields', () => { + const definition = parseCodeEnvironment( + 'name: app\nroot: ./project\nsetup:\n command: npm ci\n', + ); + assert.equal(definition.setup?.timeoutMs, 300_000); + for (const suffix of [ + 'scope: { users: [anyone] }', + 'actions: [{}]', + 'unknown: true', + 'setup: { command: npm ci, timeoutMs: 600000 }', + 'setup: { command: npm ci, timeoutMs: -1 }', + 'setup: { command: npm ci, env: { SECRET: x } }', + 'name: duplicate', + 'repo: https://token@github.com/a/b', + ]) + assert.throws(() => + parseCodeEnvironment(`name: app\nroot: ./project\n${suffix}\n`), + ); + assert.throws(() => parseCodeEnvironment('name: &id app\nroot: *id')); + assert.throws(() => parseCodeEnvironment('x'.repeat(65_537))); + for (const field of ['setup', 'actions']) { + const command = '漢'.repeat(12_000); + const suffix = + field === 'setup' + ? `setup: { command: '${command}' }` + : `actions: [{ name: test, command: '${command}' }]`; + assert.throws(() => + parseCodeEnvironment(`name: app\nroot: project\n${suffix}`), + ); + } +}); + +test('named actions use the loaded definition, reject stale revisions and preserve command restrictions', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-action-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const local = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'app', root: directory }], + }); + const executed: WorkspaceExecuteCommandRequest[] = []; + const commands = new SandboxWorkspaceTools({ + workspaceTools: local, + commandWorkspaces: ['app'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute(request) { + executed.push(request); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'app', + stdout: '', + stderr: '', + exitCode: 0, + timedOut: false, + truncated: false, + }; + }, + }, + }); + const environments = [ + { + path: '/operator/environment.yaml', + fingerprint: 'a'.repeat(64), + definition: { + name: 'app', + root: directory, + actions: [ + { name: 'test', command: 'npm test', timeoutMs: 2000 }, + ], + }, + }, + ]; + const tools = new EnvironmentWorkspaceTools(commands, environments); + assert.ok(isValidBridgeWorkspaceToolCapabilities(tools.capabilities)); + assert.deepEqual(tools.capabilities.workspaces[0].environment?.actions, [ + 'test', + ]); + assert.equal( + JSON.stringify(tools.capabilities).includes('npm test'), + false, + ); + const request: WorkspaceExecuteCommandRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'app', + command: 'untrusted placeholder', + timeoutMs: 5000, + environmentAction: { name: 'test', fingerprint: 'a'.repeat(64) }, + }; + await tools.execute(request); + assert.equal(executed[0].command, 'npm test'); + assert.equal(executed[0].timeoutMs, 2000); + assert.equal(executed[0].environmentAction, undefined); + for (const altered of [ + { ...request, workspaceId: 'other' }, + { ...request, cwd: 'nested' }, + { + ...request, + environmentAction: { name: 'test', fingerprint: 'b'.repeat(64) }, + }, + { + ...request, + environmentAction: { name: 'other', fingerprint: 'a'.repeat(64) }, + }, + ]) + await assert.rejects( + tools.execute(altered), + /unavailable or its definition changed/, + ); + await assert.rejects(commands.execute(request), /not resolved/); + const readOnly = new EnvironmentWorkspaceTools(local, environments); + assert.deepEqual( + readOnly.capabilities.workspaces[0].environment?.actions, + [], + ); + await assert.rejects(readOnly.execute(request)); + assert.equal(executed.length, 1); +}); + +test('environment roots resolve relative to the definition and fingerprints cover setup', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-definition-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: "printf first" }\n', + ); + const first = await loadCodeEnvironment(path); + assert.ok(first.definition.root.endsWith('/project')); + await assertEnvironmentDefinitionsOutsideRoots( + [first], + [{ id: 'app', root: first.definition.root }], + ); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: "printf second" }\n', + ); + assert.notEqual( + (await loadCodeEnvironment(path)).fingerprint, + first.fingerprint, + ); + await assert.rejects(() => + assertEnvironmentDefinitionsOutsideRoots( + [first], + [ + { + id: 'parent', + root: first.definition.root.slice(0, -'/project'.length), + }, + ], + ), + ); + await symlink(path, join(directory, 'project', 'alias.yaml')); + const alias = await loadCodeEnvironment( + join(directory, 'project', 'alias.yaml'), + ); + await assert.rejects(() => + assertEnvironmentDefinitionsOutsideRoots( + [alias], + [{ id: 'app', root: first.definition.root }], + ), + ); + assert.equal( + (await loadCodeEnvironment(join(directory, 'project', 'alias.yaml'))) + .path, + first.path, + ); +}); + +test('rejects a trusted definition with an in-workspace hard link', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-hardlink-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile(path, 'name: app\nroot: project\n'); + await link(path, join(directory, 'project', 'alias.yaml')); + await assert.rejects(loadCodeEnvironment(path), /one link/); +}); + +test('rejects nested aliases passing through a workspace-controlled link', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-nested-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + const trusted = join(directory, 'trusted'); + await mkdir(root); + await mkdir(trusted); + await writeFile( + join(trusted, 'environment.yaml'), + `name: app\nroot: ${root}\n`, + ); + await symlink(trusted, join(root, 'pivot')); + await symlink(join(root, 'pivot'), join(directory, 'alias')); + const loaded = await loadCodeEnvironment( + join(directory, 'alias', 'environment.yaml'), + ); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [{ id: 'app', root }], + ), + /outside|mount alias/, + ); +}); + +test('reads complete definitions despite short filesystem reads', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-short-read-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: echo prepared }\n', + ); + const sample = await open(path); + const prototype = Object.getPrototypeOf(sample); + const read = prototype.read; + await sample.close(); + t.mock.method( + prototype, + 'read', + function ( + this: unknown, + buffer: Buffer, + offset: number, + length: number, + position: number, + ) { + return read.call( + this, + buffer, + offset, + Math.min(length, 7), + position, + ); + }, + ); + assert.equal( + (await loadCodeEnvironment(path)).definition.setup?.command, + 'echo prepared', + ); +}); + +test('rejects a root routed through another workspace and malformed UTF-8', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-root-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const rootA = join(directory, 'a'); + const rootB = join(directory, 'b'); + await mkdir(rootA); + await mkdir(rootB); + await symlink(rootA, join(rootB, 'pivot')); + const path = join(directory, 'environment.yaml'); + await writeFile(path, `name: a\nroot: ${join(rootB, 'pivot')}\n`); + const loaded = await loadCodeEnvironment(path); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [ + { id: 'a', root: rootA }, + { id: 'b', root: rootB }, + ], + ), + /root traversal|mount alias/, + ); + await symlink(rootA, join(rootA, 'self-pivot')); + await writeFile(path, `name: a\nroot: ${join(rootA, 'self-pivot')}\n`); + const selfControlled = await loadCodeEnvironment(path); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [selfControlled], + [{ id: 'a', root: rootA }], + ), + /root traversal|mount alias/, + ); + await writeFile( + path, + Buffer.concat([ + Buffer.from(`name: a\nroot: ${rootA}\nsetup: { command: echo `), + Buffer.from([0xff]), + Buffer.from(' }'), + ]), + ); + await assert.rejects(loadCodeEnvironment(path), /encoded data/); +}); + +test('rejects a FIFO definition without waiting for a writer', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-fifo-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'environment.yaml'); + execFileSync('mkfifo', ['-m', '600', path], { timeout: 2000 }); + await assert.rejects(loadCodeEnvironment(path), /Invalid environment file/); +}); + +test('rejects a filesystem-identical control directory despite a different root path', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-identity-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const trusted = join(directory, 'trusted'); + const alias = join(directory, 'alias'); + const project = join(directory, 'project'); + await mkdir(trusted); + await mkdir(project); + await symlink(trusted, alias); + const path = join(trusted, 'environment.yaml'); + await writeFile(path, `name: app\nroot: ${project}\n`); + const loaded = await loadCodeEnvironment(path); + // Unlike realpath-based containment, inode comparison also covers bind-mount aliases. + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [{ id: 'alias', root: alias }], + ), + /outside|mount alias/, + ); +}); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts new file mode 100644 index 00000000..700fc0dc --- /dev/null +++ b/packages/code/src/environment.ts @@ -0,0 +1,399 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { open, realpath, stat } from 'node:fs/promises'; +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; +import { parseDocument } from 'yaml'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, +} from './private-storage.js'; +import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, +} from './protocol.js'; +import type { LocalWorkspaceConfig } from './workspace.js'; +import { WorkspaceToolError } from './workspace.js'; +import { + createEnvironmentMountIsolation, + readEnvironmentMountTable, +} from './environment-mount.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceToolRequest, WorkspaceToolResult } from './protocol.js'; + +export interface CodeEnvironmentDefinition { + name: string; + root: string; + repo?: string; + ref?: string; + setup?: { command: string; timeoutMs: number }; + actions?: { name: string; command: string; timeoutMs: number }[]; +} + +export interface LoadedCodeEnvironment { + path: string; + sourceParents?: string[]; + rootPaths?: string[]; + definition: CodeEnvironmentDefinition; + fingerprint: string; +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function text(value: unknown, max: number): value is string { + return ( + typeof value === 'string' && + value.trim().length > 0 && + value.length <= max && + !value.includes('\0') + ); +} + +export function parseCodeEnvironment( + source: string, +): CodeEnvironmentDefinition { + if (Buffer.byteLength(source) > 65_536) + throw new Error('Environment file exceeds 64 KiB'); + const document = parseDocument(source, { + schema: 'core', + uniqueKeys: true, + }); + if (document.errors.length || document.warnings.length) { + throw new Error('Invalid environment YAML'); + } + const value: unknown = document.toJS({ maxAliasCount: 0 }); + if ( + !record(value) || + Object.keys(value).some( + key => + !['name', 'root', 'repo', 'ref', 'setup', 'actions'].includes( + key, + ), + ) || + !text(value.name, 64) || + !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(value.name) || + !text(value.root, 4096) || + (value.repo !== undefined && + (!text(value.repo, 256) || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(value.repo))) || + (value.ref !== undefined && + (!text(value.ref, 256) || /[\r\n]/.test(value.ref))) + ) { + throw new Error( + 'Invalid environment definition: expected name, root, optional repo, ref and setup', + ); + } + let setup: CodeEnvironmentDefinition['setup']; + if (value.setup !== undefined) { + if ( + !record(value.setup) || + Object.keys(value.setup).some( + key => !['command', 'timeoutMs'].includes(key), + ) || + !text(value.setup.command, 16_384) || + Buffer.byteLength(value.setup.command) > + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) { + throw new Error('Invalid environment setup'); + } + const timeoutMs = + value.setup.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS; + if ( + typeof timeoutMs !== 'number' || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS + ) { + throw new Error( + `Environment setup timeout must be between 1 and ${BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS} ms`, + ); + } + setup = { command: value.setup.command, timeoutMs }; + } + let actions: CodeEnvironmentDefinition['actions']; + if (value.actions !== undefined) { + if (!Array.isArray(value.actions) || value.actions.length > 32) + throw new Error('Invalid environment actions'); + const names = new Set(); + actions = value.actions.map((action: unknown) => { + if ( + !record(action) || + Object.keys(action).some( + key => !['name', 'command', 'timeoutMs'].includes(key), + ) || + !text(action.name, 64) || + !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(action.name) || + names.has(action.name) || + !text(action.command, 16_384) || + Buffer.byteLength(action.command) > + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) + throw new Error('Invalid environment action'); + const timeoutMs = action.timeoutMs ?? 30_000; + if ( + typeof timeoutMs !== 'number' || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS + ) + throw new Error('Invalid environment action timeout'); + names.add(action.name); + return { name: action.name, command: action.command, timeoutMs }; + }); + } + return { + name: value.name, + root: value.root, + ...(typeof value.repo === 'string' ? { repo: value.repo } : {}), + ...(typeof value.ref === 'string' ? { ref: value.ref } : {}), + ...(setup ? { setup } : {}), + ...(actions ? { actions } : {}), + }; +} + +/** Resolve actions only against the worker-owned snapshot, after normal command admission. */ +export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly environments: Map; + + constructor( + private readonly delegate: WorkspaceToolExecutor, + environments: LoadedCodeEnvironment[], + ) { + this.mutationFailuresAreAtomic = delegate.mutationFailuresAreAtomic; + this.environments = new Map( + environments.map(environment => [ + environment.definition.name, + environment, + ]), + ); + this.capabilities = { + ...delegate.capabilities, + workspaces: delegate.capabilities.workspaces.map(workspace => { + const environment = this.environments.get(workspace.id); + if (!environment) return workspace; + const operations = + workspace.operations ?? delegate.capabilities.operations; + return { + ...workspace, + environment: { + fingerprint: environment.fingerprint, + ...(environment.definition.repo + ? { repo: environment.definition.repo } + : {}), + ...(environment.definition.ref + ? { ref: environment.definition.ref } + : {}), + actions: operations.includes('execute_command') + ? (environment.definition.actions ?? []).map( + action => action.name, + ) + : [], + }, + }; + }), + }; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if ( + request.operation !== 'execute_command' || + !request.environmentAction + ) { + return this.delegate.execute(request, signal); + } + const environment = this.environments.get(request.workspaceId); + const action = environment?.definition.actions?.find( + action => action.name === request.environmentAction?.name, + ); + if ( + !environment || + environment.fingerprint !== request.environmentAction.fingerprint || + !action || + (request.cwd !== undefined && request.cwd !== '.') + ) { + throw new WorkspaceToolError( + 'Environment action is unavailable or its definition changed', + 'INVALID_REQUEST', + ); + } + const { environmentAction: _action, ...commandRequest } = request; + return this.delegate.execute( + { + ...commandRequest, + command: action.command, + timeoutMs: Math.min( + request.timeoutMs ?? action.timeoutMs, + action.timeoutMs, + ), + cwd: '.', + }, + signal, + ); + } +} + +export async function loadCodeEnvironment( + path: string, +): Promise { + const sourcePath = resolve(path); + const sourceParents = await assertPrivateStorageAncestors(sourcePath); + const canonicalPath = await realpath(sourcePath); + const handle = await open( + canonicalPath, + constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW, + ); + let definition: CodeEnvironmentDefinition; + try { + const metadata = await handle.stat(); + const self = process.getuid?.(); + if ( + metadata.nlink !== 1 || + (metadata.mode & 0o022) !== 0 || + (self !== undefined && metadata.uid !== self && metadata.uid !== 0) + ) { + throw new Error( + 'Environment definitions must have one link, a trusted owner and no group or other write permissions', + ); + } + await assertPrivateStorageAcl(handle, canonicalPath); + if (!metadata.isFile() || metadata.size > 65_536) + throw new Error('Invalid environment file'); + const buffer = Buffer.alloc(65_537); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const result = await handle.read( + buffer, + bytesRead, + buffer.length - bytesRead, + bytesRead, + ); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + const after = await handle.stat(); + if ( + bytesRead !== metadata.size || + after.size !== metadata.size || + after.mtimeMs !== metadata.mtimeMs || + after.ctimeMs !== metadata.ctimeMs + ) { + throw new Error('Environment definition changed while reading'); + } + definition = parseCodeEnvironment( + new TextDecoder('utf-8', { fatal: true }).decode( + buffer.subarray(0, bytesRead), + ), + ); + } finally { + await handle.close(); + } + const rootPath = resolve(dirname(canonicalPath), definition.root); + const rootPaths = await assertPrivateStorageAncestors(rootPath); + const root = await realpath(rootPath); + if (!(await stat(root)).isDirectory()) + throw new Error('Environment root must be a directory'); + definition = { ...definition, root }; + return { + path: canonicalPath, + sourceParents, + rootPaths, + definition, + fingerprint: createHash('sha256') + .update(JSON.stringify(definition)) + .digest('hex'), + }; +} + +/** A workspace must never be able to rewrite a definition used on the next startup. */ +export async function assertEnvironmentDefinitionsOutsideRoots( + environments: readonly LoadedCodeEnvironment[], + roots: readonly LocalWorkspaceConfig[], +): Promise { + if (!environments.length) return; + const mountTable = await readEnvironmentMountTable(); + if (mountTable !== undefined) { + const assertMountIsolation = + createEnvironmentMountIsolation(mountTable); + assertMountIsolation( + environments.flatMap(environment => [ + environment.path, + ...(environment.sourceParents ?? []), + ]), + roots.map(root => root.root), + ); + for (const environment of environments) { + assertMountIsolation( + environment.rootPaths ?? [], + roots + .filter(root => root.id !== environment.definition.name) + .map(root => root.root), + ); + assertMountIsolation( + (environment.rootPaths ?? []).filter( + path => path !== environment.definition.root, + ), + roots + .filter(root => root.id === environment.definition.name) + .map(root => root.root), + ); + } + } + const identities = new Map>(); + const identity = (path: string): Promise => { + let result = identities.get(path); + if (!result) { + result = stat(path).then( + metadata => `${metadata.dev}:${metadata.ino}`, + ); + identities.set(path, result); + } + return result; + }; + for (const environment of environments) { + for (const root of roots) { + const rootIdentity = await identity(root.root); + // No granted workspace may control how this root resolves on restart. + { + for (const component of environment.rootPaths ?? []) { + const path = relative(root.root, component); + if (path === '' && root.id === environment.definition.name) + continue; + if ( + (await identity(component)) === rootIdentity || + path === '' || + (!isAbsolute(path) && + path !== '..' && + !path.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Environment root traversal crosses a workspace-controlled component', + ); + } + } + } + for (const controlPath of [ + environment.path, + ...(environment.sourceParents ?? []), + ]) { + const path = relative(root.root, controlPath); + if ( + (await identity(controlPath)) === rootIdentity || + path === '' || + (!isAbsolute(path) && + path !== '..' && + !path.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Environment definitions must be outside every registered workspace root', + ); + } + } + } + } +} diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts index 25f23a2a..d896de33 100644 --- a/packages/code/src/private-storage.ts +++ b/packages/code/src/private-storage.ts @@ -48,12 +48,15 @@ export async function removePrivateStorageAcl( * links one component at a time so even intermediate link targets are checked. * Other local accounts cannot replace a checked entry: its parent is either * non-writable or sticky and the entry belongs to this account or root. + * Returns every traversed entry, including intermediate symlinks, so callers + * can also enforce containment restrictions without resolving those entries away. */ export async function assertPrivateStorageAncestors( path: string, allowMissing = false, -): Promise { +): Promise { assertPrivateStorageSupported(); + const visited: string[] = []; const uid = process.getuid!(); let current = '/'; const pending = (isAbsolute(path) ? path : `${process.cwd()}/${path}`).split('/'); @@ -63,7 +66,8 @@ export async function assertPrivateStorageAncestors( if (allowMissing && error.code === 'ENOENT') return undefined; throw error; }); - if (metadata === undefined) return; + if (metadata === undefined) return visited; + visited.push(current); if (metadata.uid !== uid && metadata.uid !== 0) { throw new BridgeProtocolError( `${current} is owned by another account (uid ${metadata.uid}), ` + @@ -102,7 +106,7 @@ export async function assertPrivateStorageAncestors( } let next = pending.shift(); while (next === '' || next === '.') next = pending.shift(); - if (next === undefined) return; + if (next === undefined) return visited; current = next === '..' ? dirname(current) : `${current === '/' ? '' : current}/${next}`; } } diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index b92d21ac..9199fcf0 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -21,7 +21,8 @@ export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES = 100; -export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY = 4; export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; @@ -41,26 +42,119 @@ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; * locally instead of discovering the mismatch only after mutating a workspace. */ const BRIDGE_ARTIFACT_EXTENSIONS = new Set([ - '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', - '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', - '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', - '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', - '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', - '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', - '.xml', '.yaml', '.yml', - '.ics', '.ical', '.ifb', '.icalendar', - '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', - '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', - '.odt', '.ods', '.odp', '.rtf', - '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', - '.tif', '.tiff', '.webp', - '.eot', '.ttf', '.woff', '.woff2', - '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', - '.tf', '.tfvars', '.tfstate', '.hcl', - '.dockerfile', '.Dockerfile', '.dockerignore', - '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', - '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', - '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', + '.c', + '.cs', + '.cpp', + '.go', + '.java', + '.js', + '.kt', + '.kts', + '.lua', + '.php', + '.pl', + '.ps1', + '.py', + '.r', + '.rb', + '.rs', + '.scala', + '.sh', + '.sql', + '.swift', + '.ts', + '.jsx', + '.tsx', + '.groovy', + '.css', + '.htm', + '.html', + '.less', + '.sass', + '.scss', + '.svg', + '.svelte', + '.vue', + '.adoc', + '.asciidoc', + '.md', + '.rst', + '.tex', + '.txt', + '.wiki', + '.csv', + '.json', + '.bson', + '.json5', + '.jsonl', + '.parquet', + '.tsv', + '.xml', + '.yaml', + '.yml', + '.ics', + '.ical', + '.ifb', + '.icalendar', + '.conf', + '.env', + '.gitignore', + '.ini', + '.properties', + '.toml', + '.doc', + '.docx', + '.pdf', + '.ppt', + '.pptx', + '.xls', + '.xlsx', + '.odt', + '.ods', + '.odp', + '.rtf', + '.avif', + '.bmp', + '.gif', + '.ico', + '.jpeg', + '.jpg', + '.png', + '.tif', + '.tiff', + '.webp', + '.eot', + '.ttf', + '.woff', + '.woff2', + '.7z', + '.bz2', + '.gz', + '.gzip', + '.rar', + '.tar', + '.zip', + '.tf', + '.tfvars', + '.tfstate', + '.hcl', + '.dockerfile', + '.Dockerfile', + '.dockerignore', + '.helmignore', + '.helmfile', + '.jenkinsfile', + '.vagrantfile', + '.eslintrc', + '.prettierrc', + '.editorconfig', + '.nomad', + '.bat', + '.cmd', + '.deb', + '.log', + '.rpm', + '.vbs', ]); function portableBasename(name: string): string { @@ -94,7 +188,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.css': 'text/css', '.csv': 'text/csv', '.doc': 'application/msword', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.docx': + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', '.gif': 'image/gif', '.gz': 'application/gzip', '.gzip': 'application/gzip', @@ -123,7 +218,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.pdf': 'application/pdf', '.png': 'image/png', '.ppt': 'application/vnd.ms-powerpoint', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.pptx': + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', '.py': 'text/x-python', '.rst': 'text/x-rst', '.rtf': 'application/rtf', @@ -143,7 +239,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.woff': 'font/woff', '.woff2': 'font/woff2', '.xls': 'application/vnd.ms-excel', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xlsx': + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.xml': 'application/xml', '.yaml': 'application/yaml', '.yml': 'application/yaml', @@ -180,6 +277,12 @@ export interface BridgeWorkspaceDescriptor { name?: string; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; + environment?: { + fingerprint: string; + repo?: string; + ref?: string; + actions: string[]; + }; } export interface BridgeWorkspaceToolCapabilities { @@ -291,7 +394,8 @@ interface WorkspaceEditFileRequestBase { expectedBaseSha256?: string; } -export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequestBase { +export interface WorkspaceSingleEditFileRequest + extends WorkspaceEditFileRequestBase { /** Legacy single-edit form. */ oldText: string; /** Legacy single-edit form. */ @@ -299,7 +403,8 @@ export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequest edits?: never; } -export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestBase { +export interface WorkspaceBatchEditFileRequest + extends WorkspaceEditFileRequestBase { /** Ordered exact replacements applied atomically as one file mutation. */ edits: WorkspaceTextEdit[]; oldText?: never; @@ -307,7 +412,8 @@ export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestB } export type WorkspaceEditFileRequest = - WorkspaceSingleEditFileRequest | WorkspaceBatchEditFileRequest; + | WorkspaceSingleEditFileRequest + | WorkspaceBatchEditFileRequest; export interface WorkspaceTextEdit { oldText: string; @@ -330,20 +436,23 @@ interface WorkspacePreviewEditRequestBase { path: string; } -export interface WorkspaceSinglePreviewEditRequest extends WorkspacePreviewEditRequestBase { +export interface WorkspaceSinglePreviewEditRequest + extends WorkspacePreviewEditRequestBase { oldText: string; newText: string; edits?: never; } -export interface WorkspaceBatchPreviewEditRequest extends WorkspacePreviewEditRequestBase { +export interface WorkspaceBatchPreviewEditRequest + extends WorkspacePreviewEditRequestBase { edits: WorkspaceTextEdit[]; oldText?: never; newText?: never; } export type WorkspacePreviewEditRequest = - WorkspaceSinglePreviewEditRequest | WorkspaceBatchPreviewEditRequest; + | WorkspaceSinglePreviewEditRequest + | WorkspaceBatchPreviewEditRequest; export interface WorkspacePreviewEditResult { protocolVersion: BridgeProtocolVersion; @@ -368,6 +477,7 @@ export interface WorkspaceExecuteCommandRequest { timeoutMs?: number; /** Aggregate UTF-8 stdout and stderr budget. */ maxOutputBytes?: number; + environmentAction?: { name: string; fingerprint: string }; } export interface WorkspaceExecuteCommandResult { @@ -452,6 +562,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ ]); const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ + 'environmentAction', 'protocolVersion', 'operation', 'workspaceId', @@ -720,7 +831,8 @@ export function isWorkspaceToolErrorCode( } export type BridgeSettlement = - BridgeFulfilledSettlement | BridgeRejectedSettlement; + | BridgeFulfilledSettlement + | BridgeRejectedSettlement; export interface BridgeSettlementResponse { protocolVersion: BridgeProtocolVersion; @@ -806,7 +918,8 @@ export function isBridgeWorkspaceProgrammaticRequest( (body.transfer_timeout_ms !== undefined && (!Number.isSafeInteger(body.transfer_timeout_ms) || Number(body.transfer_timeout_ms) < 1 || - Number(body.transfer_timeout_ms) > BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || + Number(body.transfer_timeout_ms) > + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || (body.run_timeout !== undefined && (!Number.isSafeInteger(body.run_timeout) || Number(body.run_timeout) < 1 || @@ -826,7 +939,8 @@ export function isBridgeWorkspaceProgrammaticRequest( if ( !isSafePortableRelativePath(file.name) || file.name === '.' || - portableBasename(file.name).toLowerCase() === '_ptc_pending_result.json' || + portableBasename(file.name).toLowerCase() === + '_ptc_pending_result.json' || normalizePortableRelativePath(file.name) !== file.name || names.has(file.name) ) { @@ -875,7 +989,9 @@ export function isBridgeWorkspaceProgrammaticRequest( const segments = name.split('/'); let ancestor = ''; for (let index = 0; index < segments.length - 1; index += 1) { - ancestor = ancestor ? `${ancestor}/${segments[index]}` : segments[index]!; + ancestor = ancestor + ? `${ancestor}/${segments[index]}` + : segments[index]!; if (names.has(ancestor)) return false; } } @@ -1105,6 +1221,22 @@ export function isWorkspaceToolRequest( } if (request.operation === 'execute_command') { return ( + (request.environmentAction === undefined || + (typeof request.environmentAction === 'object' && + request.environmentAction !== null && + Object.keys(request.environmentAction).length === 2 && + typeof (request.environmentAction as { name?: unknown }) + .name === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test( + (request.environmentAction as { name: string }).name, + ) && + typeof ( + request.environmentAction as { fingerprint?: unknown } + ).fingerprint === 'string' && + /^[a-f0-9]{64}$/.test( + (request.environmentAction as { fingerprint: string }) + .fingerprint, + ))) && hasOnlyKeys(request, WORKSPACE_COMMAND_REQUEST_KEYS) && typeof request.command === 'string' && request.command.trim().length > 0 && @@ -1438,11 +1570,17 @@ export function isValidBridgeWorkspaceToolCapabilities( const descriptor = workspace as Record; if ( Object.keys(descriptor).some( - key => key !== 'id' && key !== 'name' && key !== 'operations', + key => + key !== 'id' && + key !== 'name' && + key !== 'operations' && + key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.environment !== undefined && + !isValidCodeEnvironmentDescriptor(descriptor.environment)) || (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || @@ -1469,6 +1607,37 @@ export function isValidBridgeWorkspaceToolCapabilities( }); } +export function isValidCodeEnvironmentDescriptor( + value: unknown, +): value is NonNullable { + if (typeof value !== 'object' || value === null) return false; + const environment = value as Record; + return ( + Object.keys(environment).every(key => + ['fingerprint', 'repo', 'ref', 'actions'].includes(key), + ) && + typeof environment.fingerprint === 'string' && + /^[a-f0-9]{64}$/.test(environment.fingerprint) && + (environment.repo === undefined || + (typeof environment.repo === 'string' && + environment.repo.length <= 256 && + /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(environment.repo))) && + (environment.ref === undefined || + (typeof environment.ref === 'string' && + environment.ref.trim().length > 0 && + environment.ref.length <= 256 && + !/[\0\r\n]/.test(environment.ref))) && + Array.isArray(environment.actions) && + environment.actions.length <= 32 && + environment.actions.every( + name => + typeof name === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(name), + ) && + new Set(environment.actions).size === environment.actions.length + ); +} + export function isValidBridgeWorkerCapabilities( value: unknown, ): value is BridgeWorkerCapabilities { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index ffe1b350..86e85872 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -192,6 +192,13 @@ function workspaceCapabilitiesMatch( (workspace, index) => workspace.id === executor.workspaces[index]?.id && workspace.name === executor.workspaces[index]?.name && + workspace.environment?.fingerprint === executor.workspaces[index]?.environment?.fingerprint && + workspace.environment?.repo === executor.workspaces[index]?.environment?.repo && + workspace.environment?.ref === executor.workspaces[index]?.environment?.ref && + workspace.environment?.actions.length === executor.workspaces[index]?.environment?.actions.length && + (workspace.environment?.actions.every( + (action, actionIndex) => action === executor.workspaces[index]?.environment?.actions[actionIndex], + ) ?? executor.workspaces[index]?.environment == null) && workspace.operations?.length === executor.workspaces[index]?.operations?.length && (workspace.operations?.every( @@ -236,7 +243,9 @@ function registrationCompatibleCapabilities( return []; } const { operations: _operations, ...compatibleWorkspace } = workspace; - return [compatibleWorkspace]; + return [{ ...compatibleWorkspace, ...(workspace.environment ? { + environment: { ...workspace.environment, actions: [] }, + } : {}) }]; }); if (workspaces.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; @@ -312,13 +321,17 @@ function supportedWorkspaceCapabilities( editOperations.has(operation), ); const workspaces = desired.workspaces.flatMap((workspace) => { - if (workspace.operations == null) return [workspace]; - const workspaceOperations = workspace.operations.filter((operation) => + const workspaceOperations = (workspace.operations ?? operations).filter((operation) => operations.includes(operation), ); return workspaceOperations.length === 0 ? [] - : [{ ...workspace, operations: workspaceOperations }]; + : [{ ...workspace, + ...(workspace.operations ? { operations: workspaceOperations } : {}), + ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { + environment: { ...workspace.environment, actions: [] }, + } : {}), + }]; }); if (workspaces.length === 0) return undefined; const editFileFeatures = desired.editFileFeatures?.filter((feature) => diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index f6205cd4..d97735dc 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -7,6 +7,30 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('worker clears named actions when command execution is not negotiated', async () => { + const workspaceTools = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary', environment: { fingerprint: 'a'.repeat(64), actions: ['test'] } }], + }; + const registrations: Array = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], workspaceTools }, + workspaceMutationQuarantine: mutationQuarantine(), + workspaceTools: { capabilities: workspaceTools, async execute() { throw new Error('not executed'); } }, + fetchImpl: async (_input, init) => { + registrations.push(JSON.parse(String(init?.body)).capabilities.workspaceTools); + return Response.json({ protocolVersion: 1, workerId: 'vm-1', incarnationId, + registeredAt: new Date().toISOString(), leaseTtlMs: 60000, supportedWorkspaceToolOperations: ['read_file'] }); + }, + }); + await worker.register(); + assert.ok(registrations.length > 0); + for (const registration of registrations) assert.deepEqual(registration.workspaces[0].environment.actions, []); +}); + const listWorkspaceCapabilities = { protocolVersion: 1 as const, operations: [ @@ -2422,6 +2446,32 @@ test('worker refuses to advertise workspace tools without a matching executor', ); }); +test('worker refuses environment metadata that differs from its executor', () => { + const environment = { fingerprint: 'a'.repeat(64), repo: 'owner/repo', ref: 'main', actions: [] as string[] }; + const workspaceTools = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', environment }], + }; + for (const changed of [ + undefined, + { ...environment, fingerprint: 'b'.repeat(64) }, + { ...environment, repo: 'other/repo' }, + { ...environment, ref: 'other' }, + { ...environment, actions: ['test'] }, + ]) { + assert.throws(() => new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], workspaceTools }, + workspaceTools: { + capabilities: { ...workspaceTools, workspaces: [{ id: 'primary', ...(changed ? { environment: changed } : {}) }] }, + async execute() { throw new Error('not executed'); }, + }, + }), /workspace tool capabilities require a matching executor/i); + } +}); + test('worker requires durable quarantine before advertising command execution', () => { const workspaceCapabilities = { protocolVersion: 1 as const, diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index dfda49fc..91e89f54 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1693,6 +1693,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { if (request.operation !== 'execute_command') { return this.options.workspaceTools.execute(request, signal); } + if (request.environmentAction) { + throw new WorkspaceToolError('Environment action was not resolved by this worker', 'INVALID_REQUEST'); + } if (!this.commandWorkspaces.has(request.workspaceId)) { throw new WorkspaceToolError( 'Command execution is disabled for this workspace', From f2dcb93b78578fe0fe5eeb7cf1b6a965a5d036f1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 20:01:54 -0400 Subject: [PATCH 101/116] fix: Allow Trusted Own-Root Environment Symlinks (#212) * fix: Allow Trusted Own-Root Environment Symlinks * fix: Check Alias Parent Ownership by Filesystem Identity * fix: Enforce Parent Ownership Across Every Environment Path --- .github/workflows/ci.yml | 2 + packages/code/src/environment.test.ts | 62 +++++++++++++++++++++++++++ packages/code/src/environment.ts | 28 +++++++++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f041e2d6..bbf9e302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,6 +194,8 @@ jobs: node-version: 24.16.0 - run: npm ci - run: npm run build + - name: Native environment containment tests + run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js diff --git a/packages/code/src/environment.test.ts b/packages/code/src/environment.test.ts index 9080f6a2..a8b5b036 100644 --- a/packages/code/src/environment.test.ts +++ b/packages/code/src/environment.test.ts @@ -193,6 +193,68 @@ test('environment roots resolve relative to the definition and fingerprints cove ); }); +test('accepts an own root through trusted external symlinks without allowing other root identities', async t => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'code-env-own-alias-'))); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await mkdir(root); + const alias = join(directory, 'alias'); + await symlink(root, alias); + await symlink(alias, join(directory, 'nested-alias')); + const path = join(directory, 'environment.yaml'); + for (const selected of [alias, join(directory, 'nested-alias')]) { + await writeFile(path, `name: app\nroot: ${selected}\n`); + const loaded = await loadCodeEnvironment(path); + assert.equal(loaded.definition.root, root); + await assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'other', root }]), + /root traversal|mount alias/, + ); + } +}); + +test('rejects own-root links hidden by parent aliases or filesystem casing', async t => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'code-env-parent-alias-'))); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'Project'); + await mkdir(root); + await symlink(root, join(root, 'self')); + const outside = join(directory, 'outside'); + await mkdir(outside); + await symlink(root, join(outside, 'back')); + await symlink(outside, join(root, 'pivot')); + const alias = join(directory, 'parent-alias'); + await symlink(root, alias); + const path = join(directory, 'environment.yaml'); + const selectedRoots = [join(alias, 'self'), join(alias, 'pivot', 'back')]; + try { + if (await realpath(join(directory, 'project')) === root) { + selectedRoots.push(join(directory, 'project', 'self')); + selectedRoots.push(join(directory, 'project', 'pivot', 'back')); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + for (const selected of selectedRoots) { + await writeFile(path, `name: app\nroot: ${selected}\n`); + const loaded = await loadCodeEnvironment(path); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]), + /root traversal|mount alias/, + ); + } + const definition = join(outside, 'environment.yaml'); + await writeFile(definition, `name: app\nroot: ${root}\n`); + for (const selected of selectedRoots.filter(path => path.endsWith('/back'))) { + const loaded = await loadCodeEnvironment(selected.replace(/back$/, 'environment.yaml')); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]), + /outside|mount alias/, + ); + } +}); + test('rejects a trusted definition with an in-workspace hard link', async t => { const directory = await mkdtemp(join(tmpdir(), 'code-env-hardlink-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 700fc0dc..c2b2cf1f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -355,6 +355,25 @@ export async function assertEnvironmentDefinitionsOutsideRoots( } return result; }; + // The entry's parent, not its symlink target, determines who can replace it. + // Compare ancestor identities so casing and directory aliases cannot make a + // workspace-controlled entry look external on case-insensitive filesystems. + const canonicalParents = new Map>(); + const controlsEntry = async (component: string, rootIdentity: string): Promise => { + const directory = dirname(component); + let canonical = canonicalParents.get(directory); + if (!canonical) { + canonical = realpath(directory); + canonicalParents.set(directory, canonical); + } + let parent = await canonical; + while (true) { + if ((await identity(parent)) === rootIdentity) return true; + const next = dirname(parent); + if (next === parent) return false; + parent = next; + } + }; for (const environment of environments) { for (const root of roots) { const rootIdentity = await identity(root.root); @@ -362,10 +381,14 @@ export async function assertEnvironmentDefinitionsOutsideRoots( { for (const component of environment.rootPaths ?? []) { const path = relative(root.root, component); - if (path === '' && root.id === environment.definition.name) + const sameRoot = (await identity(component)) === rootIdentity; + const controlled = await controlsEntry(component, rootIdentity); + // A trusted external alias may select its own root, but a + // link beneath that root is still writable by the workspace. + if (sameRoot && !controlled && root.id === environment.definition.name) continue; if ( - (await identity(component)) === rootIdentity || + controlled || sameRoot || path === '' || (!isAbsolute(path) && path !== '..' && @@ -383,6 +406,7 @@ export async function assertEnvironmentDefinitionsOutsideRoots( ]) { const path = relative(root.root, controlPath); if ( + (await controlsEntry(controlPath, rootIdentity)) || (await identity(controlPath)) === rootIdentity || path === '' || (!isAbsolute(path) && From 840537b9d95a3ad6c761729a198c4e9a0d020d90 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 20:02:05 -0400 Subject: [PATCH 102/116] fix: Retain Quarantine After Failed Environment Setup (#213) * fix: Retain Quarantine After Failed Environment Setup * test: Run Native Environment Setup Lifecycle in CI * fix: Document and Verify Local Setup Quarantine Recovery --- .github/workflows/ci.yml | 4 ++ packages/code/README.md | 11 ++++-- packages/code/src/cli.ts | 4 +- packages/code/src/environment-live.test.ts | 43 ++++++++++++++++++---- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbf9e302..2bdeda21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,10 @@ jobs: run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js + - name: Native environment setup lifecycle tests + env: + LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' + run: node --test dist/environment-live.test.js lambda-microvm-provisioning: name: Lambda MicroVM Provisioning diff --git a/packages/code/README.md b/packages/code/README.md index c3bb16d3..8b98618f 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -689,9 +689,14 @@ worker runs. This inspection happens at startup, not on the command hot path. Setup is an operator-authorized startup command under the configured native sandbox policy. It requires commands to be enabled, runs once per worker startup before registration, and must be idempotent for restarts. Its timeout is bounded to five -minutes and captured output to 8 KiB. Setup failure prevents registration. A crash -or uncertain termination retains the existing workspace quarantine marker; inspect -the workspace before clearing quarantine. No setup output is sent to the model. +minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero +exit, timeout, crash or uncertain termination retains the workspace quarantine marker; +inspect the workspace before running `librechat-code clear-workspace-quarantine +--worker-dir --workspace-id ` with the same +deployment and identity configuration. Only use the separate +`--reset-workspace-quarantine ` run option afterward if a server +fence also needs clearing. Only successful setup automatically clears its marker. +No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge advertises only their names and the definition fingerprint, never their shell source diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 5498af63..00eca576 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1012,12 +1012,12 @@ async function run( }, controller.signal, ); - await guard.clear('setup'); if (result.exitCode !== 0 || result.timedOut) { throw new Error( - `Environment ${id} setup failed; inspect the setup command before restarting`, + `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`, ); } + await guard.clear('setup'); process.stdout.write( `librechat-code: environment ${id} prepared\n`, ); diff --git a/packages/code/src/environment-live.test.ts b/packages/code/src/environment-live.test.ts index bb84c8b5..9d442982 100644 --- a/packages/code/src/environment-live.test.ts +++ b/packages/code/src/environment-live.test.ts @@ -8,13 +8,14 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; -for (const { succeeds, reset } of [ - { succeeds: true, reset: false }, - { succeeds: false, reset: false }, - { succeeds: true, reset: true }, +for (const { succeeds, reset, timesOut } of [ + { succeeds: true, reset: false, timesOut: false }, + { succeeds: false, reset: false, timesOut: false }, + { succeeds: false, reset: false, timesOut: true }, + { succeeds: true, reset: true, timesOut: false }, ]) { test( - `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`, + `real CLI environment setup gates registration (success=${succeeds}, reset=${reset}, timeout=${timesOut})`, { skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', timeout: 20_000, @@ -27,7 +28,7 @@ for (const { succeeds, reset } of [ const path = join(directory, 'environment.yaml'); await writeFile( path, - `name: project\nroot: project\nsetup:\n command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n timeoutMs: 5000\n`, + `name: project\nroot: project\nsetup:\n command: 'printf prepared >> prepared.txt; ${timesOut ? 'sleep 10' : `exit ${succeeds ? 0 : 2}`}'\n timeoutMs: ${timesOut ? 1000 : 5000}\n`, ); let registrations = 0; let receive: (() => void) | undefined; @@ -38,6 +39,7 @@ for (const { succeeds, reset } of [ request.resume(); if (request.url?.endsWith('/register')) { registrations++; + await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' }); if (reset) await assert.rejects( readFile(join(root, 'prepared.txt')), @@ -61,10 +63,15 @@ for (const { succeeds, reset } of [ }); const address = server.address(); assert.ok(address && typeof address !== 'string'); - const child = spawn( + const start = (clear = false) => spawn( process.execPath, [ fileURLToPath(new URL('./cli.js', import.meta.url)), + ...(clear ? [ + 'clear-workspace-quarantine', + '--worker-dir', root, + '--workspace-id', 'project', + ] : [ 'run', '--environment', path, @@ -72,6 +79,7 @@ for (const { succeeds, reset } of [ ...(reset ? ['--reset-workspace-quarantine', 'project'] : []), + ]), ], { env: { @@ -90,6 +98,7 @@ for (const { succeeds, reset } of [ stdio: ['ignore', 'pipe', 'pipe'], }, ); + const child = start(); const exited = once(child, 'exit'); t.after(() => child.kill('SIGKILL')); let stderr = ''; @@ -111,6 +120,26 @@ for (const { succeeds, reset } of [ assert.notEqual(code, 0); assert.match(stderr, /Environment project setup failed/); assert.equal(registrations, 0); + const marker = await readFile(join(directory, 'quarantine.json'), 'utf8'); + assert.equal(JSON.parse(marker).workspaceId, 'project'); + const before = await readFile(join(root, 'prepared.txt'), 'utf8'); + const retry = start(); + t.after(() => retry.kill('SIGKILL')); + let retryStderr = ''; + retry.stderr.on('data', chunk => { retryStderr += chunk.toString(); }); + const [retryCode] = await once(retry, 'exit'); + assert.notEqual(retryCode, 0); + assert.match(retryStderr, /quarantined/); + assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before); + assert.equal(await readFile(join(directory, 'quarantine.json'), 'utf8'), marker); + assert.equal(registrations, 0); + const recovery = start(true); + t.after(() => recovery.kill('SIGKILL')); + const [recoveryCode] = await once(recovery, 'exit'); + assert.equal(recoveryCode, 0); + await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' }); + assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before); + assert.equal(registrations, 0); } }, ); From 9a3f5dbd6fedf42462239d135c2c25ec8e737628 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 15 Sep 2026 14:25:50 -0400 Subject: [PATCH 103/116] fix: Allow Lambda MicroVM metadata in hardened mode (#215) --- api/src/secure-startup.test.ts | 12 ++++++++++++ api/src/secure-startup.ts | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/api/src/secure-startup.test.ts b/api/src/secure-startup.test.ts index c1558e85..e1291712 100644 --- a/api/src/secure-startup.test.ts +++ b/api/src/secure-startup.test.ts @@ -113,6 +113,18 @@ describe('hardened sandbox-runner startup config', () => { expect(() => validateHardenedSandboxStartup()).toThrow('REDIS_HOST'); }); + test('allows Lambda MicroVM image metadata while rejecting AWS credentials', () => { + setValidHardenedConfig(); + process.env.AWS_LAMBDA_MICROVM_IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:codeapi'; + process.env.AWS_LAMBDA_MICROVM_IMAGE_NAME = 'codeapi'; + process.env.AWS_LAMBDA_MICROVM_IMAGE_VERSION = '3'; + process.env.AWS_REGION = 'us-east-1'; + expect(() => validateHardenedSandboxStartup()).not.toThrow(); + + process.env.AWS_ACCESS_KEY_ID = 'access-key'; + expect(() => validateHardenedSandboxStartup()).toThrow('AWS_ACCESS_KEY_ID'); + }); + test('rejects missing manifest verifier and wrong forwarding target', () => { setValidHardenedConfig(); config.egress_gateway_url = ''; diff --git a/api/src/secure-startup.ts b/api/src/secure-startup.ts index 04f5638a..326bf810 100644 --- a/api/src/secure-startup.ts +++ b/api/src/secure-startup.ts @@ -1,6 +1,13 @@ import { config } from './config'; import { workspaceIsolationConfigErrors } from './workspace-isolation'; +const ALLOWED_LAMBDA_MICROVM_AWS_ENV = new Set([ + 'AWS_LAMBDA_MICROVM_IMAGE_ARN', + 'AWS_LAMBDA_MICROVM_IMAGE_NAME', + 'AWS_LAMBDA_MICROVM_IMAGE_VERSION', + 'AWS_REGION', +]); + export class SandboxSecureStartupError extends Error { constructor(message: string) { super(message); @@ -57,7 +64,7 @@ function forbiddenEnvNames(): string[] { if (name === 'CODEAPI_HARDENED_SANDBOX_MODE') continue; if (name.startsWith('CODEAPI_')) forbidden.push(name); if (name.startsWith('REDIS_')) forbidden.push(name); - if (name.startsWith('AWS_')) forbidden.push(name); + if (name.startsWith('AWS_') && !ALLOWED_LAMBDA_MICROVM_AWS_ENV.has(name)) forbidden.push(name); if (name.startsWith('S3_')) forbidden.push(name); if (name.startsWith('MINIO_')) forbidden.push(name); if (/(SECRET|TOKEN|PASSWORD|PRIVATE_KEY)/.test(name)) forbidden.push(name); From c03d309429f152ae11a6d9503d3804dbaab9d228 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 00:38:02 -0400 Subject: [PATCH 104/116] docs: add self-hosted worker setup runbook (#219) --- docs/remote-bridge/README.md | 5 + docs/remote-bridge/worker-runbook.md | 527 +++++++++++++++++++++++++++ packages/code/README.md | 3 + 3 files changed, 535 insertions(+) create mode 100644 docs/remote-bridge/worker-runbook.md diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index b9de3ac2..f66499b2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -3,6 +3,10 @@ Remote Code Bridge makes an operator-owned VM a stateful Code API execution environment without exposing that VM to inbound internet traffic. +For an end-to-end host setup, including pairing, named environments, systemd, +launchd, GitHub App credentials, upgrades, verification, and recovery, see the +[self-hosted worker runbook](./worker-runbook.md). + ```text LibreChat -> Code API -> Redis assignment ^ | @@ -200,6 +204,7 @@ Expose the Code API deployment as an environment under the Agents endpoint: endpoints: agents: statefulCodeSessions: + allowedEnvironments: [user, agent-user, conversation] environments: - id: my-vm name: My VM diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md new file mode 100644 index 00000000..7511a265 --- /dev/null +++ b/docs/remote-bridge/worker-runbook.md @@ -0,0 +1,527 @@ +# Self-hosted worker runbook + +This runbook attaches an operator-controlled laptop or VM to a LibreChat Code +API deployment. The worker makes an outbound HTTPS connection; it does not +open an inbound port. The same procedure works for one machine or many +principal-bound machines. + +The guide uses a named environment and the native Sandbox Runtime (SRT). It +covers a restricted personal-machine deployment and the `trusted-vm` preset, +where a separate VM boundary is responsible for most host isolation. + +## 1. Understand the boundaries + +Four independently managed components participate: + +1. LibreChat stores the environment record, resolves its principal, applies + administrator/user policy, and selects the worker for a conversation. +2. Code API authenticates the selection, queues and fences assignments, and + exposes the outbound bridge. +3. `@librechat/code` runs on the attached machine, owns local workspace + admission, rotates its bridge credential, and executes tools through SRT. +4. The machine owner controls the OS, workspace, network, credentials, and + service lifecycle. + +Pairing authenticates a worker. It does not make the host trustworthy, attest +the host policy, or replace tool approval. A `trusted-vm` worker is appropriate +only when the VM boundary is already operated as the security boundary. + +## 2. Configure LibreChat and Code API + +Deploy Code API's remote-bridge profile before pairing a machine. At minimum, +use paired authentication and dynamic routing so one bridge can serve many +principal-bound workers: + +```dotenv +CODEAPI_SANDBOX_BACKEND=remote-bridge +CODEAPI_EXECUTION_PROFILE=stateful +CODEAPI_RUNTIME_SESSION_MODE=affinity +CODEAPI_BRIDGE_AUTH_MODE=paired +CODEAPI_BRIDGE_DYNAMIC_WORKERS=true +CODEAPI_BRIDGE_TOKEN= +``` + +The administrator token belongs only on the Code API/control-plane host. Never +put it on an attached machine. Configure Redis and the remaining Code API +settings as described in the [Remote Code Bridge guide](./README.md). + +Expose that Code API endpoint to LibreChat and explicitly choose which +state-sharing scopes the deployment permits: + +```yaml +endpoints: + agents: + capabilities: + [ + deferred_tools, + execute_code, + file_search, + web_search, + artifacts, + subagents, + actions, + context, + skills, + memory, + ask_user_question, + tools, + chain, + ocr, + stateful_code_sessions, + ] + statefulCodeSessions: + allowedEnvironments: [user, agent-user, conversation] + environments: + - id: attached-workers + name: Attached machines + type: attached + baseURL: https://code.example.com/v1 + default: true +``` + +The example preserves LibreChat's default capabilities and adds the opt-in +`stateful_code_sessions` capability. Adjust the list to the deployment's +policy. Exactly one configured Code environment must be the default. The three +sharing scopes mean: + +- `user`: reuse an environment for the signed-in user; +- `agent-user`: reuse it for one agent and user; and +- `conversation`: isolate reuse to one conversation. + +These scopes determine session reuse. They do not weaken a worker's filesystem +root or share one user's principal-bound machine with another user. + +Enable stateful code sessions on the intended agent and select the attached +environment. Start with file writes and command execution set to `ask`; expose +`allow` or `deny` only when the deployment and machine policy permit them. +Worker capabilities are a ceiling: a conversation setting cannot enable a +command or write that the worker did not advertise. + +## 3. Roll out compatible consumers first + +Before enabling `--environment` on a worker: + +1. Deploy a LibreChat version that accepts named environment descriptors. +2. Deploy the matching Code API API and queue-worker processes. +3. Update `@librechat/code` on the attached machine. +4. Only then restart the worker with `--environment`. + +An old worker remains compatible with new consumers until the opt-in flag is +used. An old strict consumer can reject a new worker's environment metadata. +During a rolling deployment, update every API/queue replica before changing +workers. + +Record the exact source commit or package version at every tier. Do not infer a +worker's version from the Code API server: the worker is a separate process on +a separate machine. + +## 4. Prepare the worker host + +Install: + +- Node.js 20.11 or newer (Node.js 24 is supported); +- Git; +- `bubblewrap`, `socat`, and `ripgrep` on Linux; +- Bash 5.2 or newer and `jq` when Bash Programmatic Tool Calling is enabled; + and +- GitHub CLI and Git LFS only when the workflows need them. + +For example, install the system dependencies on Ubuntu with: + +```bash +sudo apt-get update +sudo apt-get install -y bash bubblewrap git jq ripgrep socat +``` + +On macOS, install the optional PTC and GitHub tools with: + +```bash +brew install bash gh git-lfs jq ripgrep +``` + +Install Node.js through the host's managed package source or version manager. +Bun is not required by `@librechat/code`. + +Keep source, application state, environment definitions, and credentials in +separate paths. For example: + +```text +/opt/librechat-code/releases// pinned worker source/build +/srv/code-workspaces/ coding roots +/etc/librechat-code/environments/ operator-owned YAML definitions +~/.config/librechat/code/ paired identity +~/.config/librechat-code/github-app.pem optional GitHub App key +``` + +Every ancestor of a definition, identity, key, quarantine file, or workspace +root must be owned by the worker account or root. It must not be writable by +group or other users. Sticky shared directories such as `/tmp` are handled +separately, but should not hold durable configuration. + +The workspace remains writable by its owner. For a dedicated service account, +a typical root is: + +```bash +sudo install -d -o librechat-code -g librechat-code -m 0750 /srv/code-workspaces +``` + +Do not register a home directory or another root containing credentials, +shell history, SSH keys, or unrelated projects. + +## 5. Install a pinned worker + +Use a published version when available. To install from source, keep a pinned +checkout and build only the worker package: + +```bash +git clone https://github.com/LibreChat-AI/code-interpreter.git /opt/librechat-code/source +cd /opt/librechat-code/source +git fetch origin main +git checkout --detach +npm ci --prefix packages/code +npm run build --prefix packages/code +cd packages/code +sudo npm link +``` + +Confirm that `/usr/local/bin/librechat-code` resolves to the intended build. +Do not replace a running release until the new build and its native imports +have succeeded. Keeping releases in commit-named directories makes rollback a +service-path change instead of a rebuild. + +## 6. Pair the machine + +Create a pairing in LibreChat's Code environments UI when available. The +pairing must be bound to the intended deployment, tenant, user, role, or group. +The code is single-use and expires after ten minutes. + +Redeem it on the worker machine: + +```bash +librechat-code pair https://code.example.com/v1 '' \ + --worker-id code-example123 +``` + +Run pairing as the same operating-system account that will run the worker. If +the systemd service uses `User=librechat-code`, run the command as that account +or supply an explicit identity path the account can read and replace. + +The CLI generates the Ed25519 private key locally and saves the identity under +`~/.config/librechat/code/` with owner-only permissions. Do not transmit or +copy that file through chat. The bridge credential expires after fifteen +minutes, but a running worker rotates it automatically. A normal restart does +not require re-pairing. + +For a custom location, use `--identity` during pairing and set +`LIBRECHAT_CODE_IDENTITY_FILE` in the service. Keep the worker ID stable: agent +defaults and conversations refer to the environment record associated with +that identity. + +## 7. Define named environments + +Store definitions outside every workspace root. A broad, multi-project VM can +preserve an existing `primary` binding without pretending the root is one Git +repository: + +```yaml +# /etc/librechat-code/environments/primary.yaml +name: primary +root: /srv/code-workspaces +``` + +For a single project, descriptive repository metadata and fixed actions may be +useful: + +```yaml +name: app-dev +root: /srv/code-workspaces/app +repo: example/app +ref: main +setup: + command: npm ci + timeoutMs: 300000 +actions: + - name: typecheck + command: npm run typecheck + timeoutMs: 120000 + - name: test + command: npm test + timeoutMs: 300000 +``` + +Important semantics: + +- `name` is both the workspace ID and its current display name. Preserve an + existing ID such as `primary` to preserve agent/conversation bindings. +- `repo` and `ref` are labels. They do not clone, fetch, or check out anything. +- `root` must already exist. Relative roots resolve from the definition file. +- Setup runs before registration on every worker start. It must be idempotent. +- A setup failure or timeout prevents registration and leaves a durable + quarantine marker for operator inspection. +- Actions are fixed operator commands. The model selects only the action name + and fingerprint; it cannot inject arguments, a command, or a working + directory. +- Actions still pass through LibreChat approval and worker command policy. +- Up to 32 roots may be declared, and they must not overlap. A broad parent + environment cannot coexist with child project environments. + +Definitions contain policy rather than secrets. A root-owned file may be +readable by the service account, but must not be group/other writable. For +example: + +```bash +sudo install -d -o root -g librechat-code -m 0750 /etc/librechat-code/environments +sudo install -o root -g librechat-code -m 0640 primary.yaml \ + /etc/librechat-code/environments/primary.yaml +``` + +Do not combine `--environment` with `--worker-dir`, `--default-workspace`, +`--workspace`, `--workspace-id`, or `--workspace-name`. Remove the equivalent +`LIBRECHAT_CODE_WORKER_DIR`, `LIBRECHAT_CODE_WORKSPACE_ID`, and +`LIBRECHAT_CODE_WORKSPACE_NAME` settings too. + +## 8. Choose a command policy + +For a personal machine, use the default `restricted` policy and explicitly +allow only required network destinations. + +For a separately secured VM whose outer boundary is managed by the operator, +the worker may use: + +```text +--allow-workspace-writes +--allow-workspace-commands +--command-policy-preset trusted-vm +``` + +`trusted-vm` is a policy preset, not an unsandboxed execution mode. SRT still +protects the bridge identity, GitHub credentials, worker configuration, and +control sockets. The preset deliberately permits broader workspace and network +behavior because the VM owner accepts responsibility for the host boundary. + +LibreChat's tool approval remains independent. Enabling commands on a worker +does not authorize a user or agent to bypass `ask` or `deny` policy. + +## 9. Optionally configure a GitHub App + +Prefer a GitHub App over a personal token. Install it only on repositories the +agent may use and grant the minimum permissions its workflows require. Git +clone/fetch/push generally needs repository Contents access; API-based pull +request workflows also need Pull requests access. + +Store the downloaded private key outside every workspace. Unlike an +environment definition, the key must have no group or other access and must be +readable by the service account: + +```bash +install -d -m 0700 ~/.config/librechat-code +install -m 0600 app.private-key.pem ~/.config/librechat-code/github-app.pem +``` + +Configure the worker, preferably in a separate service drop-in: + +```ini +[Service] +Environment=LIBRECHAT_CODE_GITHUB_APP_ID=12345 +Environment=LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 +Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/librechat-code/github-app.pem +``` + +The trusted worker mints short-lived installation tokens. Sandboxed commands +receive masked Git/`gh` credentials only for the configured GitHub hosts; the +token is not written to the repository, remote URL, or Git configuration. + +## 10. Run under systemd + +Use a dedicated service account in a multi-user deployment. This example keeps +the paired identity in its default location: + +```ini +# /etc/systemd/system/librechat-code.service +[Unit] +Description=LibreChat attached code worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=librechat-code +Group=librechat-code +WorkingDirectory=/srv/code-workspaces +Environment=HOME=/home/librechat-code +Environment=NODE_ENV=production +Environment=LIBRECHAT_CODE_WORKER_ID=code-example123 +ExecStart=/usr/local/bin/librechat-code run \ + --environment /etc/librechat-code/environments/primary.yaml \ + --allow-workspace-writes \ + --allow-workspace-commands +Restart=always +RestartSec=5s +TimeoutStopSec=35s +KillMode=control-group +UMask=0077 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +For a trusted VM, append `--command-policy-preset trusted-vm` to `ExecStart`. +After installing or changing a unit or drop-in, reload it before restart: + +```bash +sudo systemd-analyze verify librechat-code.service +sudo systemctl daemon-reload +sudo systemctl enable --now librechat-code.service +``` + +`systemctl restart` alone does not load a changed unit definition. + +## 11. Run under launchd on macOS + +Use absolute executable and release paths in the property list. Keep the paired +identity in the logged-in user's private configuration directory: + +```xml +ProgramArguments + + /absolute/path/to/node + /opt/librechat-code/releases/COMMIT/packages/code/dist/cli.js + run + --environment + /Users/worker/.config/librechat/code/environments/primary.yaml + --allow-workspace-writes + --allow-workspace-commands + +EnvironmentVariables + + LIBRECHAT_CODE_WORKER_ID + code-example123 + LIBRECHAT_CODE_IDENTITY_FILE + /Users/worker/.config/librechat/code/code-example123.json + +``` + +Editing the plist does not update launchd's cached job. Reload it: + +```bash +launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/ai.librechat.code.plist +launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/ai.librechat.code.plist +``` + +`launchctl kickstart -k` restarts the already-loaded definition and therefore +continues using stale paths after a plist edit. + +## 12. Verify the complete path + +Do not stop at “the process is running.” Check: + +1. The service command points to the intended version and environment file. +2. The native executor child started. +3. The worker has an established outbound HTTPS connection to Code API. +4. Code API reports the worker `online: true` and `ready: true` with the expected + workspace IDs and operations. +5. LibreChat lists the environment for the expected principal. +6. A disposable chat can select the workspace, read a file, perform an approved + write, execute a command, and retain state on the next turn. +7. Stop/cancellation prevents a delayed mutation. +8. A denied action remains denied even when the worker uses `trusted-vm`. + +Useful host checks: + +```bash +systemctl show librechat-code.service -p ExecStart -p MainPID -p NRestarts +journalctl -u librechat-code.service --since '10 minutes ago' +ss -tpn | grep librechat-code +``` + +The Code API status endpoint requires its administrator credential. Filter the +response before sharing it; do not expose tokens, pairings, bindings, or host +paths in logs or chat. + +## 13. Upgrade and roll back + +For each upgrade: + +1. Read the release notes and confirm whether LibreChat/Code API consumers must + land first. +2. Stage and build the new worker beside the current release. +3. Run focused package/native checks. +4. Update the service path or pinned checkout. +5. Reload the service manager definition when it changed. +6. Restart once and verify the complete path above. +7. Retain the previous release until the worker has completed real work. + +For source-linked installations, verify both `git rev-parse HEAD` and the +actual executable target. Updating a Code API checkout on another host does not +update this worker. + +Rollback by restoring the previous executable/service path and restarting. Do +not roll a new-metadata worker back behind the minimum consumer version while +it still advertises named environments. + +## 14. Recover safely + +### Expired bridge credential + +A running worker refreshes its short-lived credential automatically. If a +machine is offline long enough that refresh can no longer authenticate, issue +a fresh one-time pairing for the same worker ID and redeem it with a newly +generated keypair. Reusing the worker ID preserves the LibreChat environment +record and its agent assignments; creating a new ID creates a new environment. + +### Failed environment setup or uncertain mutation + +Inspect or restore the affected workspace first. Then, with the normal worker +stopped, clear the local quarantine using the same identity/deployment context: + +```bash +librechat-code clear-workspace-quarantine \ + --worker-dir /srv/code-workspaces/app \ + --workspace-id app-dev +``` + +If Code API also retains a server-side workspace fence, run the normal worker +configuration once with `--reset-workspace-quarantine app-dev`, wait for it to +exit successfully, and then start the normal service. The reset flag does not +replace the local clear command. + +Never clear quarantine merely to make the worker start. It represents a setup, +command, cancellation, or settlement whose effects may be incomplete. + +## 15. Common failures + +- **`--environment cannot be combined...`:** remove old workspace flags and + equivalent environment variables. +- **Definition or root rejected as replaceable:** remove group/other write + permission from every path ancestor; keep owner write. +- **Worker starts but old command/path remains:** run + `systemctl daemon-reload`, or fully boot out/bootstrap a changed launchd + plist. +- **Worker online but not ready:** check native sandbox preparation, definition + validation, setup, quarantine, and readiness logs. +- **Setup repeats on restart:** setup is intentionally per-start; make it + idempotent or remove it. +- **Git works on the host but not in tools:** verify the App installation, + permissions, private-key mode/owner, and allowed GitHub domains. +- **Repository label is present but files are absent:** `repo`/`ref` are + metadata; clone or mount the repository yourself. +- **Existing chats lose their workspace:** preserve the original workspace ID + in `name`, commonly `primary`. +- **Multiple project roots are rejected:** roots cannot overlap; remove the + broad parent or keep it as the only environment. + +## Final checklist + +- [ ] LibreChat and every Code API replica support the worker protocol. +- [ ] Worker version/source commit is recorded. +- [ ] Pairing is principal-bound and the identity file is private. +- [ ] Definitions are outside roots and immutable to sandboxed tools. +- [ ] Workspace ancestors are not group/other writable. +- [ ] GitHub App is optional, least-privilege, and installed only where needed. +- [ ] Approval policy remains enforced independently of worker capability. +- [ ] Service manager uses the intended executable and configuration. +- [ ] Worker is online, ready, and advertises the expected workspace. +- [ ] Read, approved mutation, command, persistence, denial, and cancellation + are tested. +- [ ] Upgrade and quarantine-recovery procedures are recorded for the operator. diff --git a/packages/code/README.md b/packages/code/README.md index 8b98618f..3cc61677 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,6 +3,9 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. +For a complete machine setup and operations guide, see the +[self-hosted worker runbook](../../docs/remote-bridge/worker-runbook.md). + The CLI owns the runtime-supervisor seam. Native workspace commands use Anthropic's open-source Sandbox Runtime (SRT) on the worker machine. The bundled endpoint adapter can also connect to an already-running loopback Code From 6dbf03ba13db9e0c894d06b73412a43cfd39cb0f Mon Sep 17 00:00:00 2001 From: Jackson Riding <99007683+jacksonriding@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:09:43 +1000 Subject: [PATCH 105/116] fix: bound memory buffering for streamed file uploads (#218) --- service/src/file-server.ts | 67 +------------------- service/src/minio-client.test.ts | 101 +++++++++++++++++++++++++++++++ service/src/minio-client.ts | 69 +++++++++++++++++++++ 3 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 service/src/minio-client.test.ts create mode 100644 service/src/minio-client.ts diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 22302042..00b91e4a 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -11,11 +11,11 @@ import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; -import { Client } from 'minio'; +import { createMinioClient } from './minio-client'; import { nanoid } from 'nanoid'; import { PassThrough } from 'stream'; import { pipeline } from 'stream/promises'; -import type { BucketItem, BucketItemStat, ClientOptions } from 'minio'; +import type { BucketItem, BucketItemStat, Client } from 'minio'; import type { Readable } from 'stream'; import type * as tls from 'tls'; import type * as t from './types'; @@ -40,69 +40,6 @@ app.use(httpMetricsMiddleware); const bucketName = process.env.MINIO_BUCKET ?? 'test-bucket'; -type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; - -async function createMinioClient(): Promise { - const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; - const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); - const useIrsa = irsaExplicit || irsaEnvVars; - - const baseConfig: ClientOptions = { - endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', - port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), - useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', - region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', - }; - - if (useIrsa) { - logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { - tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, - roleArn: process.env.AWS_ROLE_ARN, - region: baseConfig.region, - }); - - /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module - * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) - */ - let IamAwsProviderClass: new (opts: object) => unknown; - try { - const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (primaryError) { - try { - // Fallback for bun: resolve path using require if available (CJS context) - let resolvePath = 'node_modules/minio/'; - try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); - } catch { - // require.resolve not available (ESM context), use default path - } - const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (fallbackError) { - logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); - throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); - } - } - - const credentialsProvider = new IamAwsProviderClass({}); - - return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], - }); - } - - logger.info('Using explicit credentials for MinIO/S3 authentication'); - return new Client({ - ...baseConfig, - accessKey: process.env.MINIO_ACCESS_KEY ?? '', - secretKey: process.env.MINIO_SECRET_KEY ?? '', - sessionToken: process.env.MINIO_SESSION_TOKEN, - }); -} - let minioClient: Client; let storageInitialized = false; diff --git a/service/src/minio-client.test.ts b/service/src/minio-client.test.ts new file mode 100644 index 00000000..13259d03 --- /dev/null +++ b/service/src/minio-client.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { createMinioClient } from './minio-client'; + +const MiB = 1024 * 1024; + +// Exercise the real SDK against a local S3 HTTP fixture. No storage account, +// Redis, or file-server listener is needed to test the production client. +test.each([1024, 8 * MiB, 20 * MiB + 17])( + 'unknown-length upload of %i bytes uses bounded parts without losing bytes', + async size => { + const parts = new Map(); + const lengths: number[] = []; + const uploaded: { body?: Buffer; contentType?: string | null; originalFilename?: string | null } = {}; + const xml = (body: string) => new Response(body, { + headers: { 'Content-Type': 'application/xml' }, + }); + const server = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + async fetch(req) { + const url = new URL(req.url); + if (req.method === 'GET' && url.searchParams.has('uploads')) { + return xml('false'); + } + if (req.method === 'POST' && url.searchParams.has('uploads')) { + uploaded.contentType = req.headers.get('content-type'); + uploaded.originalFilename = req.headers.get('x-amz-meta-original-filename'); + return xml('test-upload'); + } + if (req.method === 'PUT' && url.searchParams.has('partNumber')) { + const body = Buffer.from(await req.arrayBuffer()); + lengths.push(body.length); + if (Number(req.headers.get('content-length')) !== body.length || + req.headers.get('content-md5') !== createHash('md5').update(body).digest('base64')) { + return new Response('Invalid part length or checksum', { status: 400 }); + } + parts.set(Number(url.searchParams.get('partNumber')), body); + return new Response(null, { + headers: { ETag: `"${createHash('md5').update(body).digest('hex')}"` }, + }); + } + if (req.method === 'POST' && url.searchParams.has('uploadId')) { + const manifest = await req.text(); + const ordered = [...manifest.matchAll(/(\d+)<\/PartNumber>/g)] + .map(match => parts.get(Number(match[1]))); + if (ordered.length !== parts.size || ordered.some(part => !part)) { + return new Response('Invalid multipart completion', { status: 400 }); + } + uploaded.body = Buffer.concat(ordered as Buffer[]); + return xml('http://localhost/test-bucket/input.bintest-bucketinput.bin"complete"'); + } + return new Response('Unexpected S3 request', { status: 400 }); + }, + }); + const settings: Record = { + MINIO_ENDPOINT: '127.0.0.1', + MINIO_PORT: String(server.port), + MINIO_NO_PORT: 'false', + MINIO_USE_SSL: 'false', + MINIO_REGION: 'us-east-1', + MINIO_USE_IRSA: 'false', + AWS_WEB_IDENTITY_TOKEN_FILE: undefined, + AWS_ROLE_ARN: undefined, + MINIO_ACCESS_KEY: 'test-access', + MINIO_SECRET_KEY: 'test-secret', + MINIO_SESSION_TOKEN: undefined, + }; + const saved = Object.fromEntries(Object.keys(settings).map(key => [key, process.env[key]])); + try { + for (const [key, value] of Object.entries(settings)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + const client = await createMinioClient(); + const expected = Buffer.alloc(size); + for (let i = 0; i < expected.length; i++) expected[i] = i % 251; + function* chunks() { + for (let offset = 0; offset < size; offset += 64 * 1024) { + yield expected.subarray(offset, Math.min(size, offset + 64 * 1024)); + } + } + await client.putObject('test-bucket', 'input.bin', Readable.from(chunks()), undefined, { + 'Content-Type': 'application/octet-stream', + 'X-Amz-Meta-Original-Filename': 'input.bin', + }); + expect(lengths.length).toBe(Math.ceil(size / (8 * MiB))); + expect(lengths.every(length => length <= 8 * MiB)).toBe(true); + expect(uploaded.body?.equals(expected)).toBe(true); + expect(uploaded.contentType).toBe('application/octet-stream'); + expect(uploaded.originalFilename).toBe('input.bin'); + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + await server.stop(true); + } + }, +); diff --git a/service/src/minio-client.ts b/service/src/minio-client.ts new file mode 100644 index 00000000..07af2891 --- /dev/null +++ b/service/src/minio-client.ts @@ -0,0 +1,69 @@ +import { Client, type ClientOptions } from 'minio'; +import logger from './fileServerLogger'; + +type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; + +export async function createMinioClient(): Promise { + const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; + const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); + const useIrsa = irsaExplicit || irsaEnvVars; + + const baseConfig: ClientOptions = { + // Unknown-length streams otherwise grow SDK parts to 528 MiB (the 5 TiB + // object limit / 10,000 parts). Bound each multipart buffer instead. + partSize: 8 * 1024 * 1024, + endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', + port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), + useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', + region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', + }; + + if (useIrsa) { + logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { + tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, + roleArn: process.env.AWS_ROLE_ARN, + region: baseConfig.region, + }); + + /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module + * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) + */ + let IamAwsProviderClass: new (opts: object) => unknown; + try { + const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (primaryError) { + try { + // Fallback for bun: resolve path using require if available (CJS context) + let resolvePath = 'node_modules/minio/'; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); + } catch { + // require.resolve not available (ESM context), use default path + } + const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (fallbackError) { + logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); + throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); + } + } + + const credentialsProvider = new IamAwsProviderClass({}); + + return new Client({ + ...baseConfig, + credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], + }); + } + + logger.info('Using explicit credentials for MinIO/S3 authentication'); + return new Client({ + ...baseConfig, + accessKey: process.env.MINIO_ACCESS_KEY ?? '', + secretKey: process.env.MINIO_SECRET_KEY ?? '', + sessionToken: process.env.MINIO_SESSION_TOKEN, + }); +} + From 3a3003a4ca4e31b9c120f3bd55238a4a74a8aabd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 01:26:03 -0400 Subject: [PATCH 106/116] ci: Automate Auditable Main Releases (#216) * fix: decouple repository release versions * ci: automate releases after successful main builds --- .github/scripts/next-release-version.sh | 65 +++++++++++++++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 100 +++++++++++++++++++----- CONTRIBUTING.md | 10 ++- README.md | 4 +- docs/RELEASING.md | 66 ++++++++++------ tests/release-versioning.sh | 56 +++++++++++++ 7 files changed, 255 insertions(+), 49 deletions(-) create mode 100755 .github/scripts/next-release-version.sh create mode 100755 tests/release-versioning.sh diff --git a/.github/scripts/next-release-version.sh b/.github/scripts/next-release-version.sh new file mode 100755 index 00000000..a5766479 --- /dev/null +++ b/.github/scripts/next-release-version.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +CURRENT_TAG="$1" +REVISION_RANGE="$2" + +if [[ ! "$CURRENT_TAG" =~ ^v([0-9]+)[.]([0-9]+)[.]([0-9]+)$ ]]; then + echo "current release must be a stable vMAJOR.MINOR.PATCH tag (got '$CURRENT_TAG')" >&2 + exit 2 +fi + +# Documentation, workflow, and test-only changes remain auditable in git but do +# not produce a deployable release. Any unrecognised path is treated as +# deployable so a newly added runtime component cannot silently miss a release. +DEPLOYABLE=false +while IFS= read -r path; do + case "$path" in + .github/*|docs/*|tests/*|*.md|*/README|*/README.*|*.test.*|*.spec.*) + ;; + *) + DEPLOYABLE=true + break + ;; + esac +done < <(git diff --name-only "$REVISION_RANGE") + +if [ "$DEPLOYABLE" = "false" ]; then + exit 0 +fi + +COMMIT_MESSAGES="$(git log --format='%s%n%b' "$REVISION_RANGE")" +BUMP=patch + +if grep -Eq '^[[:alnum:]_-]+(\([^)]*\))?!:' <<<"$COMMIT_MESSAGES" \ + || grep -Eq '^BREAKING([ -])CHANGE:' <<<"$COMMIT_MESSAGES"; then + BUMP=major +elif grep -Eq '^feat(\([^)]*\))?:' <<<"$COMMIT_MESSAGES"; then + BUMP=minor +fi + +VERSION="${CURRENT_TAG#v}" +IFS=. read -r MAJOR MINOR PATCH <<<"$VERSION" + +case "$BUMP" in + major) + MAJOR=$((MAJOR + 1)) + MINOR=0 + PATCH=0 + ;; + minor) + MINOR=$((MINOR + 1)) + PATCH=0 + ;; + patch) + PATCH=$((PATCH + 1)) + ;; +esac + +printf 'v%s.%s.%s\n' "$MAJOR" "$MINOR" "$PATCH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bdeda21..07622ee7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: - name: Compose bridge configuration run: node tests/compose-bridge-config.cjs + - name: Release versioning + run: tests/release-versioning.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55c7a116..c47b1a0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,11 @@ # file is inert inside the monorepo — GitHub only runs workflows from the repo # root — and becomes a root workflow in the published repo. # -# Two entry points feed one job: +# Three entry points feed one job: +# +# * successful CI on main — automatically releases deployable changes. The +# next repository version follows Conventional Commit intent; changes that +# only touch docs, workflows, or tests do not cut a release. # # * workflow_dispatch — pick a version in the Actions UI. The chart is # packaged before the tag is created, so a packaging failure aborts while @@ -18,10 +22,13 @@ name: Release on: + workflow_run: + workflows: ['CI'] + types: [completed] workflow_dispatch: inputs: version: - description: 'Version to release, e.g. v2.0.0 or v2.1.0-rc1. Must match helm/codeapi/Chart.yaml appVersion.' + description: 'Repository version to release, e.g. v1.0.0 or v1.1.0-rc1.' required: true type: string draft: @@ -36,12 +43,20 @@ permissions: contents: write concurrency: - group: release-${{ github.event.inputs.version || github.ref_name }} + # Automatic runs serialize against one another. If main advances before an + # older run starts, version resolution skips the stale SHA and the newest + # successful run releases the full range instead. + group: release-${{ github.event_name == 'workflow_run' && 'main' || github.event.inputs.version || github.ref_name }} cancel-in-progress: false jobs: release: name: Tag and publish + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -50,19 +65,60 @@ jobs: # Full history and tags: resolving whether this release is the newest # stable one compares it against every other tag in the repository. fetch-depth: 0 + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.ref }} - name: Resolve and validate version id: version env: EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} INPUT_VERSION: ${{ github.event.inputs.version }} INPUT_DRAFT: ${{ github.event.inputs.draft }} REF_NAME: ${{ github.ref_name }} REF_TYPE: ${{ github.ref_type }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + SKIP=false + if [ "$EVENT_NAME" = "workflow_run" ]; then + if [ "$(git rev-parse HEAD)" != "$HEAD_SHA" ]; then + echo "::error::Checked out SHA does not match the successful CI run" + exit 1 + fi + + git fetch --no-tags origin main:refs/remotes/origin/main + if [ "$(git rev-parse refs/remotes/origin/main)" != "$HEAD_SHA" ]; then + echo "main advanced after this CI run; the newer successful run will release the combined changes" + SKIP=true + fi + + # A rerun after tag creation but before release publication resumes + # the missing release rather than incrementing the version again. + EXACT_TAG="$({ git tag --points-at HEAD || true; } | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' | sort -V | tail -n 1)" + if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then + if gh release view "$EXACT_TAG" >/dev/null 2>&1; then + echo "$EXACT_TAG already publishes this commit; nothing to do" + SKIP=true + else + VERSION="$EXACT_TAG" + fi + elif [ "$SKIP" = "false" ]; then + PREVIOUS_TAG="$(git tag --merged HEAD \ + | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ + | sort -V \ + | tail -n 1)" + if [ -z "$PREVIOUS_TAG" ]; then + echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" + exit 1 + fi + VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" + if [ -z "$VERSION" ]; then + echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" + SKIP=true + fi + fi + elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then # Releases describe what shipped to main. Dispatching from a topic # branch would tag a commit that is not on the release line. if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then @@ -74,6 +130,11 @@ jobs: VERSION="$REF_NAME" fi + if [ "$SKIP" = "true" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # A bare "2.0.0" typed into the dispatch box is accepted; everything # downstream works with the v-prefixed form the tag actually uses. case "$VERSION" in @@ -82,14 +143,15 @@ jobs: esac if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then - echo "::error::Release tags must be v.. or v..-rcN, for example v2.0.0 or v2.1.0-rc1 (got '$VERSION')" + echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" exit 1 fi - # v2.1.0-rc1 -> 2.1.0. Release candidates carry the version they are - # candidates for, so they compare against the same appVersion. - BASE_VERSION="${VERSION%%-rc*}" - BASE_VERSION="${BASE_VERSION#v}" + if [ "$EVENT_NAME" = "workflow_run" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Calculated tag $VERSION already exists on a different commit" + exit 1 + fi read_chart_field() { grep -m1 "^$1:" helm/codeapi/Chart.yaml \ @@ -98,14 +160,6 @@ jobs: APP_VERSION="$(read_chart_field appVersion)" CHART_VERSION="$(read_chart_field version)" - # The tag is the app version. Requiring the bump to have landed on - # main first keeps a deployed chart from reporting a version that no - # release ever carried. - if [ "$APP_VERSION" != "$BASE_VERSION" ]; then - echo "::error::Tag $VERSION does not match helm/codeapi/Chart.yaml appVersion ($APP_VERSION). Land the appVersion bump on main before releasing." - exit 1 - fi - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." @@ -142,8 +196,8 @@ jobs: fi { + echo "skip=false" echo "version=$VERSION" - echo "base_version=$BASE_VERSION" echo "app_version=$APP_VERSION" echo "chart_version=$CHART_VERSION" echo "prerelease=$PRERELEASE" @@ -157,6 +211,7 @@ jobs: # ci.yml depend on it. - name: Package Helm chart id: chart + if: steps.version.outputs.skip != 'true' run: | set -euo pipefail @@ -183,17 +238,20 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Create tag - if: github.event_name == 'workflow_dispatch' + if: steps.version.outputs.skip != 'true' && github.event_name != 'push' env: VERSION: ${{ steps.version.outputs.version }} run: | set -euo pipefail git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git tag -a "$VERSION" -m "$VERSION" - git push origin "refs/tags/$VERSION" + if ! git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + git tag -a "$VERSION" -m "$VERSION" + git push origin "refs/tags/$VERSION" + fi - name: Publish release + if: steps.version.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} VERSION: ${{ steps.version.outputs.version }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37ecc984..e7b95797 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,10 +25,12 @@ Practical consequences: ## Releases Tagged releases are cut from `main` as `vMAJOR.MINOR.PATCH` (with `-rcN` for -release candidates), and each one carries the packaged Helm chart. The version -comes from `helm/codeapi/Chart.yaml`'s `appVersion`, so a version bump lands on -`main` through the pull request flow above before it can be released. See -[docs/RELEASING.md](docs/RELEASING.md) for the full process. +release candidates), and each one carries the packaged Helm chart. Repository, +API, service, and chart versions advance independently; component version bumps +land on `main` through the pull request flow above before they are included in a +release. Successful `main` CI automatically releases deployable changes while +documentation, workflow, and test-only changes are skipped. See +[docs/RELEASING.md](docs/RELEASING.md) for the full process and manual path. ## Development diff --git a/README.md b/README.md index bad5b66e..3856b254 100644 --- a/README.md +++ b/README.md @@ -108,14 +108,14 @@ Deployments should pin a [tagged release](https://github.com/LibreChat-AI/code-i rather than track `main`, which moves whenever an internal snapshot is merged: ```bash -git clone --branch v2.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git +git clone --branch v1.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git ``` Every release attaches `codeapi-.tgz`, the packaged Helm chart with its Redis and MinIO subcharts vendored: ```bash -helm install codeapi ./codeapi-0.3.0.tgz -f my-values.yaml +helm install codeapi ./codeapi-0.3.1.tgz -f my-values.yaml ``` Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as diff --git a/docs/RELEASING.md b/docs/RELEASING.md index dd00c96d..3b2ff3ef 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -6,44 +6,66 @@ tags are cut. ## Versioning A release is named `vMAJOR.MINOR.PATCH`, optionally with a `-rcN` suffix for a -release candidate — `v2.0.0`, `v2.1.0-rc1`. That version is the **app -version**: `helm/codeapi/Chart.yaml`'s `appVersion` is its source of truth, and -the release workflow refuses any tag that disagrees with it. A release -candidate carries the version it is a candidate for, so `v2.1.0-rc1` also -requires `appVersion: "2.1.0"`. +release candidate — `v1.0.0`, `v1.1.0-rc1`. This is the public repository's +release sequence and is independent from the versions of the components it +contains. The first public release is therefore `v1.0.0` even though the API, +service, and Helm chart already have their own version histories. -Two other version numbers are deliberately independent: +Component versions are deliberately independent: +- `helm/codeapi/Chart.yaml`'s `appVersion` identifies the API version deployed + by the chart. - `helm/codeapi/Chart.yaml`'s `version` is the **chart** version. Bump it when the chart's templates or values change, not when the app changes. It names the packaged chart attached to the release (`codeapi-.tgz`). - `service/package.json`'s `version` tracks the Lambda service package alone. -By convention `api/package.json`'s `version` is kept in step with `appVersion`, -so the API package and the tag agree. Nothing enforces it. +By convention `api/package.json`'s `version` is kept in step with `appVersion`. +Nothing enforces it. -## Cutting a release +## Automatic releases -1. Land the `appVersion` bump on `main` first. `main` takes no direct pushes - (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so it arrives through a sync - pull request from the internal monorepo or a community pull request. Bump - the chart `version` too if the chart changed. +After the full **CI** workflow succeeds for the current tip of `main`, the +release workflow examines everything since the last stable repository tag. It +cuts a release when that range changes deployable files and skips ranges that +only change documentation, GitHub workflows, or tests. If several merges land +while CI is running, the newest successful run releases them together. + +The next version follows Conventional Commit intent across the unreleased +range: + +- a `BREAKING CHANGE:` footer or `type!:` subject bumps the major version; +- a `feat:` subject bumps the minor version; +- every other deployable change bumps the patch version. + +This makes the safe fallback a patch release even when a merge title does not +follow the convention. The workflow packages the Helm chart before creating +the tag, so a packaging failure leaves the version available for a retry. A +rerun also resumes publication if the tag was created before a later step +failed. + +## Manual releases + +1. Land every intended component version bump on `main` first. `main` takes no + direct pushes (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so changes arrive + through a sync pull request from the internal monorepo or a community pull + request. Bump the chart `version` only if the chart changed. 2. Run the **Release** workflow from the Actions tab against `main`, entering - the version (`v2.1.0`). Tick *draft* to review the generated notes before - they go public. + the next repository version (`v1.1.0`). Tick *draft* to review the generated + notes before they go public. -The workflow validates the version, packages the Helm chart, then creates the -annotated tag and publishes the release. Packaging runs before tagging so a -failure — a rate-limited subchart pull, most likely — leaves the version -unused and the run safe to retry. +Use this path when intentionally overriding the automatically selected version, +cutting a release candidate, or recovering while automatic releases are +disabled. The workflow validates the version, packages the Helm chart, then +creates the annotated tag and publishes the release. A tag pushed by hand works as well, and takes the same path from validation onward: ```bash git checkout main && git pull -git tag -a v2.1.0 -m v2.1.0 -git push origin v2.1.0 +git tag -a v1.1.0 -m v1.1.0 +git push origin v1.1.0 ``` ## What the release contains @@ -64,7 +86,7 @@ repository, so re-cutting an older patch cannot drag it backwards. Delete the release and its tag, then re-run the workflow: ```bash -gh release delete v2.1.0 --cleanup-tag --yes +gh release delete v1.1.0 --cleanup-tag --yes ``` Republishing the same version is only safe while nobody has deployed it. Once diff --git a/tests/release-versioning.sh b/tests/release-versioning.sh new file mode 100755 index 00000000..3ab80bef --- /dev/null +++ b/tests/release-versioning.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RESOLVER="$ROOT/.github/scripts/next-release-version.sh" +TEST_REPO="$(mktemp -d)" +trap 'rm -rf "$TEST_REPO"' EXIT + +git -C "$TEST_REPO" init -q +git -C "$TEST_REPO" config user.name test +git -C "$TEST_REPO" config user.email test@example.com + +commit_file() { + local path="$1" + local content="$2" + local message="$3" + + mkdir -p "$TEST_REPO/$(dirname "$path")" + printf '%s\n' "$content" > "$TEST_REPO/$path" + git -C "$TEST_REPO" add "$path" + git -C "$TEST_REPO" commit -q -m "$message" +} + +assert_version() { + local expected="$1" + local actual + actual="$(cd "$TEST_REPO" && bash "$RESOLVER" v1.2.3 v1.2.3..HEAD)" + if [ "$actual" != "$expected" ]; then + echo "expected '$expected', got '$actual'" >&2 + exit 1 + fi +} + +commit_file api/runtime.ts initial 'chore: initial release' +git -C "$TEST_REPO" tag v1.2.3 + +commit_file docs/guide.md docs 'docs: clarify deployment' +assert_version '' + +commit_file api/runtime.ts fix 'fix: repair execution' +assert_version v1.2.4 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file api/runtime.ts feature 'feat: add execution mode' +assert_version v1.3.0 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file api/runtime.ts breaking 'feat!: replace execution protocol' +assert_version v2.0.0 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file service/config.ts config 'chore: tune runtime defaults' +assert_version v1.2.4 + +echo 'release versioning tests passed' From f6cdfb3658a0f61c81cb6c63b73eaacf03417463 Mon Sep 17 00:00:00 2001 From: Jackson Riding <99007683+jacksonriding@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:26:27 +1000 Subject: [PATCH 107/116] fix: forward input-file limit into sandbox guests (#217) --- launcher/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7771b9b9..27f08f92 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -433,6 +433,7 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_LOG_LEVEL", "SANDBOX_MAX_CONCURRENT_JOBS", "SANDBOX_MAX_FILE_SIZE", + "SANDBOX_MAX_INPUT_FILES", "SANDBOX_MAX_NESTING_DEPTH", "SANDBOX_MAX_OPEN_FILES", "SANDBOX_MAX_OUTPUT_FILES", @@ -850,6 +851,13 @@ mod tests { } } + #[test] + fn guest_env_allowlist_forwards_input_file_limit_in_both_egress_modes() { + for egress_gateway_enabled in [false, true] { + assert!(is_allowed_guest_env_key("SANDBOX_MAX_INPUT_FILES", egress_gateway_enabled)); + } + } + #[test] fn guest_env_allowlist_preserves_legacy_file_server_url_only_without_egress_gateway() { assert!(is_allowed_guest_env_key("FILE_SERVER_URL", false)); From b35c503fd2fe7be412d95c0eef6db50a09aad280 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 01:51:42 -0400 Subject: [PATCH 108/116] feat: Inspect Local Coding Projects (#221) * feat: add bounded local project inventory * fix: report incomplete Git metadata reads * fix: preserve incomplete discovery and remote identities * fix: finalize discovery budgets and nested remote identities * fix(code): stop project traversal at filesystem budget boundaries --- docs/remote-bridge/projects.md | 69 ++++++ packages/code/README.md | 37 +++ packages/code/src/cli.ts | 11 + packages/code/src/projects.test.ts | 368 +++++++++++++++++++++++++++++ packages/code/src/projects.ts | 295 +++++++++++++++++++++++ 5 files changed, 780 insertions(+) create mode 100644 docs/remote-bridge/projects.md create mode 100644 packages/code/src/projects.test.ts create mode 100644 packages/code/src/projects.ts diff --git a/docs/remote-bridge/projects.md b/docs/remote-bridge/projects.md new file mode 100644 index 00000000..987ce112 --- /dev/null +++ b/docs/remote-bridge/projects.md @@ -0,0 +1,69 @@ +# Projects and worktrees + +The intended experience is to select a project on an attached machine, choose +the current checkout or a new worktree, and have subsequent tool calls and +approval resumes use that selection without repeating a working directory. + +## Delivery sequence + +1. Local project inventory (`librechat-code projects --root `). + Discover bounded Git metadata without changing registration or authority. +2. Negotiated project selection across the worker, Code API, and LibreChat. + Persist the selection and enforce the same project boundary in file tools, + commands, programmatic execution, environment actions, and approval resumes. +3. Worktree creation and setup with durable operation receipts. Publish a new + selection only after Git creation, setup, and registration have completed. +4. Worktree selection in the composer and an approved agent operation. Allow + an authenticated owner to narrow a selection to a newly created worktree + with a compare-and-set against the prior conversation decision. +5. Explicit listing and removal, with binding checks and recovery for uncertain + outcomes. Retention policy determines eligibility, not automatic permission + to destroy uncommitted work or unpublished commits. + +Only the first step is implemented by the inventory command. Existing explicit +workspace registration remains available for independent project directories. + +## Boundaries that must remain consistent + +- Discovery metadata is advisory. A remote is not an authorization grant, a + trusted repository identity, or automatically a codegraph repository ID. + Keep the host in normalized remotes to distinguish identically named repos. +- A discovery root may authorize enumeration while an execution root narrows + writes to one selected project. A broad parent must not remain an independent + concurrent execution lane alongside its descendants. +- Path-derived IDs must be scoped by the registered root. Admission must + validate the current directory identity; inventory cannot reserve a path + against replacement after discovery. +- Discovery must not run on every status request. A future worker catalog + needs bounded caching, coalesced refreshes, and explicit generation changes. +- A linked worktree shares Git metadata with its parent. Project IDs alone + cannot make those metadata mutations independent. Admission needs both a + filesystem boundary and coordination for the common Git directory. +- A worktree beneath its parent checkout overlaps that checkout. Either use + disjoint execution roots under a discovery grant or explicitly exclude and + coordinate descendant worktrees before relaxing root exclusion. +- Setup and dependency links must remain within the execution policy. Sharing + writable dependency directories between supposedly isolated worktrees + reintroduces overlap and requires an explicit operator decision. +- Dynamic registration requires versioned capabilities and fenced catalog + generations. Old consumers must not silently drop a project selection and + execute against its broader parent. Deploy consumers before producers. +- Create, setup, registration, and conversation binding form a recoverable + lifecycle. A network retry must find the same worktree, not create a second + one. Failed setup leaves it unavailable; uncertain mutation quarantines it. +- Approval decisions must include the exact target and operation. Creating a + branch changes repository state and follows mutation policy. An additive + operation is not automatically exempt from required approval. +- Cleanup must coordinate live bindings and active execution. A missing remote + branch alone does not prove a worktree is disposable. + +## Acceptance cases for selection and lifecycle + +Verify separate repositories under one discovery root, two chats sharing one +project, two worktrees sharing Git metadata, directory replacement, stale +catalog generations, old/new consumer combinations, and cross-principal access. +Exercise file tools, commands, programmatic execution, and environment actions +through the same persisted selection. Include pause/resume, cancellation during +creation and setup, process death before registration, retry after binding, and +removal racing an active conversation. Use disposable local fixtures before +testing the hosted deployment. diff --git a/packages/code/README.md b/packages/code/README.md index 3cc61677..48bf769d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -14,6 +14,43 @@ container/NsJail profile. The worker connects outbound to Code API, long-polls for assignments, sends them to the local runtime, and returns fenced results. The VM does not need an inbound public port. +## Inspect local projects + +Before registering a directory containing several checkouts, inspect its Git +projects on the worker machine: + +```bash +librechat-code projects --root /srv/projects +``` + +The command prints JSON with `projects`, `truncated`, and `incomplete`. Each +project contains a path relative to the requested directory, a path-derived ID, +and the current origin, branch, and HEAD. Origins are normalized to +`host[:port]/namespace/repository`, including nested namespaces; URL credentials, +query strings, and fragments are omitted. An unsupported configured origin is +redacted to null and marks the inventory incomplete. +A detached HEAD has a null branch; an unborn branch has a null HEAD. IDs stay +stable when branches change, but moving or renaming a directory changes its ID. +IDs are local to the supplied discovery root. + +Discovery runs only when requested. Its default limits are three directory +levels, 10,000 entries, 256 projects, and a ten-second processing budget with +bounded Git subprocess output and timeouts. +The time budget starts before resolving the root and is checked between native +filesystem operations; it cannot interrupt a kernel call stalled on a filesystem. +Use a responsive local filesystem. Discovery skips hidden directories, +dependencies, symlinks, and children of an identified repository. Linked +worktrees and submodules using a `.git` file are skipped and set `incomplete`: +their shared Git metadata needs separate admission before independent execution. +An empty project list does not prevent registering a non-Git directory. + +This is a local inventory command. It does not clone, register roots, pair a +worker, change the sandbox, or automatically select a conversation workspace. +For the existing picker and independent lease slots, explicitly register the +chosen non-overlapping project directories with `--workspace` or `--environment`. +Do not also register their parent directory. Treat the inventory as a snapshot; +normal workspace admission must validate any directory selected from it. + ## Pair Hardened deployments use a one-time code instead of copying a long-lived diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 00eca576..7406e267 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,7 @@ import { realpath, stat } from 'node:fs/promises'; import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { discoverProjects } from './projects.js'; import { loadCodeEnvironment, assertEnvironmentDefinitionsOutsideRoots, @@ -1237,6 +1238,16 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'projects') { + const root = option(args, '--root'); + if (!root || args.slice(1).some((arg, index, rest) => + arg !== '--root' && rest[index - 1] !== '--root' && !arg.startsWith('--root='))) { + throw new Error('Usage: librechat-code projects --root '); + } + const inventory = await discoverProjects({ root }); + process.stdout.write(`${JSON.stringify(inventory, null, 2)}\n`); + return; + } if (args[0] === 'relay') { await relay(); return; diff --git a/packages/code/src/projects.test.ts b/packages/code/src/projects.test.ts new file mode 100644 index 00000000..98ce0949 --- /dev/null +++ b/packages/code/src/projects.test.ts @@ -0,0 +1,368 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + chmod, + mkdtemp, + mkdir, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import fs from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import { discoverProjects, projectRemote } from './projects.js'; + +const exec = promisify(execFile); +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), 'code-projects-')); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} +async function repo(root: string, path: string) { + const directory = join(root, path); + await mkdir(directory, { recursive: true }); + await exec('git', ['init', '--initial-branch=dev', directory]); + return directory; +} + +test('discovers real sibling repositories with stable IDs and bounded metadata', async t => { + const root = await fixture(t); + const a = await repo(root, 'a'); + await repo(root, 'nested/b'); + await exec('git', [ + '-C', + a, + 'remote', + 'add', + 'origin', + 'https://user:secret@github.com/example/app.git?token=secret', + ]); + await exec('git', [ + '-C', + a, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '--allow-empty', + '-m', + 'initial', + ]); + const before = await discoverProjects({ root }); + assert.equal(before.incomplete, false); + assert.equal(before.truncated, false); + assert.deepEqual( + before.projects.map(p => p.path), + ['a', 'nested/b'] + ); + assert.equal(before.projects[0].remote, 'github.com/example/app'); + assert.equal(before.projects[0].branch, 'dev'); + assert.match(before.projects[0].head!, /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/); + assert.equal(before.projects[1].head, null); + assert.ok(!JSON.stringify(before).includes('secret')); + await exec('git', ['-C', a, 'checkout', '-b', 'next']); + const after = await discoverProjects({ root }); + assert.deepEqual( + after.projects.map(p => p.id), + before.projects.map(p => p.id) + ); + assert.equal(after.projects[0].branch, 'next'); +}); + +test('does not walk dependencies, hidden directories, symlinks or repository children', async t => { + const root = await fixture(t); + await repo(root, 'node_modules/ignored'); + await repo(root, '.hidden/ignored'); + await repo(root, 'parent'); + await repo(root, 'parent/nested'); + const outside = await fixture(t); + await repo(outside, 'external'); + await symlink(outside, join(root, 'alias'), 'dir'); + const inventory = await discoverProjects({ root }); + assert.deepEqual( + inventory.projects.map(p => p.path), + ['parent'] + ); +}); + +test('linked worktrees are reported incomplete until shared git metadata is admitted', async t => { + const root = await fixture(t); + await mkdir(join(root, 'linked')); + await writeFile( + join(root, 'linked', '.git'), + 'gitdir: /outside/metadata\n' + ); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.deepEqual(inventory.projects, []); +}); + +test('oversized Git metadata is incomplete rather than silently reported absent', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + await exec('git', [ + '-C', + directory, + 'config', + 'remote.origin.url', + 'x'.repeat(10_000), + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].remote, null); +}); + +test('a valid branch beyond the metadata bound reports incomplete', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + const branch = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)].join( + '/' + ); + await exec('git', [ + '-C', + directory, + 'symbolic-ref', + 'HEAD', + `refs/heads/${branch}`, + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].branch, null); +}); + +test('unreadable Git markers report incomplete', async t => { + if (process.platform === 'win32' || process.getuid?.() === 0) { + t.skip('requires POSIX permissions under an unprivileged account'); + return; + } + const root = await fixture(t); + const directory = await repo(root, 'app'); + await chmod(directory, 0o400); + try { + assert.equal( + (await discoverProjects({ root: directory })).incomplete, + true + ); + } finally { + await chmod(directory, 0o700); + } +}); + +test('root resolution consumes the processing budget and pre-abort wins', async t => { + const root = await fixture(t); + let ticks = 0; + t.mock.method(Date, 'now', () => (ticks++ === 0 ? 0 : 20_000)); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.truncated, true); + assert.deepEqual(inventory.projects, []); + const reason = new Error('cancelled before filesystem access'); + await assert.rejects( + discoverProjects({ + root: join(root, 'missing'), + signal: AbortSignal.abort(reason), + }), + error => error === reason + ); +}); + +test('empty directory completion checks a late deadline and cancellation', async t => { + const root = await fixture(t); + const original = fs.opendir; + let clock = 0; + let cancel: AbortController | undefined; + t.mock.method(Date, 'now', () => clock); + const openMock = t.mock.method( + fs, + 'opendir', + async (path: Parameters[0]) => { + const directory = await original(path); + clock = 20_000; + cancel?.abort(); + return directory; + } + ); + syncBuiltinESMExports(); + try { + assert.equal((await discoverProjects({ root })).truncated, true); + clock = 0; + cancel = new AbortController(); + await assert.rejects( + discoverProjects({ root, signal: cancel.signal }), + { name: 'AbortError' } + ); + } finally { + openMock.mock.restore(); + syncBuiltinESMExports(); + } +}); + +test('late filesystem boundaries stop before starting the next operation', async t => { + const root = await fixture(t); + for (const boundary of [3, 4, 5]) { + for (const abort of [false, true]) { + let calls = 0; + let clock = 0; + const controller = new AbortController(); + const now = t.mock.method(Date, 'now', () => clock); + const mocks = ['lstat', 'realpath', 'opendir'].map(name => { + const original = fs[name as 'lstat']; + return t.mock.method( + fs, + name as 'lstat', + async (...args: Parameters) => { + calls++; + try { + return await original(...args); + } finally { + if (calls === boundary) { + clock = 20_000; + if (abort) controller.abort(); + } + } + } + ); + }); + syncBuiltinESMExports(); + try { + const discovery = discoverProjects({ + root, + signal: controller.signal, + }); + if (abort) + await assert.rejects(discovery, { name: 'AbortError' }); + else assert.equal((await discovery).truncated, true); + assert.equal(calls, boundary); + } finally { + for (const mock of mocks) mock.mock.restore(); + now.mock.restore(); + syncBuiltinESMExports(); + } + } + } +}); + +test('unsupported configured origins are distinguishable from missing origins', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + await exec('git', [ + '-C', + directory, + 'config', + 'remote.origin.url', + '/private/local/repo', + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].remote, null); +}); + +test('project, entry and depth ceilings report partial discovery', async t => { + const root = await fixture(t); + await repo(root, 'a'); + await repo(root, 'b'); + await repo(root, 'nested/deeper/c'); + assert.equal( + (await discoverProjects({ root, maxProjects: 1 })).truncated, + true + ); + assert.equal( + (await discoverProjects({ root, maxEntries: 1 })).truncated, + true + ); + assert.equal( + (await discoverProjects({ root, maxDepth: 1 })).truncated, + true + ); + await assert.rejects(discoverProjects({ root, maxDepth: 100 }), /limit/); + await assert.rejects( + discoverProjects({ root, signal: AbortSignal.abort() }) + ); +}); + +test('root checkout uses dot and detached HEAD has no branch', async t => { + const root = await fixture(t); + await repo(root, '.'); + await exec('git', [ + '-C', + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '--allow-empty', + '-m', + 'initial', + ]); + await exec('git', ['-C', root, 'checkout', '--detach']); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.projects[0].path, '.'); + assert.equal(inventory.projects[0].branch, null); +}); + +test('repository identity retains host and drops credentials, query and fragments', () => { + assert.equal( + projectRemote('ssh://git@example.com:2222/org/repo.git'), + 'example.com:2222/org/repo' + ); + assert.equal( + projectRemote('https://example.com:8443/org/repo.git'), + 'example.com:8443/org/repo' + ); + assert.equal( + projectRemote('git@github.com:org/repo.git'), + 'github.com/org/repo' + ); + assert.equal( + projectRemote('ssh://git@example.com/org/repo.git'), + 'example.com/org/repo' + ); + assert.equal( + projectRemote('https://token@example.com/org/repo.git?secret#fragment'), + 'example.com/org/repo' + ); + assert.equal(projectRemote('/home/user/private'), null); + assert.equal(projectRemote('file:///home/user/private'), null); + assert.equal( + projectRemote('https://example.com/group/subgroup/repo.git'), + 'example.com/group/subgroup/repo' + ); + assert.equal( + projectRemote('git@example.com:group/subgroup/repo.git'), + 'example.com/group/subgroup/repo' + ); +}); + +test('CLI inventories a real checkout without pairing or starting a worker', async t => { + const root = await fixture(t); + await repo(root, 'app'); + const { stdout, stderr } = await exec( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'projects', + '--root', + root, + ], + { env: { PATH: process.env.PATH }, timeout: 15_000 } + ); + const result = JSON.parse(stdout); + assert.equal(stderr, ''); + assert.equal(result.projects[0].path, 'app'); + assert.equal(result.projects[0].branch, 'dev'); + assert.equal(result.incomplete, false); + await assert.rejects( + exec(process.execPath, [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'projects', + ]), + /Usage: librechat-code projects/ + ); +}); diff --git a/packages/code/src/projects.ts b/packages/code/src/projects.ts new file mode 100644 index 00000000..1e2fcbd3 --- /dev/null +++ b/packages/code/src/projects.ts @@ -0,0 +1,295 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstat, opendir, realpath } from 'node:fs/promises'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +const exec = promisify(execFile); +const skipped = new Set(['node_modules', 'vendor']); + +export interface LocalProject { + id: string; + path: string; + remote: string | null; + branch: string | null; + head: string | null; +} + +export interface ProjectInventory { + projects: LocalProject[]; + truncated: boolean; + incomplete: boolean; +} + +export interface ProjectDiscoveryOptions { + root: string; + maxDepth?: number; + maxProjects?: number; + maxEntries?: number; + timeoutMs?: number; + signal?: AbortSignal; +} + +/** Public repository identity only: never propagate credentials or URL query data. */ +export function projectRemote(value: string): string | null { + let host: string; + let path: string; + try { + const scp = /^(?:[^/@:\s]+@)?([^/:\s]+):([^\s]+)$/.exec(value); + if (scp && !value.includes('://')) { + host = scp[1]; + path = scp[2]; + } else { + const url = new URL(value); + if (!['https:', 'http:', 'ssh:', 'git:'].includes(url.protocol)) + return null; + host = url.host; + path = url.pathname.replace(/^\//, ''); + } + path = path.replace(/\.git$/, ''); + if ( + !/^[A-Za-z0-9.-]+(?::[0-9]+)?$/.test(host) || + !/^[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)+$/.test(path) + ) + return null; + if (path.split('/').some(part => part === '.' || part === '..')) + return null; + return `${host.toLowerCase()}/${path}`; + } catch { + return null; + } +} + +function limit( + value: number | undefined, + fallback: number, + maximum: number +): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < 1 || resolved > maximum) + throw new Error('Invalid project discovery limit'); + return resolved; +} + +/** Bounded local inventory; it does not grant roots or mutate a checkout. */ +export async function discoverProjects( + options: ProjectDiscoveryOptions +): Promise { + const maxDepth = limit(options.maxDepth, 3, 16); + const maxProjects = limit(options.maxProjects, 256, 256); + const maxEntries = limit(options.maxEntries, 10_000, 100_000); + const timeoutMs = limit(options.timeoutMs, 10_000, 60_000); + const deadline = Date.now() + timeoutMs; + const result: ProjectInventory = { + projects: [], + truncated: false, + incomplete: false, + }; + let entries = 0; + const expired = (): boolean => { + options.signal?.throwIfAborted(); + if (Date.now() < deadline) return false; + result.truncated = true; + return true; + }; + options.signal?.throwIfAborted(); + const root = await realpath(options.root); + if (expired()) return result; + const rootStat = await lstat(root); + if (expired()) return result; + if (!rootStat.isDirectory()) + throw new Error('Project root must be a directory'); + const queue = [{ path: root, depth: 0 }]; + const git = async ( + path: string, + args: string[], + expectedExitCodes: number[] = [] + ): Promise => { + if (expired()) return null; + try { + const { stdout } = await exec( + 'git', + [ + '--no-optional-locks', + '-C', + path, + '-c', + 'core.fsmonitor=false', + ...args, + ], + { + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }, + encoding: 'utf8', + maxBuffer: 4096, + timeout: Math.max(1, Math.min(1500, deadline - Date.now())), + signal: options.signal, + } + ); + return stdout.trim(); + } catch (error) { + options.signal?.throwIfAborted(); + const expected = + error instanceof Error && + 'code' in error && + typeof error.code === 'number' && + expectedExitCodes.includes(error.code); + if (!expected) result.incomplete = true; + return null; + } + }; + for (let index = 0; index < queue.length; index++) { + if (expired()) break; + const current = queue[index]; + try { + // Revalidate queued directories; never traverse a replaced symlink. + const currentStat = await lstat(current.path); + if (expired()) break; + if (currentStat.isSymbolicLink()) { + result.incomplete = true; + continue; + } + const canonical = await realpath(current.path); + if (expired()) break; + const rel = relative(root, canonical); + if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + result.incomplete = true; + continue; + } + const marker = await lstat(resolve(current.path, '.git')).catch( + error => { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + return undefined; + } + ); + if (expired()) break; + if (marker) { + // Linked worktrees and submodules need separate shared-gitdir admission. + if (!marker.isDirectory() || marker.isSymbolicLink()) { + result.incomplete = true; + continue; + } + if (result.projects.length === maxProjects) { + result.truncated = true; + break; + } + const top = await git(current.path, [ + 'rev-parse', + '--show-toplevel', + ]); + if (expired()) break; + const canonicalTop = top + ? await realpath(top).catch(() => null) + : null; + if (expired()) break; + if (!top || canonicalTop !== canonical) { + result.incomplete = true; + continue; + } + const remote = await git( + current.path, + [ + 'config', + '--local', + '--no-includes', + '--get', + 'remote.origin.url', + ], + [1] + ); + const branch = await git( + current.path, + ['symbolic-ref', '--quiet', '--short', 'HEAD'], + [1] + ); + const head = await git( + current.path, + ['rev-parse', '--verify', 'HEAD'], + branch ? [128] : [] + ); + const path = rel.split(sep).join('/') || '.'; + const normalizedRemote = remote ? projectRemote(remote) : null; + const validBranch = + branch && + branch.length <= 256 && + !/[\x00-\x1f\x7f]/.test(branch) + ? branch + : null; + const validHead = + head && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(head) + ? head + : null; + if ( + (remote !== null && normalizedRemote === null) || + (branch !== null && validBranch === null) || + (head !== null && validHead === null) + ) + result.incomplete = true; + result.projects.push({ + id: `project-${createHash('sha256') + .update(path) + .digest('hex') + .slice(0, 32)}`, + path, + remote: normalizedRemote, + branch: validBranch, + head: validHead, + }); + continue; + } + if (current.depth === maxDepth) { + result.truncated = true; + continue; + } + const directory = await opendir(current.path); + // Close a newly opened handle even when cancellation won during open. + try { + if (expired()) { + await directory.close(); + break; + } + } catch (error) { + await directory.close(); + throw error; + } + for await (const entry of directory) { + if (expired() || ++entries > maxEntries) { + result.truncated = true; + result.projects.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + ); + return result; + } + if ( + entry.isDirectory() && + !entry.name.startsWith('.') && + !skipped.has(entry.name) + ) + queue.push({ + path: resolve(current.path, entry.name), + depth: current.depth + 1, + }); + } + } catch { + options.signal?.throwIfAborted(); + result.incomplete = true; + } + } + result.projects.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + ); + expired(); + return result; +} From 999c2e590995c4c48623e5e054e73b9036ba2e23 Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 16:41:35 +0200 Subject: [PATCH 109/116] docs(project): add upstream v1.1.0 integration plan --- ...-09-17-upstream-v1.1.0-integration-plan.md | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md diff --git a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md new file mode 100644 index 00000000..1b33311e --- /dev/null +++ b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md @@ -0,0 +1,125 @@ +# Upstream v1.1.0 integration plan + +Status: draft +Package: full path (security, architecture, deployment seams) +Branch (planned): chore/reconcile-upstream-v1.1.0 +Target: uzh-bf/code-interpreter main +Upstream release: LibreChat-AI/code-interpreter v1.1.0 (tag b35c503fd2fe, chart 0.3.1, published 2026-09-16) + +## Approval summary + +The UZH fork is still required: v1.1.0 absorbs none of the ten fork behaviors. Upstream remains single-issuer JWT trust, raw Winston logging, five-attempt Redis reconnection, event-only job completion, single-namespace Helm, chart-owned secrets/HPA, hook-based package-init, and has no GHCR publication workflow. The fork is active production GitOps infrastructure (df-cloud Argo Applications, pinned UZH GHCR images). + +This package merges upstream v1.1.0 into the fork while replaying all fork behaviors onto it. The delta is large (108 upstream commits since merge-base 2c7fb8fc): the BYOM remote-worker system, hosted apps, PTC cancellation, workspace tools and admission capacity, artifact delivery/truncation, express 5 / nanoid 5, chart 0.3.1, and new security fixes (bounded upload memory, input-file limits into guests, credential ACL). A trial merge produced 8 conflicts: 3 Dockerfiles, queue.ts, programmatic-router.ts, librechat-jwt.ts, rollup.config.js, tsconfig.json. + +Nothing in current fork behavior is dropped. Two patches get explicit re-review: the spec-guard chmod 0555 delta (upstream now ships sandbox-rootfs-setup with bind-mount rootfs handling) and the PVC package-init de-hook (production runs baked images; the patch keeps rollback compatibility only). Both conservative resolutions are named below. + +Deployment, image rollout, GitOps promotion, and any cluster change are excluded. The package ends at a green-CI draft PR on a task branch plus PR dispositions. Merging into main, closing superseded dependabot PRs, and the PR #21 follow-up are separately gated decisions named below. + +Done means: reconciled branch with all tests/builds/renders green at the exact head, final review passed, patch ledger updated, draft PR open, and dispositions recorded. + +## Evidence and binding contracts + +### Live state (verified 2026-09-17) + +- origin/main = f5bf3b4c1 (2026-09-11, PR #23 nonfatal telemetry). Local checkout is stale (fix/spec-guard-readable at June state, 26 behind); local refs cannot fetch (sandbox .git read-only). All remote state read via git ls-remote + gh api. +- Upstream real remote is LibreChat-AI/code-interpreter (the configured ClickHouse URL redirects there). v1.1.0 = b35c503fd2fe; upstream main is only 4 commits ahead of the tag. Target the tag, not main. +- Divergence: fork is 20 ahead / 108 behind v1.1.0 from merge-base 2c7fb8fc (2026-08-18, the PR #17 reconcile point). +- Trial merge origin/main + v1.1.0 (scratch clone, kept at /tmp/ci-reconcile-1): 8 conflicted files - api/Dockerfile, docker/Dockerfile.worker-sandbox, launcher/Dockerfile, service/rollup.config.js, service/src/auth/librechat-jwt.ts, service/src/queue.ts, service/src/service/programmatic-router.ts, service/tsconfig.json. Helm templates, values.yaml, ci.yml, egress-ledger.ts, and logger sinks auto-merged. + +### Fork patch disposition (each verified against v1.1.0 source) + +| # | Behavior | v1.1.0 status | Disposition | +| --- | --- | --- | --- | +| 1 | Publish exact-SHA UZH GHCR images | no equivalent workflow (upstream release.yml publishes tags, not images) | Keep; validate matrix against v1.1.0 Dockerfiles | +| 2 | spec-guard readable roots (chmod 0555) | upstream ships docker/rootfs-setup.c + sandbox-rootfs-setup, still chmod 0111 | Keep 0555 for both binaries; adopt rootfs-setup.c; validate via image build + CI smoke | +| 3 | Split/harden untrusted sandbox namespace | still single-namespace, no namespaceOverride | Keep (PRD two-namespace PSA split) | +| 4 | Cede ownership to external controllers | still chart-owned secrets/HPA/replicas | Keep; integrate upstream's new api Recreate fence + pullPolicy: Always | +| 5 | Argo-safe PVC package-init | still Helm hook + TTL | Keep as rollback compatibility; mark for retirement review after rollout | +| 6 | BullMQ poll fallback (queue-wait.ts) | still event-only waitUntilFinished; new waitForJobWithCancellation | Keep; compose poll race into the cancellation-aware waiter | +| 7 | Egress ledger indefinite reconnect | queue/cancellation clients now reconnect indefinitely via redisReconnectDelay; egress-ledger.ts still times > 5 -> null | Keep, narrowed: only egress-ledger.ts remains fork-owned; the queue-client finite-retry concern is resolved by this merge | +| 8 | JWT issuer-scoped trust table | still single-issuer CODEAPI_JWT_ISSUER; new code_worker_id claim | Keep; add code_worker_id to trust-table verifier | +| 9 | Values-free operational logs | raw Winston/Pino unchanged; ~100 new upstream files unevaluated | Keep; full bypass re-inventory over new files | +| 10 | Nonfatal telemetry (PR #23) | optionalTelemetry wrapper absent; upstream changed surrounding code | Keep; re-apply over v1.1.0 telemetry-core | + +PR dispositions: dependabot #4 (otel 2.8.0 already on main), #14/#16/#22 (superseded by v1.1.0 express 5/nanoid 5) - close after reconcile, named authority. PR #21 (public contract docs, draft, stale base) - its documented routes change under v1.1.0; re-derive after reconcile or close; recommendation: re-derive on a fresh branch once this lands. feat/source-sans-pro-fonts has uncommitted worktree changes (+32 lines, 2 Dockerfiles) - preserve and re-apply after the reconcile as a separate follow-up; never discard. + +### Binding contracts + +### External review reconciliation (2026-09-17) + +A second, independent architectural review of this fork was supplied by the user. Its in-scope engineering claims were verified against source here; results: + +- [CONFIRMED] Upstream v1.1.0 provides indefinite reconnect for queue/cancellation clients (service/src/redis-options.ts redisReconnectDelay, used by queue.ts). The fork still relies on finite retry in egress-ledger.ts only. This narrows patch 7 as recorded above; no request-path fork behavior is lost. +- [CONFIRMED] Redis TLS sets rejectUnauthorized false in both fork and upstream (service/src/queue.ts, egress-ledger.ts, file-server.ts, tool-call-server.ts). Pre-existing and upstream-owned; the merge neither introduces nor fixes it. Recorded as a separate follow-up (generic upstream contribution: certificate validation with dev-only exception), NOT a slice of this package. +- [CONFIRMED] .github/workflows/open-sync-pr.yml is not a reconcile bot; it only opens a PR for internal sync/main snapshots. PR #17 was a manual reconcile merge. +- [CONFIRMED] Queue retention (age 60 / count) and job.timestamp-derived deadlines mean completed-job records are not durable execution history; this is upstream execution semantics, unchanged by the merge. +- [OUT OF SCOPE] The review's strategic sequence (capacity-aware admission, separate budgets, durable status, Spot attempt fencing, Klicker teaching profile, departmental pilots) is future work. This package is its Package 1 (baseline and upstream reconciliation) only. + +- Upstream execution-profile queue names, timeouts, cancellation registry, and new claims are authoritative; fork patches adapt around them (per PR #17 precedent). +- Existing STG/PRD values must keep rendering one package-init Job + PVC only in source=pvc compatibility mode; baked-image mode stays default. +- Values-free policy: no identifiers, filenames, payloads, raw errors/stacks, child output, or caller values in enabled sinks; unknown errors serialize as internal. +- JWT: unverified issuer only locates a trust entry; key/alg/issuer/audience/principal-source verification is fail-closed; one key belongs to exactly one issuer. +- No secrets, personal data, or production payloads enter commits, tests, or PR text. + +## Ownership and sequence + +### Delegation Map + +Workstream A - semantic merge seams (owner: main session; tightly coupled, unresolved security/architecture decisions stay local): S1, S2, S3, S5. +Workstream B - bounded read-only inventories (owner: executor subagent, combo/deepseek-v4.1-flash max, parallelizable after S1): S4-inventory, image-matrix verification. +Workstream C - verification battery and delivery (owner: main session): S6, S7. + +### Slices + +S1 - Merge baseline and mechanical conflicts (main). +Create chore/reconcile-upstream-v1.1.0 from origin/main; merge tag v1.1.0 (--no-ff). Resolve: Dockerfiles (adopt rootfs-setup.c/guest-dns.sh/hosted-app launcher; keep chmod 0555 on both compiled binaries); tsconfig.json/rollup.config.js (retain shared/operational-log.ts alongside new upstream build inputs); queue.ts (combine exports: waitForJobFinished + jobCancellationRegistry + closeQueueConnections). Acceptance: merge commits clean, zero conflict markers, git diff --check passes. + +S2 - Queue completion seam (main). +Compose the UZH events+poll race (queue-wait.ts) into upstream's waitForJobWithCancellation at both router call sites; cancellation checks and poll fallback must not starve each other; loser cleanup on first terminal result. Acceptance: unit tests cover event completion, poll completion/failure, cancellation during wait, timeout, missing job, listener cleanup. + +S3 - JWT trust seam (main). +Re-apply the trust-table verifier over the v1.1.0 verifier; integrate code_worker_id and any new claim handling; keep legacy single-issuer fallback and external: principal sources. Acceptance: existing trust tests plus cross-entry negative tests for the new claim pass. + +S4 - Values-free logging re-inventory + telemetry (inventory delegated, application main). +Inventory every new upstream file for raw console, Winston/Pino constructors, child bindings, error stacks, child-process forwarding bypasses; apply the shared policy at each enabled sink; re-apply optionalTelemetry over v1.1.0 telemetry-core.ts. Acceptance: logger capture tests green; telemetry suite green; inventory recorded in the PR body (not a new permanent file). + +S5 - Chart and image integration (main). +Re-apply fork Helm patches over chart 0.3.1 (namespaceOverride, KEDA replica omission, secrets.create, automount toggles, service links, surge rollout, PVC de-hook); integrate upstream's api Recreate + pullPolicy: Always, ledgerCompact, input-manifest settings; verify the 7-image GHCR matrix against v1.1.0 Dockerfiles (incl. new packages/code/src build inputs). Acceptance: helm template renders STG+PRD values with correct ownership (no chart Secret/HPA in external mode, package-init only in pvc mode); docker buildx builds all seven images. + +S6 - Ledger update and verification battery (main). +Update docs/fork/patches.md (new inventory basis SHAs, disposition changes, retirement-review note for PVC patch). Run: api + service bun tests and builds, launcher cargo tests if toolchain available, deployment-config tests, workflow YAML parse, git diff --check. Acceptance: all green at exact head. + +S7 - Final review and draft PR (main + final-reviewer). +Full-path finish gate: one integrated final review (correctness, maintainability, security - trust boundary and auth changes are in scope; architecture lens for the BYOM/hosted-app seams) over the complete committed range. Then rs-mr-description-writer draft PR (standing authority). Acceptance: review passed or findings resolved; draft PR open with verification receipts. + +## Test portfolio + +| Risk | Obligation | Seam | Failure it must catch | +| --- | --- | --- | --- | +| Missed QueueEvents turns completed jobs into timeouts | extend existing | queue-wait + cancellation interplay | poll path loses to cancellation registry or vice versa | +| Cross-issuer key confusion | extend existing | librechat-jwt trust table | token verified against wrong entry's key/audience | +| New upstream file leaks values into logs | extend existing | logger policy inventory | enabled sink without shared sanitizer | +| Telemetry failure kills requests | extend existing | telemetry-core | constructor/exporter failure propagates to request path | +| Chart regression breaks Argo/KEDA ownership | extend existing | helm template renders | chart re-claims secrets/HPA/replicas or wakes scale-to-zero | +| Images fail to build from merged Dockerfiles | add new | GHCR matrix + buildx | target/path missing after v1.1.0 restructure | +| Egress ledger stays dead after Redis outage | existing | egress-ledger reconnect | retry stops after five attempts | + +## Working context + +- Execution context: sandbox .git is read-only (fetch/commit/branch blocked in the primary checkout). Execution runs in a writable clone (/tmp/ci-reconcile-1 pattern, already validated by the trial merge); push from there to the task branch. Primary checkout stays untouched except this draft; local refs are stale and remote state is read through gh. +- The plan file becomes the execution branch's first commit (docs(project): add upstream v1.1.0 integration plan) - the standard flow is impossible here only because of the read-only .git; the clone reproduces it. +- Note: .github/workflows/open-sync-pr.yml is NOT a reconcile bot; it only opens a PR when an internal monorepo snapshot lands on sync/main. PR #17 was a manual reconcile merge, and this package follows that pattern. +- main has no branch protection; human review remains blocking before merge per user policy, not repo mechanics. +- Rollback: ordinary source revert of the merge commit; deployment rollback stays declarative through df-cloud/Helm (out of scope). + +## Authority + +Granted within this package: branch creation, merge work, commits, task-branch push, draft PR, running tests/builds/renders locally, read-only gh evidence. Standing implementation delivery applies. + +Separately gated (asked at the end, not now): merge into main; closing dependabot PRs #4/#14/#16/#22; PR #21 re-derivation or closure; image publication run; any GitOps/deployment promotion; deleting any branch or worktree. + +## Progress + +- 2026-09-17: three-way analysis complete; trial merge mapped 8 conflicts; all ten fork behaviors verified still-needed against v1.1.0 (patch 7 narrowed per external review); plan drafted. +- 2026-09-17: planner gate BLOCKED - native planner route (gpt-6-astra) failed terminally on an account usage limit; one generic-continuity GLM fallback launched but did not return within the wait. Plan presented with the gate recorded as blocked, not passed. From 967e622089693791903a6f1c1b1a40c8aaa7ab7f Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 17:08:53 +0200 Subject: [PATCH 110/116] docs(fork): re-inventory patch ledger for upstream v1.1.0 Record the v1.1.0 re-audit basis (fork f5bf3b4, upstream tag b35c503, merge base 2c7fb8fc), narrow the egress-ledger patch to the singleton ledger now that upstream reconnects queue clients indefinitely, add the PR #23 nonfatal-telemetry patch, extend the spec-guard and PVC package-init sections with their v1.1.0 evidence and retirement review, and record the values-free logging re-inventory. --- docs/fork/patches.md | 104 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/docs/fork/patches.md b/docs/fork/patches.md index 2782daef..a9ec8660 100644 --- a/docs/fork/patches.md +++ b/docs/fork/patches.md @@ -16,6 +16,29 @@ versions. Historical commits are evidence, not an automatic cherry-pick series. - Limitation: the fork SHA identifies the audited pre-reconciliation branch; the reconciliation PR records the resulting exact head and GitHub merge SHA +### Re-audit basis: upstream v1.1.0 integration (2026-09-17) + +This ledger was re-inventoried when the fork merged upstream release v1.1.0. The +dispositions and contract text above were re-checked against the new upstream +tree; the patch index and every required-behavior section below reflect that +re-check. Where a section's source-evidence quotes the older `2c7fb8fc` +baseline, the v1.1.0 status is recorded in the same section. + +- Fork ref and SHA: `uzh/main` at + `f5bf3b4c17cd6f83cec4323256805957a46a8475` (PR #23 nonfatal telemetry) +- Upstream ref and SHA: LibreChat-AI/code-interpreter tag `v1.1.0` at + `b35c503fd2fe7be412d95c0eef6db50a09aad280` (chart 0.3.1, appVersion 2.0.0) +- Merge base: `2c7fb8fcd7113f0f78b2085e80adf651ea4e5359` (PR #17 reconcile point) +- Audited date: 2026-09-17 +- Method: semantic merge of the exact refs above (scratch clone), per-patch + source and test inspection against v1.1.0, rendered Helm verification, and the + reconciliation PR recording the exact head +- Delta: 108 upstream commits since the merge base. Eight conflicted files were + resolved; Helm templates, `values.yaml`, `ci.yml`, `egress-ledger.ts`, and the + logger sinks auto-merged and were verified rather than replayed. +- Limitation: the fork SHA identifies the pre-integration fork branch; the + reconciliation PR records the resulting exact head and GitHub merge SHA. + States: Active, Review on sync, Draft, History only, Retired. ## Patch index @@ -31,6 +54,7 @@ States: Active, Review on sync, Draft, History only, Retired. | Reconnect the egress ledger after Redis outages | Active | `5e459dd` | Managed Redis | | Bind JWT trust to verified issuers | Active | `f68acf0` | JWT verification keys and issuer configuration | | Keep operational logs values-free | Active | `c87a14d`, `bf83dbe`, `689be7d`, `42a9743` | Winston and Pino logging sinks and public failures | +| Preserve requests through telemetry failures | Active | PR #23 (`f5bf3b4`) | OpenTelemetry SDK and the shared telemetry core | ## Publish exact-SHA UZH images @@ -81,11 +105,19 @@ Source and current-upstream evidence: - Commit `73292bde26900095e8bbd52385a2fa4f84cb25ce` defines the final behavior. - Upstream `2c7fb8fcd7113f0f78b2085e80adf651ea4e5359` does not include the UZH readable-root changes or the matching CI compile-and-smoke check. +- Upstream `v1.1.0` adds `docker/rootfs-setup.c`, `api/src/guest-dns.sh`, and the + bind-mount rootfs handling, but still compiles both binaries with `chmod 0111`. + The merge adopts the new rootfs setup and the hosted-app launcher while keeping + `chmod 0555` on both `spec-guard` and `sandbox-rootfs-setup` in `api/Dockerfile`, + `docker/Dockerfile.worker-sandbox`, and `launcher/Dockerfile`. Replay and drop condition: - Start from upstream Dockerfiles and retain only the directory permissions required by the current `spec-guard` execution path and CI smoke. +- Apply the same 0555 delta to any new compiled sandbox binary upstream adds; the + v1.1.0 merge shows this patch must be re-derived per compiled binary, not + replayed as a wholesale file replacement. - Drop when upstream images pass an equivalent CI check and the deployed sandbox can execute all supported runtimes without the UZH permission delta. @@ -205,6 +237,10 @@ Source and current-upstream evidence: `c1509a88a3189aaf666fe9409ec0c9c539f30c1d` define the final lifecycle. - Upstream `2c7fb8fcd7113f0f78b2085e80adf651ea4e5359` introduced baked-image and PVC package sources, but its PVC Job remains a Helm hook with a TTL. +- Upstream `v1.1.0` (chart 0.3.1) still renders the PVC package-init Job as a Helm + hook with a TTL. The merge retains the Argo-managed retained Job and PVC at sync + wave `-5`, and renders exactly one package-init Job plus PVC only when + `packages.source=pvc`. Replay and drop condition: @@ -212,6 +248,10 @@ Replay and drop condition: to `source=pvc` resources. Existing UZH environments must explicitly retain `source=pvc` while their old directory-root image is pinned, then switch to `source=image` only with a matching baked runner SHA. +- Retirement review: production now runs baked images, so this patch is rollback + compatibility only. Re-check it once every UZH environment has moved to + `source=image` and the pinned directory-root image is no longer needed for + rollback; the patch can then be retired with the compatibility mode. - Drop when upstream's PVC mode renders a retained non-hook Job/PVC that remains a no-op across unchanged Argo syncs and supports the split sandbox namespace. @@ -271,10 +311,16 @@ Source and current-upstream evidence: - Commit `5e459dd4f2d8bea6ae7a3004f15051dff26abae0` defines the final retry policy. - Upstream `2c7fb8fcd7113f0f78b2085e80adf651ea4e5359` still stops reconnecting after five attempts. +- Upstream `v1.1.0` now reconnects the queue and cancellation Redis clients + indefinitely through `redisReconnectDelay` in `service/src/redis-options.ts`, so + only the singleton egress ledger keeps the UZH policy. `queue.ts` uses the + upstream reconnect helper; the fork's finite-retry concern there is resolved. Replay and drop condition: - Preserve upstream ledger behavior and replace only its finite retry strategy. +- Narrowed to `service/src/egress-ledger.ts`; do not replay the old queue-client + retry policy over the upstream `redisReconnectDelay` helper. - Drop when upstream retries indefinitely or a supervised lifecycle reliably recreates the Redis client after terminal disconnect, with a readiness recovery test covering an outage longer than five attempts. @@ -328,6 +374,15 @@ Source and current-upstream evidence: - Upstream `297fead1a0cd997b0e3e6e55f77fbe83b376be1a` and the reconciled UZH baseline `83c4f7b105b6b3e69eda12701ad4ec437acba08f` serialize runtime messages, identifiers, child output, and arbitrary error details without this policy. +- Re-inventoried for v1.1.0: the four upstream-added logger consumers + (`api/src/hosted-app.ts`, `service/src/hosted-app/queue.ts`, + `service/src/hosted-app/worker.ts`, and `service/src/workspace-tools/outcome.ts`) + import the sanitized sinks and pass structured values for the allowlist to + drop. The only direct console call is the known fixed message in + `api/src/tool-call-socket-proxy.ts`, and the policy is centralized in sink + creation (`api/src/logger.ts` via `createOperationalLogger`; + `service/src/logger.ts` via `operationalLogFormat`). No enabled-path bypass was + found. Replay and drop condition: @@ -379,6 +434,43 @@ Replay and drop condition: assignment, fail-closed configuration, bounded external sources, and legacy fallback with matching positive and cross-entry negative tests. +## Preserve requests through telemetry failures + +Required behavior: + +- Keep optional OpenTelemetry instrumentation from failing a request: a failing + tracer, propagator, exporter, processor, resource, provider, registration, or + shutdown path must degrade telemetry without propagating to the request or + response lifecycle. +- Route every optional telemetry construction and lifecycle step through the + shared `optionalTelemetry` guard so a single fault cannot abort request + handling. + +Owned paths: + +- `shared/telemetry-core.ts` +- `shared/telemetry-test-suite.ts` +- `api/src/telemetry.test.ts` +- `service/src/telemetry.test.ts` + +Source and current-upstream evidence: + +- PR #23 (`f5bf3b4c17cd6f83cec4323256805957a46a8475`), "fix(telemetry): preserve + requests through instrumentation failures", introduced the guard and the + shared fault matrix. +- Upstream `v1.1.0` has no `optionalTelemetry` guard in `shared/telemetry-core.ts`; + `api/src/telemetry.ts` and `service/src/telemetry.ts` are otherwise identical to + upstream, so the fork delta is confined to the shared core and the shared test + suite. + +Replay and drop condition: + +- Re-apply the guard around the current upstream `telemetry-core.ts` construction + and shutdown paths, keeping upstream's config shape and request attributes. +- Drop when upstream isolates optional instrumentation failures so no fault in the + OpenTelemetry construction or lifecycle reaches the request path, with a test + matrix covering each fault class. + ## Retired debris - Merge commit `356123a` is history-only transport for the package-init fix; @@ -397,6 +489,12 @@ Replay and drop condition: - Fork-authored non-merge commits were collapsed into the nine logical final behaviors above. The values-free logging package adds thirteen owned or shared paths outside the original 23-path audit. The issuer-trust package adds three - owned paths outside that audit and shares the existing Helm README path. The - only fork merge commit is classified as history-only; no fork-authored - final-tree path is left unowned. + owned paths outside that audit and shares the existing Helm README path. PR #23 + adds the telemetry patch above. The only fork merge commit is classified as + history-only; no fork-authored final-tree path is left unowned. +- v1.1.0 integration coverage: the replay and re-audit above account for all ten + logical behaviors, including the narrowed egress-ledger patch (only + `service/src/egress-ledger.ts` remains fork-owned) and the telemetry patch added + by PR #23. The auto-merged paths (Helm templates, `values.yaml`, `ci.yml`, + `egress-ledger.ts`, logger sinks) were verified against v1.1.0 rather than + replayed; no fork-authored final-tree path is left unowned after the merge. From 53ed390e1e73e1055e53d60bad19d5a3274ee74c Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 18:04:22 +0200 Subject: [PATCH 111/116] fix(reconcile): address final-review findings on the v1.1.0 merge Resolve the completion-event channel from its QueueEvents payload instead of re-reading a possibly-evicted job, and give an in-flight event one poll interval of grace before honoring a poll failure so a completed job is never reported as missing. Restore the poll fallback's retention depth in programmatic-router.ts (removeOnComplete.count 1 -> 100, as router.ts already had), own the fallback poller's abort signal in waitForJobWithCancellation so it stops when any other outcome settles, and bound the upstream code_worker_id claim per trust entry via codeWorkerIdPrefixes so an external issuer cannot name another issuer's bridge worker in a multi-entry trust table. --- docs/fork/patches.md | 19 +++++++++ service/src/auth/librechat-jwt.test.ts | 40 +++++++++++++++++++ service/src/auth/librechat-jwt.ts | 45 +++++++++++++++++++++- service/src/job-cancellation.ts | 14 ++++++- service/src/queue-wait.test.ts | 10 +++-- service/src/queue-wait.ts | 31 ++++++++++----- service/src/service/programmatic-router.ts | 20 ++++++---- 7 files changed, 155 insertions(+), 24 deletions(-) diff --git a/docs/fork/patches.md b/docs/fork/patches.md index a9ec8660..94ca3a10 100644 --- a/docs/fork/patches.md +++ b/docs/fork/patches.md @@ -285,6 +285,16 @@ Source and current-upstream evidence: - Upstream `2c7fb8fcd7113f0f78b2085e80adf651ea4e5359` adds execution-profile queues and `jobCompletionWaitTimeoutMs`, but still waits only on QueueEvents. +- Upstream `v1.1.0` adds `waitForJobWithCancellation`. The merge composes the + fork poll race into it. The completion event settles from its own payload, as + BullMQ's `waitUntilFinished` does, so an evicted completed job is not reported + as an error; the composed poll owns an abort signal and stops once any other + outcome settles. +- Retention depth is part of this contract: a completed job must still exist in + Redis while the poll fallback can observe it. Both enqueue sites in + `service/src/service/programmatic-router.ts` keep `removeOnComplete.count=100` + (upstream ships `1`), matching `service/src/service/router.ts`. + Replay and drop condition: - Start from upstream routers and route their current timeout through @@ -408,6 +418,10 @@ Required behavior: `external:` namespace without embedding a consumer-specific source, and isolate their tenant storage namespaces by that validated source. +- When more than one trust entry shares the verifier, bound the upstream + `code_worker_id` claim per entry through an explicit prefix allowlist so an + external issuer cannot name another issuer's bridge worker. + Owned paths: - `docker-compose.yaml` @@ -426,6 +440,11 @@ Source and current-upstream evidence: baseline `83c4f7b105b6b3e69eda12701ad4ec437acba08f` retain only one effective issuer policy. +- Upstream `v1.1.0` adds the `code_worker_id` claim. The merge integrates it + into the fork trust table and adds `codeWorkerIdPrefixes`: a multi-entry + table must declare the prefixes its external entry may mint, while a + single-entry table stays unconstrained for backward compatibility. + Replay and drop condition: - Reapply the trust-table seam around the current upstream verifier rather than diff --git a/service/src/auth/librechat-jwt.test.ts b/service/src/auth/librechat-jwt.test.ts index 557f1e7a..ffbfe5fe 100644 --- a/service/src/auth/librechat-jwt.test.ts +++ b/service/src/auth/librechat-jwt.test.ts @@ -257,6 +257,7 @@ describe('LibreChat JWT auth provider', () => { audiences: ['partner-codeapi'], keyIds: ['partner-kid'], principalSources: ['external:partner'], + codeWorkerIdPrefixes: ['partner-'], }), ]); @@ -265,12 +266,25 @@ describe('LibreChat JWT auth provider', () => { iss: 'partner', aud: 'partner-codeapi', principal_source: 'external:partner', + code_worker_id: 'partner-worker-1', }); const partnerPrincipal = verifyLibreChatJwt( signJwt(partnerClaims, { kid: 'partner-kid' }, partner.privateKey), ); expect(partnerPrincipal.principalSource).toBe('external:partner'); expect(partnerPrincipal.tenantId).toBe('external:partner:tenant_abc'); + expect(partnerPrincipal.codeWorkerId).toBe('partner-worker-1'); + + // A worker ID outside the entry's declared prefixes must not be accepted, + // so an external issuer cannot name another issuer's bridge worker. + expectJwtReason( + signJwt( + { ...partnerClaims, code_worker_id: 'code-user_123' }, + { kid: 'partner-kid' }, + partner.privateKey, + ), + 'malformed_claims', + ); expectJwtReason(signJwt(partnerClaims), 'unknown_kid'); expectJwtReason( @@ -316,6 +330,32 @@ describe('LibreChat JWT auth provider', () => { expectJwtReason(signJwt(baseClaims()), 'config'); }); + test('requires an external trust entry to bound code_worker_id when multiple entries share the verifier', () => { + const partner = generateKeyPairSync('ed25519'); + const partnerJwk = partner.publicKey.export({ format: 'jwk' }); + process.env.CODEAPI_JWT_JWKS_JSON = JSON.stringify({ + keys: [ + { ...publicJwk, kid: 'test-kid', alg: 'EdDSA' }, + { ...partnerJwk, kid: 'partner-kid', alg: 'EdDSA' }, + ], + }); + const partnerEntry = trustEntry({ + issuer: 'partner', + audiences: ['partner-codeapi'], + keyIds: ['partner-kid'], + principalSources: ['external:partner'], + }); + + setModernTrustEntries([trustEntry(), partnerEntry]); + expectJwtReason(signJwt(baseClaims()), 'config'); + + process.env.CODEAPI_JWT_JWKS_JSON = JSON.stringify({ + keys: [{ ...publicJwk, kid: 'test-kid', alg: 'EdDSA' }], + }); + // A single-entry table has no cross-issuer ambiguity and stays unconstrained. + setModernTrustEntries([trustEntry()]); + expect(verifyLibreChatJwt(signJwt(baseClaims())).codeWorkerId).toBe('code-user_123'); + }); test('rejects duplicate key IDs across verification key sources', () => { process.env.CODEAPI_JWT_PUBLIC_KEY = JSON.stringify(publicJwk); process.env.CODEAPI_JWT_KID = 'test-kid'; diff --git a/service/src/auth/librechat-jwt.ts b/service/src/auth/librechat-jwt.ts index 78819979..79d6ff5c 100644 --- a/service/src/auth/librechat-jwt.ts +++ b/service/src/auth/librechat-jwt.ts @@ -54,6 +54,7 @@ interface JwtTrustEntry { keyIds: Set; allowedAlgs: Set; principalSources: Set; + codeWorkerIdPrefixes: Set; } interface VerificationConfig { @@ -93,6 +94,7 @@ const TRUST_ENTRY_FIELDS = new Set([ 'keyIds', 'allowedAlgorithms', 'principalSources', + 'codeWorkerIdPrefixes', ]); function base64UrlDecode(value: string): Buffer { @@ -367,6 +369,9 @@ function parseModernTrustEntries(keys: Map, raw: string) record.principalSources, `JWT trust entry ${index} principalSources`, ); + const workerIdPrefixes = record.codeWorkerIdPrefixes === undefined + ? [] + : assertUniqueStrings(record.codeWorkerIdPrefixes, `JWT trust entry ${index} codeWorkerIdPrefixes`); if (!algorithmValues.every((value): value is JwtAlg => SUPPORTED_ALGORITHMS.has(value as JwtAlg))) { throw new CodeApiJwtAuthError('config', `JWT trust entry ${index} has an unsupported algorithm`); } @@ -385,6 +390,18 @@ function parseModernTrustEntries(keys: Map, raw: string) } assignedExternalSources.add(source); } + // An external issuer mints its own `code_worker_id`. When more than one + // trust entry can verify tokens, the claim must be bounded per entry or an + // external issuer could name another issuer's bridge worker and route + // workspace-tool commands to it. A single-entry table has no such ambiguity + // and stays unconstrained for backward compatibility. + const hasExternalSource = sourceValues.some(source => source.startsWith('external:')); + if (parsed.length > 1 && hasExternalSource && workerIdPrefixes.length === 0) { + throw new CodeApiJwtAuthError( + 'config', + `JWT trust entry ${index} must declare codeWorkerIdPrefixes for its external principal source`, + ); + } const allowedAlgs = new Set(algorithmValues); for (const keyId of keyIds) { if (assignedKeyIds.has(keyId)) { @@ -409,6 +426,7 @@ function parseModernTrustEntries(keys: Map, raw: string) keyIds: new Set(keyIds), allowedAlgs, principalSources: new Set(sourceValues), + codeWorkerIdPrefixes: new Set(workerIdPrefixes), }); } @@ -434,6 +452,7 @@ function buildTrustEntries(keys: Map): Map(['librechat_jwt', 'openid_reuse']), + codeWorkerIdPrefixes: new Set(), }], ]); } @@ -594,6 +613,27 @@ function tenantNamespace(tenantId: string, principalSource: JwtPrincipalSource): : tenantId; } +function boundCodeWorkerId( + codeWorkerId: string | undefined, + trustEntry: JwtTrustEntry, +): string | undefined { + if (codeWorkerId === undefined) { + return undefined; + } + if (trustEntry.codeWorkerIdPrefixes.size === 0) { + return codeWorkerId; + } + for (const prefix of trustEntry.codeWorkerIdPrefixes) { + if (codeWorkerId.startsWith(prefix)) { + return codeWorkerId; + } + } + throw new CodeApiJwtAuthError( + 'malformed_claims', + 'code_worker_id is not permitted for this issuer', + ); +} + function validateClaims( claims: LibreChatJwtClaims, config: VerificationConfig, @@ -606,7 +646,10 @@ function validateClaims( const nbf = assertNumericDate(claims.nbf, 'nbf'); const exp = assertNumericDate(claims.exp, 'exp'); const planId = optionalString(claims.plan_id, 'plan_id'); - const codeWorkerId = optionalString(claims.code_worker_id, 'code_worker_id'); + const codeWorkerId = boundCodeWorkerId( + optionalString(claims.code_worker_id, 'code_worker_id'), + trustEntry, + ); const principalSource = assertPrincipalSource(claims.principal_source, trustEntry.principalSources); const tenantId = tenantNamespace(resolveTenantIdClaim(claims.tenant_id), principalSource); const authContextHash = assertString(claims.auth_context_hash, 'auth_context_hash'); diff --git a/service/src/job-cancellation.ts b/service/src/job-cancellation.ts index 465c0188..57431011 100644 --- a/service/src/job-cancellation.ts +++ b/service/src/job-cancellation.ts @@ -566,7 +566,7 @@ export async function waitForJobWithCancellation(args: { /** UZH fork: additive BullMQ job-state polling. Resolves when polling observes * a terminal state that a missed or lagged QueueEvents event would otherwise * hide. It NEVER rejects, so it cannot change upstream failure semantics. */ - fallbackCompletion?: Promise; + fallbackCompletion?: (signal: AbortSignal) => Promise; }): Promise { const { commands, @@ -581,6 +581,12 @@ export async function waitForJobWithCancellation(args: { // Subscription startup can itself wait for Redis recovery. Own the losing // promise immediately, before any await, rather than after registration. void completion.catch(() => undefined); + // UZH fork: the additive polling fallback is owned here so that its signal is + // aborted once any other outcome settles. Without that, an evicted or + // cancelled job would leave the poller reading Redis for the whole timeout. + const fallbackAbortController = new AbortController(); + const fallbackCompletion = args.fallbackCompletion?.(fallbackAbortController.signal); + void fallbackCompletion?.catch(() => undefined); const target = { queueName: job.queueName, jobId: String(job.id) }; const deadlineAtMs = args.deadlineAtMs ?? Date.now() + timeoutMs; let fencing: Promise> | undefined; @@ -596,6 +602,7 @@ export async function waitForJobWithCancellation(args: { await registry.register(target, externalController); } catch (error) { void completion.catch(() => undefined); + fallbackAbortController.abort(); const outcome = await fence(); if (outcome.status === 'completed') return outcome.result; await removeJobIfWaiting(job).catch(() => false); @@ -648,7 +655,7 @@ export async function waitForJobWithCancellation(args: { // UZH fork: the polling fallback is additive. On failure it parks on a // never-settling promise so upstream's completion, cancellation, and timeout // outcomes remain authoritative. - const fallback = args.fallbackCompletion?.then( + const fallback = fallbackCompletion?.then( value => value, () => new Promise(() => {}), ); @@ -664,6 +671,9 @@ export async function waitForJobWithCancellation(args: { throw error; } finally { removeAbortListener(); + // Stop the additive poller as soon as any outcome wins, so it never keeps + // reading Redis after completion, cancellation, or timeout. + fallbackAbortController.abort(); await registry .unregister(target, externalController) .catch(() => undefined); diff --git a/service/src/queue-wait.test.ts b/service/src/queue-wait.test.ts index 070dd7fe..c5939814 100644 --- a/service/src/queue-wait.test.ts +++ b/service/src/queue-wait.test.ts @@ -33,9 +33,9 @@ describe('waitForJobFinished', () => { test('uses the QueueEvents result when it arrives first', async () => { const result = { ok: true, source: 'events' } satisfies TestResult; const job = fakeJob({}); - const queue = fakeQueue(async () => ({ - returnvalue: result, - } as Job)); + // The job is already evicted, so a re-read would fail; the event payload + // must be authoritative. + const queue = fakeQueue(async () => undefined); const queueEvents = fakeQueueEvents(); const waitForResult = waitForJobFinished(job, queue, queueEvents, 1000); queueEvents.emit('completed', { jobId: 'job-1', returnvalue: result }); @@ -66,7 +66,9 @@ describe('waitForJobFinished', () => { await expect(waitForResult).resolves.toEqual(result); await wait(300); - expect(getJobCalls).toBe(2); + // The completion settles from the event payload, so polling stops after the + // single in-flight read instead of issuing a re-fetch. + expect(getJobCalls).toBe(1); }); test('polls completed jobs when QueueEvents do not arrive', async () => { diff --git a/service/src/queue-wait.ts b/service/src/queue-wait.ts index 3dfc3f64..bbba235b 100644 --- a/service/src/queue-wait.ts +++ b/service/src/queue-wait.ts @@ -31,7 +31,6 @@ function throwIfAborted(jobId: string, signal?: AbortSignal): void { function waitForJobEvent( jobId: string, - queue: Queue, queueEvents: QueueEvents, timeoutMs: number, signal?: AbortSignal, @@ -63,17 +62,16 @@ function waitForJobEvent( void promise.then(resolve, reject); }; - const onCompleted = (event: { jobId: string }) => { + const onCompleted = (event: { jobId: string; returnvalue?: TReturn }) => { if (event.jobId !== jobId) { return; } - settleWith((async () => { - const currentJob = await queue.getJob(jobId); - if (!currentJob) { - throw new Error(`Job ${jobId} no longer exists after completion event`); - } - return currentJob.returnvalue; - })()); + // Resolve from the event payload, as BullMQ's own `waitUntilFinished` + // does, rather than re-reading the job from Redis. A completed job may + // already have been evicted by the retention policy, which would turn a + // successful execution into an error even though the event carries the + // result. + settleWith(Promise.resolve(event.returnvalue as TReturn)); }; const onFailed = (event: { jobId: string; failedReason?: string }) => { if (event.jobId === jobId) { @@ -164,7 +162,6 @@ async function waitForJobFinished( const eventWait = waitForJobEvent( jobId, - queue, queueEvents, timeoutMs, eventAbortController.signal, @@ -178,6 +175,20 @@ async function waitForJobFinished( eventWait, pollWait, ]); + } catch (error) { + // A poll failure must not preempt a completion event that is already in + // flight: the event carries the result, while the poll can only observe the + // job record, which the retention policy may have evicted. Give the event + // channel one poll interval to deliver before honoring the poll error, so a + // successful execution is never reported as a missing job. + const grace = await Promise.race([ + eventWait.then(value => ({ settled: true as const, value }), () => ({ settled: false as const })), + wait(JOB_RESULT_POLL_INTERVAL_MS).then(() => ({ settled: false as const })), + ]); + if (grace.settled) { + return grace.value; + } + throw error; } finally { pollAbortController.abort(); eventAbortController.abort(); diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index f5fe45c7..b16c59b9 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -453,7 +453,9 @@ async function runReplayIteration( egressGrantToken: sandboxSecurity.egressGrantToken, }, { - removeOnComplete: { age: 60, count: 1 }, + // Retention must stay deeper than the poll fallback's reach: the + // fallback can only recover a completed job that is still in Redis. + removeOnComplete: { age: 60, count: 100 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, jobId: cancellationTarget.jobId, @@ -484,11 +486,13 @@ async function runReplayIteration( signal, // UZH fork: recover completion when QueueEvents lag, without weakening the // upstream cancellation fence or timeout authority. - fallbackCompletion: pollJobUntilFinished( - job as never, - queue as never, - JOB_COMPLETION_WAIT_TIMEOUT_MS, - ) as Promise, + fallbackCompletion: signal => + pollJobUntilFinished( + job as never, + queue as never, + JOB_COMPLETION_WAIT_TIMEOUT_MS, + signal, + ) as Promise, }); } @@ -1826,7 +1830,9 @@ async function handleBlocking( egressGrantToken: sandboxSecurity.egressGrantToken, }, { - removeOnComplete: { age: 60, count: 1 }, + // Retention must stay deeper than the poll fallback's reach: the + // fallback can only recover a completed job that is still in Redis. + removeOnComplete: { age: 60, count: 100 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, jobId: session_id, From 5e49e0d495e5441a7d6aa185d77251ed90399e47 Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 18:11:24 +0200 Subject: [PATCH 112/116] docs(project): record v1.1.0 integration progress and ignore local review reports Keeps plan-hardening and review reports under docs/project/_local/ out of history, and records the executed slices with their verification receipts. --- .gitignore | 3 +++ ...2026-09-17-upstream-v1.1.0-integration-plan.md | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 958b3332..ccf9d4eb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ helm/*/Chart.lock # Local sandbox runtime data (docker volume mount) data/ +# Local review and plan-hardening reports, never committed +docs/project/_local/ + # Editor config .vscode/ diff --git a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md index 1b33311e..5a42380e 100644 --- a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md +++ b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md @@ -123,3 +123,18 @@ Separately gated (asked at the end, not now): merge into main; closing dependabo - 2026-09-17: three-way analysis complete; trial merge mapped 8 conflicts; all ten fork behaviors verified still-needed against v1.1.0 (patch 7 narrowed per external review); plan drafted. - 2026-09-17: planner gate BLOCKED - native planner route (gpt-6-astra) failed terminally on an account usage limit; one generic-continuity GLM fallback launched but did not return within the wait. Plan presented with the gate recorded as blocked, not passed. +- 2026-09-17: S1-S7 executed in the writable clone /tmp/ci-reconcile-1 (primary .git read-only). +- 2026-09-17: S1-S7 executed in the writable clone /tmp/ci-reconcile-1 + (primary .git read-only). + - S1 merge `680df07`: 8 conflicts resolved; an accidental `api/.build` + bundle sweep was caught by the S6 diff check and removed before push. + - S6 ledger `967e622`. + - Verification green at the head: service 1076 pass / 0 fail, api 481 + pass / 0 fail, helm lint and renders, deployment-config tests, + compose/release tests, spec-guard CI compile and smoke (ubuntu + container), docker buildx --check on api and worker-sandbox. The + packages/code failures and the kvm_guest_dns failure are byte-identical + to pristine v1.1.0, so they are inherited, not regressions. + - S7 final review (Claude Opus 5, trusted read-only) returned + PASS_WITH_CONCERNS with three required fixes; all three are addressed in + `53ed390` with covering tests. Report at docs/project/_local/reviews/. From 598793ac2d97dc2478f3cbed68f06b9d69d26b97 Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 18:15:13 +0200 Subject: [PATCH 113/116] docs(project): record draft PR #24 and remaining gated dispositions --- .../2026-09-17-upstream-v1.1.0-integration-plan.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md index 5a42380e..4c57685e 100644 --- a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md +++ b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md @@ -138,3 +138,11 @@ Separately gated (asked at the end, not now): merge into main; closing dependabo - S7 final review (Claude Opus 5, trusted read-only) returned PASS_WITH_CONCERNS with three required fixes; all three are addressed in `53ed390` with covering tests. Report at docs/project/_local/reviews/. +- 2026-09-17: draft PR #24 opened (chore(reconcile): integrate upstream + v1.1.0 with fork behaviors retained) at + https://github.com/uzh-bf/code-interpreter/pull/24; CI run 35245379060 + queued on head 5e49e0d. This is the terminal condition of the package. +- 2026-09-17: remaining dispositions are separately gated, not executed: + merge into main; close dependabot PRs #4/#14/#16/#22; PR #21 re-derivation; + image publication; GitOps/deployment promotion; deleting this branch or its + worktree. From a0ebd3c0bd1a1d8cf5fd9b367d80fec90c49e5da Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 19:49:11 +0200 Subject: [PATCH 114/116] fix(reconcile): close confirmation-review findings on the v1.1.0 merge The focused confirmation review of 53ed390 confirmed the three required fixes but found one regression and two unpinned behaviors: - queue-wait.ts declared the QueueEvents payload with the typed result, which introduced two TS2345 errors against BullMQ's string declaration. Declare the field as BullMQ does and convert through unknown instead. - The fallback poller's abort contract had no test coverage: removing either abort call left the suite green. Cover the pending/aborted lifecycle, the registration-failure abort, and the additive rejection contract. - codeWorkerIdPrefixes had no cross-entry uniqueness check, so two external entries could declare duplicate or nested prefixes and each admit the other's bridge workers. Reject overlapping prefixes at config time, mirroring the existing keyId and external-source checks. - Rename the fallback poller's parameter so it no longer shadows the client-disconnect signal in programmatic-router.ts. --- service/src/auth/librechat-jwt.test.ts | 40 ++++++++ service/src/auth/librechat-jwt.ts | 16 +++ service/src/job-cancellation.test.ts | 111 +++++++++++++++++++++ service/src/queue-wait.ts | 8 +- service/src/service/programmatic-router.ts | 4 +- 5 files changed, 174 insertions(+), 5 deletions(-) diff --git a/service/src/auth/librechat-jwt.test.ts b/service/src/auth/librechat-jwt.test.ts index ffbfe5fe..febc77c1 100644 --- a/service/src/auth/librechat-jwt.test.ts +++ b/service/src/auth/librechat-jwt.test.ts @@ -356,6 +356,46 @@ describe('LibreChat JWT auth provider', () => { setModernTrustEntries([trustEntry()]); expect(verifyLibreChatJwt(signJwt(baseClaims())).codeWorkerId).toBe('code-user_123'); }); + + test('rejects overlapping code worker ID prefixes across trust entries', () => { + const partner = generateKeyPairSync('ed25519'); + const partnerJwk = partner.publicKey.export({ format: 'jwk' }); + const second = generateKeyPairSync('ed25519'); + const secondJwk = second.publicKey.export({ format: 'jwk' }); + process.env.CODEAPI_JWT_JWKS_JSON = JSON.stringify({ + keys: [ + { ...publicJwk, kid: 'test-kid', alg: 'EdDSA' }, + { ...partnerJwk, kid: 'partner-kid', alg: 'EdDSA' }, + { ...secondJwk, kid: 'second-kid', alg: 'EdDSA' }, + ], + }); + const partnerEntry = trustEntry({ + issuer: 'partner', + audiences: ['partner-codeapi'], + keyIds: ['partner-kid'], + principalSources: ['external:partner'], + codeWorkerIdPrefixes: ['partner-'], + }); + const secondEntry = (prefix: string): Record => + trustEntry({ + issuer: 'second', + audiences: ['second-codeapi'], + keyIds: ['second-kid'], + principalSources: ['external:second'], + codeWorkerIdPrefixes: [prefix], + }); + + // Duplicate and nested prefixes let each external entry mint a worker ID + // the other entry also accepts, so the table is rejected at config time. + for (const prefix of ['partner-', 'partner-x-']) { + setModernTrustEntries([trustEntry(), partnerEntry, secondEntry(prefix)]); + expectJwtReason(signJwt(baseClaims()), 'config'); + } + + setModernTrustEntries([trustEntry(), partnerEntry, secondEntry('second-')]); + expect(verifyLibreChatJwt(signJwt(baseClaims())).principalSource).toBe('openid_reuse'); + }); + test('rejects duplicate key IDs across verification key sources', () => { process.env.CODEAPI_JWT_PUBLIC_KEY = JSON.stringify(publicJwk); process.env.CODEAPI_JWT_KID = 'test-kid'; diff --git a/service/src/auth/librechat-jwt.ts b/service/src/auth/librechat-jwt.ts index 79d6ff5c..ee75ecb8 100644 --- a/service/src/auth/librechat-jwt.ts +++ b/service/src/auth/librechat-jwt.ts @@ -345,6 +345,7 @@ function parseModernTrustEntries(keys: Map, raw: string) const entries = new Map(); const assignedKeyIds = new Set(); const assignedExternalSources = new Set(); + const assignedWorkerIdPrefixes = new Set(); for (const [index, value] of parsed.entries()) { if (value === null || typeof value !== 'object' || Array.isArray(value)) { throw new CodeApiJwtAuthError('config', `JWT trust entry ${index} must be an object`); @@ -402,6 +403,21 @@ function parseModernTrustEntries(keys: Map, raw: string) `JWT trust entry ${index} must declare codeWorkerIdPrefixes for its external principal source`, ); } + // Two entries that accept the same worker ID would each admit the other's + // bridge workers, which is what the per-entry bound exists to prevent. The + // prefixes must therefore be disjoint across entries. + for (const prefix of workerIdPrefixes) { + const overlapping = [...assignedWorkerIdPrefixes].find( + assigned => assigned.startsWith(prefix) || prefix.startsWith(assigned), + ); + if (overlapping !== undefined) { + throw new CodeApiJwtAuthError( + 'config', + `codeWorkerIdPrefixes must be disjoint across trust entries: ${prefix} overlaps ${overlapping}`, + ); + } + assignedWorkerIdPrefixes.add(prefix); + } const allowedAlgs = new Set(algorithmValues); for (const keyId of keyIds) { if (assignedKeyIds.has(keyId)) { diff --git a/service/src/job-cancellation.test.ts b/service/src/job-cancellation.test.ts index e23186e4..59f7c9af 100644 --- a/service/src/job-cancellation.test.ts +++ b/service/src/job-cancellation.test.ts @@ -412,6 +412,117 @@ test('registration failure fences and removes the already-enqueued job', async ( await registry.close(); }); +test('owns the additive fallback poller and stops it once completion wins', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + let release!: (value: string) => void; + const job = { + id: 'fallback-lifecycle', + queueName: 'other', + waitUntilFinished: () => + new Promise(resolve => { + release = resolve; + }), + getState: async () => 'active', + remove: async () => {}, + } as unknown as Job; + let fallbackSignal: AbortSignal | undefined; + let pollerStopped = false; + + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + fallbackCompletion: signal => { + fallbackSignal = signal; + return new Promise(resolve => { + signal.addEventListener( + 'abort', + () => { + pollerStopped = true; + resolve('polled'); + }, + { once: true }, + ); + }); + }, + }); + await new Promise(resolve => setImmediate(resolve)); + // The poller runs while the wait is pending and must stop as soon as any + // other outcome settles, so it never keeps reading Redis for the full + // timeout after completion, cancellation, or timeout. + expect(fallbackSignal?.aborted).toBe(false); + expect(pollerStopped).toBe(false); + + release('completed'); + await expect(waiting).resolves.toBe('completed'); + expect(fallbackSignal?.aborted).toBe(true); + expect(pollerStopped).toBe(true); + await registry.close(); +}); + +test('aborts the fallback poller when subscription registration fails', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + const registry = new JobCancellationRegistry(redis(fake)); + const job = { + id: 'fallback-register-failure', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => {}, + } as unknown as Job; + let fallbackSignal: AbortSignal | undefined; + + await expect( + waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + fallbackCompletion: signal => { + fallbackSignal = signal; + return new Promise(() => {}); + }, + }), + ).rejects.toThrow('subscriber unavailable'); + + expect(fallbackSignal?.aborted).toBe(true); + await registry.close(); +}); + +test('a rejecting fallback poller cannot override the completion outcome', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const job = { + id: 'fallback-rejection', + queueName: 'other', + waitUntilFinished: () => Promise.resolve('completed'), + getState: async () => 'active', + remove: async () => {}, + } as unknown as Job; + + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + fallbackCompletion: () => Promise.reject(new Error('poller failed')), + }); + + // The poller is additive: upstream completion, cancellation, and timeout + // remain the only outcomes that can settle the wait. + await expect(waiting).resolves.toBe('completed'); + await registry.close(); +}); + test('a result rejection is owned while subscription registration is pending', async () => { const fake = new FakeRedis(); let release!: () => void; diff --git a/service/src/queue-wait.ts b/service/src/queue-wait.ts index bbba235b..7c1e08cc 100644 --- a/service/src/queue-wait.ts +++ b/service/src/queue-wait.ts @@ -62,7 +62,7 @@ function waitForJobEvent( void promise.then(resolve, reject); }; - const onCompleted = (event: { jobId: string; returnvalue?: TReturn }) => { + const onCompleted = (event: { jobId: string; returnvalue?: string }) => { if (event.jobId !== jobId) { return; } @@ -70,8 +70,10 @@ function waitForJobEvent( // does, rather than re-reading the job from Redis. A completed job may // already have been evicted by the retention policy, which would turn a // successful execution into an error even though the event carries the - // result. - settleWith(Promise.resolve(event.returnvalue as TReturn)); + // result. BullMQ declares the payload field as a string and parses it + // back into the typed result before emitting, so the cast goes through + // `unknown`. + settleWith(Promise.resolve(event.returnvalue as unknown as TReturn)); }; const onFailed = (event: { jobId: string; failedReason?: string }) => { if (event.jobId === jobId) { diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index b16c59b9..2c42ad87 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -486,12 +486,12 @@ async function runReplayIteration( signal, // UZH fork: recover completion when QueueEvents lag, without weakening the // upstream cancellation fence or timeout authority. - fallbackCompletion: signal => + fallbackCompletion: fallbackSignal => pollJobUntilFinished( job as never, queue as never, JOB_COMPLETION_WAIT_TIMEOUT_MS, - signal, + fallbackSignal, ) as Promise, }); } From 4f610b7205c46d96f0dce259046444da7af46f48 Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 19:49:53 +0200 Subject: [PATCH 115/116] docs(project): record the confirmation review and its fix commit --- ...-09-17-upstream-v1.1.0-integration-plan.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md index 4c57685e..7b8d7e8d 100644 --- a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md +++ b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md @@ -140,8 +140,23 @@ Separately gated (asked at the end, not now): merge into main; closing dependabo `53ed390` with covering tests. Report at docs/project/_local/reviews/. - 2026-09-17: draft PR #24 opened (chore(reconcile): integrate upstream v1.1.0 with fork behaviors retained) at - https://github.com/uzh-bf/code-interpreter/pull/24; CI run 35245379060 - queued on head 5e49e0d. This is the terminal condition of the package. + https://github.com/uzh-bf/code-interpreter/pull/24. CI run 35245439468 is + green on head 598793a: all ten checks SUCCESS and mergeStateStatus CLEAN. +- 2026-09-17: focused confirmation review of the fix commit `53ed390` + (Claude Opus 5, trusted read-only) returned CONFIRMED_WITH_CONCERNS: + fixes 1 and 3 correct, fix 2 correct but unpinned, plus a TS2345 + regression introduced by the fix commit itself. All actionable findings + are closed in `a0ebd3c` (typed BullMQ payload, fallback poller lifecycle + tests, disjoint codeWorkerIdPrefixes, renamed shadowed parameter), each + pinned by a mutation check. Report at + docs/project/_local/reviews/2026-09-17-upstream-v1.1.0-integration-confirmation-review.md. +- 2026-09-17: accepted follow-ups recorded, not slices: a multi-entry trust + table whose entries declare no `external:` source still leaves + `code_worker_id` unbounded (same class as the inherited internal-source + finding; broadening the gate would invalidate existing multi-issuer configs + at startup), the fixed 250 ms completion grace window, Redis TLS + `rejectUnauthorized: false`, and the +32 uncommitted lines in the + `trees/source-sans-pro-fonts` worktree. - 2026-09-17: remaining dispositions are separately gated, not executed: merge into main; close dependabot PRs #4/#14/#16/#22; PR #21 re-derivation; image publication; GitOps/deployment promotion; deleting this branch or its From 277bea906825359d8d60d41d33c05ad598b37e98 Mon Sep 17 00:00:00 2001 From: trial Date: Thu, 17 Sep 2026 20:29:16 +0200 Subject: [PATCH 116/116] docs(project): mark the v1.1.0 integration plan executed --- .../2026-09-17-upstream-v1.1.0-integration-plan.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md index 7b8d7e8d..543cd194 100644 --- a/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md +++ b/docs/project/2026-09-17-upstream-v1.1.0-integration-plan.md @@ -1,6 +1,7 @@ # Upstream v1.1.0 integration plan -Status: draft +Status: executed — terminal condition met (green-CI draft PR #24 on the task +branch, ledger updated, final review and its confirmation pass closed) Package: full path (security, architecture, deployment seams) Branch (planned): chore/reconcile-upstream-v1.1.0 Target: uzh-bf/code-interpreter main @@ -157,6 +158,11 @@ Separately gated (asked at the end, not now): merge into main; closing dependabo at startup), the fixed 250 ms completion grace window, Redis TLS `rejectUnauthorized: false`, and the +32 uncommitted lines in the `trees/source-sans-pro-fonts` worktree. +- 2026-09-17: terminal condition met. Draft PR #24 reports all ten checks + SUCCESS with mergeStateStatus CLEAN at the branch head (see the PR's checks + for the exact head; the last recorded run is 35255121627 on `4f610b7`). No + further work is authorized in this package; every remaining action is a + separately gated decision listed above. - 2026-09-17: remaining dispositions are separately gated, not executed: merge into main; close dependabot PRs #4/#14/#16/#22; PR #21 re-derivation; image publication; GitOps/deployment promotion; deleting this branch or its