fix: Retain Quarantine After Failed Environment Setup

.github/workflows/ci.yml

@@ -196,6 +196,10 @@ jobs:
       - run: npm run build
       - name: Native ACL and credential lifecycle tests
         run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js
+      - name: Native environment setup lifecycle tests
+        env:
+          LIBRECHAT_CODE_LIVE_SRT_TESTS: '1'
+        run: node --test dist/environment-live.test.js
 
   lambda-microvm-provisioning:
     name: Lambda MicroVM Provisioning

packages/code/README.md

@@ -689,9 +689,14 @@ worker runs. This inspection happens at startup, not on the command hot path.
 Setup is an operator-authorized startup command under the configured native sandbox
 policy. It requires commands to be enabled, runs once per worker startup before
 registration, and must be idempotent for restarts. Its timeout is bounded to five
-minutes and captured output to 8 KiB. Setup failure prevents registration. A crash
-or uncertain termination retains the existing workspace quarantine marker; inspect
-the workspace before clearing quarantine. No setup output is sent to the model.
+minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero
+exit, timeout, crash or uncertain termination retains the workspace quarantine marker;
+inspect the workspace before running `librechat-code clear-workspace-quarantine
+--worker-dir <environment-root> --workspace-id <environment-name>` with the same
+deployment and identity configuration. Only use the separate
+`--reset-workspace-quarantine <environment-name>` run option afterward if a server
+fence also needs clearing. Only successful setup automatically clears its marker.
+No setup output is sent to the model.
 
 Named actions are fixed commands without model-supplied substitution. The bridge
 advertises only their names and the definition fingerprint, never their shell source

packages/code/src/cli.ts

@@ -1012,12 +1012,12 @@ async function run(
                 },
                 controller.signal,
             );
-            await guard.clear('setup');
             if (result.exitCode !== 0 || result.timedOut) {
                 throw new Error(
-                    `Environment ${id} setup failed; inspect the setup command before restarting`,
+                    `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`,
                 );
             }
+            await guard.clear('setup');
             process.stdout.write(
                 `librechat-code: environment ${id} prepared\n`,
             );

packages/code/src/environment-live.test.ts

@@ -8,13 +8,14 @@ import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import test from 'node:test';
 
-for (const { succeeds, reset } of [
-    { succeeds: true, reset: false },
-    { succeeds: false, reset: false },
-    { succeeds: true, reset: true },
+for (const { succeeds, reset, timesOut } of [
+    { succeeds: true, reset: false, timesOut: false },
+    { succeeds: false, reset: false, timesOut: false },
+    { succeeds: false, reset: false, timesOut: true },
+    { succeeds: true, reset: true, timesOut: false },
 ]) {
     test(
-        `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`,
+        `real CLI environment setup gates registration (success=${succeeds}, reset=${reset}, timeout=${timesOut})`,
         {
             skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1',
             timeout: 20_000,
@@ -27,7 +28,7 @@ for (const { succeeds, reset } of [
             const path = join(directory, 'environment.yaml');
             await writeFile(
                 path,
-                `name: project\nroot: project\nsetup:\n  command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n  timeoutMs: 5000\n`,
+                `name: project\nroot: project\nsetup:\n  command: 'printf prepared >> prepared.txt; ${timesOut ? 'sleep 10' : `exit ${succeeds ? 0 : 2}`}'\n  timeoutMs: ${timesOut ? 1000 : 5000}\n`,
             );
             let registrations = 0;
             let receive: (() => void) | undefined;
@@ -38,6 +39,7 @@ for (const { succeeds, reset } of [
                 request.resume();
                 if (request.url?.endsWith('/register')) {
                     registrations++;
+                    await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' });
                     if (reset)
                         await assert.rejects(
                             readFile(join(root, 'prepared.txt')),
@@ -61,17 +63,23 @@ for (const { succeeds, reset } of [
             });
             const address = server.address();
             assert.ok(address && typeof address !== 'string');
-            const child = spawn(
+            const start = (clear = false) => spawn(
                 process.execPath,
                 [
                     fileURLToPath(new URL('./cli.js', import.meta.url)),
+                    ...(clear ? [
+                        'clear-workspace-quarantine',
+                        '--worker-dir', root,
+                        '--workspace-id', 'project',
+                    ] : [
                     'run',
                     '--environment',
                     path,
                     '--allow-workspace-commands',
                     ...(reset
                         ? ['--reset-workspace-quarantine', 'project']
                         : []),
+                    ]),
                 ],
                 {
                     env: {
@@ -90,6 +98,7 @@ for (const { succeeds, reset } of [
                     stdio: ['ignore', 'pipe', 'pipe'],
                 },
             );
+            const child = start();
             const exited = once(child, 'exit');
             t.after(() => child.kill('SIGKILL'));
             let stderr = '';
@@ -111,6 +120,26 @@ for (const { succeeds, reset } of [
                 assert.notEqual(code, 0);
                 assert.match(stderr, /Environment project setup failed/);
                 assert.equal(registrations, 0);
+                const marker = await readFile(join(directory, 'quarantine.json'), 'utf8');
+                assert.equal(JSON.parse(marker).workspaceId, 'project');
+                const before = await readFile(join(root, 'prepared.txt'), 'utf8');
+                const retry = start();
+                t.after(() => retry.kill('SIGKILL'));
+                let retryStderr = '';
+                retry.stderr.on('data', chunk => { retryStderr += chunk.toString(); });
+                const [retryCode] = await once(retry, 'exit');
+                assert.notEqual(retryCode, 0);
+                assert.match(retryStderr, /quarantined/);
+                assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before);
+                assert.equal(await readFile(join(directory, 'quarantine.json'), 'utf8'), marker);
+                assert.equal(registrations, 0);
+                const recovery = start(true);
+                t.after(() => recovery.kill('SIGKILL'));
+                const [recoveryCode] = await once(recovery, 'exit');
+                assert.equal(recoveryCode, 0);
+                await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' });
+                assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before);
+                assert.equal(registrations, 0);
             }
         },
     );