Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ export async function handleFatal(kind: CrashKind, error: unknown): Promise<void
// would block process.exit indefinitely, defeating this handler's one job.
async function finalizeActiveRunOnCrash(error: unknown): Promise<void> {
const run = getActiveRun();
if (run === null || !run.active) return;
if (run === null) return;
const message = error instanceof Error ? error.message : String(error);
try {
await saveCrashState(run.cwd, run.sessionId, {
Expand Down Expand Up @@ -209,10 +209,12 @@ export function installCrashHandlers(): void {
// Mirrors finalizeActiveRunOnCrash but is not itself a crash — a signal is a
// clean, externally-requested termination (operator, shell, orchestrator),
// so the run is left "failed" (interrupted) rather than "crashed", and no
// crash report is written for it.
// crash report is written for it. Callers must markCrashed() before this so
// chained saveState renames cannot clobber the terminal write (same contract
// as the uncaughtException path).
async function finalizeActiveRunOnSignal(signal: NodeJS.Signals): Promise<void> {
const run = getActiveRun();
if (run === null || !run.active) return;
if (run === null) return;
try {
await saveCrashState(run.cwd, run.sessionId, {
status: "failed",
Expand Down Expand Up @@ -270,6 +272,9 @@ export function installSignalHandlers(): void {
`host dispose failed handling ${signal}: ${disposeErr instanceof Error ? disposeErr.message : String(disposeErr)}\n`,
);
}
// Same fence as handleFatal: any snapshot still queued in writeChains must
// see isCrashed and step aside before saveCrashState renames run.json.
markCrashed();
void finalizeActiveRunOnSignal(signal).finally(() => {
process.exit(128 + SIGNAL_EXIT_NUMBER[signal]);
});
Expand Down
6 changes: 5 additions & 1 deletion src/session/active-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,14 @@
// crash path has the exact failure mode primeCrashReporting (src/crash/
// report.ts) exists to avoid for git: a stalled disk or network mount would
// block process.exit forever.
//
// Liveness has exactly one representation: presence of this handle in the
// module-level slot (see getActiveRun below). There is no separate "active"
// flag on the handle itself — a second field would just be a copy of the
// same fact, free to drift from the slot it's meant to describe.
export type RunStateHandle = {
sessionId: string;
cwd: string;
active: boolean;
task: string;
startedAt: number;
model?: string;
Expand Down
16 changes: 15 additions & 1 deletion src/session/state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ afterAll(() => {
mock.module("node:fs/promises", () => realFs);
});

const { loadState, saveState } = await import("./state.js");
const { finalizeRunState, loadState, saveState } = await import("./state.js");
const { getActiveRun, setActiveRun } = await import("./active-run.js");
type RunState = Awaited<ReturnType<typeof loadState>>;

let cwd = "";
Expand Down Expand Up @@ -72,6 +73,19 @@ test("a straggler snapshot started before a terminal write does not overwrite it
expect(final?.finishedAt).toBe(999);
});

test("a persisted terminal status agrees with the active-run handle without a second call site", async () => {
const sessionId = "sess-terminal";
setActiveRun({ sessionId, cwd, task: "task", startedAt: 1 });

await finalizeRunState(cwd, sessionId, state({ status: "done", finishedAt: 1000 }), home);

const persisted = await loadState(cwd, sessionId, home);
expect(persisted?.status).toBe("done");
// The only liveness representation left is presence in the active-run
// slot -- a terminal RunState.status must leave nothing there to read.
expect(getActiveRun()).toBeNull();
});

test("saveState calls for different sessions do not block each other", async () => {
await Promise.all([
saveState(cwd, "session-a", state({ task: "a" }), home),
Expand Down
34 changes: 33 additions & 1 deletion src/session/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { dirname, join } from "node:path";
import { type } from "arktype";

import { sessionDir } from "./index.js";
import { getTestWriteGate, isCrashed } from "./active-run.js";
import { clearActiveRun, getTestWriteGate, isCrashed } from "./active-run.js";
import { COMMAND_NAME } from "../branding.js";

const ConnectedMcpServerSchema = type({
Expand Down Expand Up @@ -108,6 +108,29 @@ export async function saveState(
return write;
}

// Single write path for a terminal RunState: pairs the on-disk status with
// clearing the in-memory active-run handle (active-run.ts) so the two facts
// are set together instead of at two independent call sites that could drift.
// Callers writing a non-terminal ("running") snapshot should call saveState
// directly — clearing the active-run handle on a running snapshot would be
// wrong, not merely redundant.
//
// The clear happens before the saveState await, not after: this run is
// closing out regardless of whether the write below succeeds, and a signal
// or uncaught exception landing during that await must see the handle
// already gone, or it races a second "crashed" write (src/index.ts's process
// handlers, via saveCrashState) against the terminal write in flight here.
// Clearing after the await leaves that exact window open on every terminal
// write, not only the crash path's own.
export async function finalizeRunState(
cwd: string,
sessionId: string,
state: RunState,
home?: string,
): Promise<void> {
clearActiveRun();
await saveState(cwd, sessionId, state, home);
}

// Crash-time terminal write. Deliberately bypasses writeChains: a hung or
// still-pending write for this session (possibly the very write mid-flight
Expand All @@ -116,6 +139,15 @@ export async function saveState(
// Callers must call markCrashed() (src/session/active-run.ts) before this, so
// any snapshot write still queued behind another one in the chain steps
// aside instead of racing this write's rename().
//
// This is a second terminal write path alongside finalizeRunState, and stays
// separate on purpose: its only callers are index.ts's process-level
// uncaughtException/unhandledRejection and signal handlers, reached when a
// crash escapes runTUI's own try/catch entirely. finalizeRunState routes
// through saveState's per-session write chain so writes apply in call order;
// that chain is exactly what a crash exit cannot afford to wait on, since
// process.exit must happen deterministically and a stuck earlier write
// (possibly the one that caused the crash) would otherwise hang it.
export async function saveCrashState(
cwd: string,
sessionId: string,
Expand Down
76 changes: 76 additions & 0 deletions src/tui/run-snapshot-kind.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { afterEach, beforeEach, describe, expect, test } from "bun:test";

import { clearActiveRun, getActiveRun, setActiveRun } from "../session/active-run.js";
import { finalizeRunState, loadState, saveState, type RunState } from "../session/state.js";
import { clearsActiveRun, type SnapshotKind } from "./runner.js";

describe("clearsActiveRun", () => {
test("only the run-ending write clears the active-run handle", () => {
expect(clearsActiveRun("run-end")).toBe(true);
expect(clearsActiveRun("progress")).toBe(false);
// The regression this pins: a /clear or /new rotation persists a
// terminal "done" for the outgoing session, but the process lives on.
// Clearing liveness here leaves every later session uncovered by the
// crash handler, so a crash after the first rotation never writes a
// terminal record and the session reads as "running" forever.
expect(clearsActiveRun("session-rotation")).toBe(false);
});
});

describe("a snapshot write dispatched by kind", () => {
let cwd = "";
let home = "";

// Mirrors writeRunSnapshot's dispatch in runner.ts so the rule above is
// exercised against the real state writers, not just asserted in isolation.
const write = async (sessionId: string, state: RunState, kind: SnapshotKind): Promise<void> => {
if (clearsActiveRun(kind)) {
await finalizeRunState(cwd, sessionId, state, home);
return;
}
await saveState(cwd, sessionId, state, home);
};

const runState = (over: Partial<RunState>): RunState => ({
status: "running",
turnsUsed: 0,
task: "task",
startedAt: 1,
...over,
});

beforeEach(() => {
cwd = mkdtempSync(join(tmpdir(), "snapshot-kind-cwd-"));
home = mkdtempSync(join(tmpdir(), "snapshot-kind-home-"));
});

afterEach(() => {
clearActiveRun();
rmSync(cwd, { recursive: true, force: true });
rmSync(home, { recursive: true, force: true });
});

test("a rotation still records the outgoing session but leaves the run crash-coverable", async () => {
setActiveRun({ sessionId: "old", cwd, task: "task", startedAt: 1 });

await write("old", runState({ status: "done", finishedAt: 10 }), "session-rotation");

expect((await loadState(cwd, "old", home))?.status).toBe("done");
// The rotated-in session is repointed on the same handle, so the handle
// must survive the write for the crash handler to have anything to close.
expect(getActiveRun()).not.toBeNull();
});

test("the run-ending write records the session and disarms the handle", async () => {
setActiveRun({ sessionId: "last", cwd, task: "task", startedAt: 1 });

await write("last", runState({ status: "done", finishedAt: 20 }), "run-end");

expect((await loadState(cwd, "last", home))?.status).toBe("done");
expect(getActiveRun()).toBeNull();
});
});
77 changes: 62 additions & 15 deletions src/tui/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ import {
import { createRunSink } from "../session/run-sink.js";
import { generateSessionId, initSessionDir, renameSession, sessionContextDir, sessionDir } from "../session/index.js";
import { resolveSessionLabel, truncateSessionLabel } from "../session/session-label.js";
import { loadState, saveState, type ConnectedMcpServer, type RunState } from "../session/state.js";
import { finalizeRunState, loadState, saveState, type ConnectedMcpServer, type RunState } from "../session/state.js";
import { setActiveRun, clearActiveRun, type RunStateHandle } from "../session/active-run.js";
import { setActiveDisposeHost, clearActiveDisposeHost } from "../session/active-host.js";
import { openInBrowser } from "../auth/oauth/browser.js";
Expand Down Expand Up @@ -241,6 +241,24 @@ export function resolveResumeSeed(pickedState: RunState | null): ResumeSeed {
};
}

/**
* Why a run.json snapshot is being written. Only "run-end" ends the run
* itself and so clears the active-run handle that the crash handler in
* index.ts reads.
*
* RunState.status cannot stand in for this. A /clear or /new rotation
* persists a terminal "done" for the outgoing session while the process
* keeps running under a fresh session id, so inferring "the run is over"
* from a non-"running" status disarms crash finalization for everything
* after the first rotation -- the session that dies then never gets its
* terminal record and reads as "running" forever.
*/
export type SnapshotKind = "progress" | "session-rotation" | "run-end";

export function clearsActiveRun(kind: SnapshotKind): boolean {
return kind === "run-end";
}

const GRANT_SCOPE_LABEL: Record<GrantScope, string> = {
session: "This session",
project: "This project",
Expand Down Expand Up @@ -503,7 +521,6 @@ export async function runTUI(initialConfig: Config): Promise<number> {
const activeRunHandle: RunStateHandle = {
sessionId,
cwd: config.cwd,
active: true,
task: runTaskTitle.trim().length > 0 ? runTaskTitle.trim() : "(conversation)",
startedAt,
model: `${config.providerName}:${config.model}`,
Expand Down Expand Up @@ -536,7 +553,15 @@ export async function runTUI(initialConfig: Config): Promise<number> {
const finalizeOnCrash = async (err: unknown): Promise<void> => {
if (finalized) return;
finalized = true;
activeRunHandle.active = false;
// Clear the active-run handle up front, before the awaits below. This
// handler isn't the only reader of the handle: index.ts installs its own
// uncaughtException/unhandledRejection listeners that call getActiveRun()
// directly and, if it's still set, write a competing "crashed" record via
// saveCrashState. finalizeRunState (state.ts) also clears the handle
// before its own saveState await, but only once it's called below — an
// escaped throw during the flushPartialOnCrash await just above would
// still reach that listener with the handle live, so it's cleared here
// too to close that earlier window.
clearActiveRun();
clearActiveDisposeHost();
await flushPartialOnCrash().catch((flushErr: unknown) => {
Expand All @@ -547,7 +572,7 @@ export async function runTUI(initialConfig: Config): Promise<number> {
process.stderr.write(`${COMMAND_NAME}: crash finalize partial flush failed: ${flushMessage}\n`);
});
const message = err instanceof Error ? err.message : String(err);
await saveState(config.cwd, sessionId, {
await finalizeRunState(config.cwd, sessionId, {
status: "failed",
turnsUsed: 0,
task: runTaskTitle.trim().length > 0 ? runTaskTitle.trim() : "(conversation)",
Expand Down Expand Up @@ -1360,6 +1385,7 @@ export async function runTUI(initialConfig: Config): Promise<number> {
const writeRunSnapshot = async (
status: RunState["status"],
extra?: Pick<RunState, "finishedAt" | "error">,
kind: SnapshotKind = "progress",
): Promise<void> => {
const task = runTaskTitle.trim().length > 0 ? runTaskTitle.trim() : "(conversation)";
const model = `${liveSource.id}:${liveSource.model}`;
Expand All @@ -1368,28 +1394,38 @@ export async function runTUI(initialConfig: Config): Promise<number> {
activeRunHandle.task = task;
activeRunHandle.startedAt = startedAt;
activeRunHandle.model = model;
await saveState(config.cwd, sessionId, {
const state: RunState = {
status,
turnsUsed: runSink.getTurnCount(),
task,
startedAt,
model,
mcpServers: connectedMcpServers,
...extra,
});
};
if (clearsActiveRun(kind)) {
await finalizeRunState(config.cwd, sessionId, state);
} else {
await saveState(config.cwd, sessionId, state);
}
};

// Progress snapshots are fired unsequenced (model switch, MCP connect, turn
// completion), so a straggler could otherwise land after the terminal write
// and resurrect status "running" — atomicWrite is last-rename-wins. Once the
// run is finalized, drop them; the terminal paths write through
// run is finalized, drop them; the run-ending path writes through
// writeRunSnapshot directly.
//
// Never a "run-end" write: everything routed here happens while the process
// is still alive and must stay crash-coverable, including the rotation
// "done" that closes out a session on /clear or /new.
const persistRunSnapshot = async (
status: RunState["status"],
extra?: Pick<RunState, "finishedAt" | "error">,
kind: Exclude<SnapshotKind, "run-end"> = "progress",
): Promise<void> => {
if (finalized) return;
await writeRunSnapshot(status, extra);
await writeRunSnapshot(status, extra, kind);
};

// Cycles persist to the context store only on inference.done; the recorder
Expand Down Expand Up @@ -1628,8 +1664,10 @@ export async function runTUI(initialConfig: Config): Promise<number> {
error: err instanceof Error ? err.message : String(err),
});
});
await persistRunSnapshot("done", { finishedAt: Date.now() });
await persistRunSnapshot("done", { finishedAt: Date.now() }, "session-rotation");
sessionId = generateSessionId();
// Repointed, not cleared: the process lives on, so the crash handler
// must keep finding this handle and close out the *new* session.
activeRunHandle.sessionId = sessionId;
startedAt = Date.now();
runTaskTitle = config.task;
Expand Down Expand Up @@ -2296,13 +2334,22 @@ export async function runTUI(initialConfig: Config): Promise<number> {
// finished run (finishedAt set) can be left reading as still in progress.
const persistedStatus: RunState["status"] = summaryStatus;
finalized = true;
activeRunHandle.active = false;
clearActiveRun();
// The run itself is over here, so this write clears the active-run handle
// (via finalizeRunState in state.ts) in the same call, rather than pairing
// the on-disk write with a separate in-memory statement at this call site.
// The dispose host has no on-disk counterpart to piggyback on, so it still
// needs its own clear here, mirroring finalizeOnCrash — otherwise a signal
// arriving after this normal exit would find a handle pointing at a
// torn-down closure.
clearActiveDisposeHost();
await writeRunSnapshot(persistedStatus, {
finishedAt,
...(sinkError !== undefined ? { error: sinkError } : {}),
});
await writeRunSnapshot(
persistedStatus,
{
finishedAt,
...(sinkError !== undefined ? { error: sinkError } : {}),
},
"run-end",
);
const runSummary = createRunSummary({
task: runTaskTitle.length > 0 ? runTaskTitle : config.task,
status: summaryStatus,
Expand Down
Loading
Loading