From 602edb48a225eecd577d19ad2ea26575f191faa7 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 10:02:21 -0700 Subject: [PATCH 1/2] Add tests for CL-6681: run history reads fire outcomes Warm-keep deliberately leaves a routine fire's delivery agent deployed after it replies, so workflow_run.status never settles out of "running" on its own. These tests pin the fix: a fire's displayed status should read as its actual outcome (completed once its reply window has passed) rather than the raw, permanently-live column, both in health.ts's own unit tests and in the Routines list's rendered markup. --- apps/web/test/routines-page.test.tsx | 26 ++++++++- packages/routines/src/health.test.ts | 79 ++++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/apps/web/test/routines-page.test.tsx b/apps/web/test/routines-page.test.tsx index cafc6bab3..a225f6069 100644 --- a/apps/web/test/routines-page.test.tsx +++ b/apps/web/test/routines-page.test.tsx @@ -75,6 +75,7 @@ describe("routineRowHealth", () => { test("Off for a disabled routine, regardless of run history", () => { const health = routineRowHealth( row({ routine: { ...routine, enabled: false } }), + listProps.now, ); expect(health.state).toBe("off"); expect(health.label).toBe("Off"); @@ -85,6 +86,7 @@ describe("routineRowHealth", () => { row({ routine: { ...routine, deadLetteredAt: "2026-01-02T00:00:00.000Z" }, }), + listProps.now, ); expect(health.state).toBe("paused"); }); @@ -96,7 +98,10 @@ describe("routineRowHealth", () => { createdAt: "2026-01-01T00:00:00.000Z", run: { status: "completed" }, }; - const health = routineRowHealth(row({ runs: [finished, finished] })); + const health = routineRowHealth( + row({ runs: [finished, finished] }), + listProps.now, + ); expect(health.state).toBe("ok"); expect(health.cleanStreak).toBe(2); }); @@ -206,7 +211,7 @@ describe("GlobalRoutinesList", () => { { runId: "run_1", triggeredBy: "schedule", - createdAt: "2026-01-01T09:00:00.000Z", + createdAt: "2026-01-01T11:55:00.000Z", run: { status: "running" }, }, ], @@ -216,6 +221,23 @@ describe("GlobalRoutinesList", () => { expect(markup).not.toContain(">running<"); }); + test("warm-keep (CL-6681): a fire whose 'running' status is stale reads as finished, not stuck Running now forever", () => { + const markup = renderList([ + row({ + runs: [ + { + runId: "run_1", + triggeredBy: "schedule", + createdAt: "2026-01-01T09:00:00.000Z", + run: { status: "running" }, + }, + ], + }), + ]); + expect(markup).not.toContain("Running now"); + expect(markup).toContain("Finished"); + }); + test("a failing routine states its failure count in words, not only in colour", () => { const markup = renderList([ row({ routine: { ...routine, consecutiveFailures: 2 } }), diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 2fa67cc10..48c518448 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test"; import { cleanFireStreak, + FIRE_RUNNING_WINDOW_MS, + fireOutcomeStatus, lastFailedFire, medianFireDurationMs, routineHealth, @@ -14,6 +16,9 @@ const healthy: RoutineHealthSubject = { deadLetteredAt: null, }; +const FIRE_CREATED_AT = "2026-01-01T00:00:00.000Z"; +const NOW = Date.parse(FIRE_CREATED_AT) + 60_000; + function fire( runId: string, overrides: Partial = {}, @@ -22,32 +27,74 @@ function fire( return { runId, triggeredBy: "schedule", - createdAt: "2026-01-01T00:00:00.000Z", + createdAt: FIRE_CREATED_AT, run, ...overrides, }; } +describe("fireOutcomeStatus", () => { + test("a running fire still inside its window reads as running", () => { + expect(fireOutcomeStatus(fire("r1", {}, { status: "running" }), NOW)).toBe( + "running", + ); + }); + + test("warm-keep (CL-6681): a running fire past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "running" }), staleNow), + ).toBe("completed"); + }); + + test("every terminal status passes through unchanged, however old", () => { + const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "failed" }), longAfter), + ).toBe("failed"); + expect( + fireOutcomeStatus(fire("r1", {}, { status: "cancelled" }), longAfter), + ).toBe("cancelled"); + }); + + test("null when the platform has no run to report", () => { + expect(fireOutcomeStatus(fire("r1", {}, {}), NOW)).toBeNull(); + }); +}); + describe("cleanFireStreak", () => { test("counts successes from the newest fire and stops at the first failure", () => { expect( - cleanFireStreak([ - fire("r5"), - fire("r4"), - fire("r3", {}, { status: "failed" }), - fire("r2"), - ]), + cleanFireStreak( + [ + fire("r5"), + fire("r4"), + fire("r3", {}, { status: "failed" }), + fire("r2"), + ], + NOW, + ), ).toBe(2); }); test("an in-flight run neither breaks nor extends the streak", () => { expect( - cleanFireStreak([fire("r2", {}, { status: "running" }), fire("r1")]), + cleanFireStreak([fire("r2", {}, { status: "running" }), fire("r1")], NOW), ).toBe(1); }); + test("a stale running fire (warm-keep) counts as a success, not in-flight forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + cleanFireStreak( + [fire("r2", {}, { status: "running" }), fire("r1")], + staleNow, + ), + ).toBe(2); + }); + test("no history is a streak of zero, not a failure", () => { - expect(cleanFireStreak([])).toBe(0); + expect(cleanFireStreak([], NOW)).toBe(0); }); }); @@ -135,10 +182,22 @@ describe("routineHealth", () => { test("an in-flight latest run reports Running now", () => { expect( - routineHealth(healthy, [fire("r1", {}, { status: "running" })]).state, + routineHealth(healthy, [fire("r1", {}, { status: "running" })], NOW) + .state, ).toBe("running"); }); + test("warm-keep (CL-6681): a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const health = routineHealth( + healthy, + [fire("r1", {}, { status: "running" })], + staleNow, + ); + expect(health.state).toBe("ok"); + expect(health.label).not.toBe("Running now"); + }); + test("consecutive failures are stated in the caption, not just the pill", () => { const health = routineHealth({ ...healthy, consecutiveFailures: 2 }, [ fire("r1", { error: "boom" }, {}), From 37dd44e172aeb7dfccf2440d74592fdb25fa6ab6 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 22 Aug 2026 10:02:30 -0700 Subject: [PATCH 2/2] Routines: run history and list status read fire outcomes (CL-6681) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployment liveness and fire outcome are two different facts, and the Routines list's status cell and the detail page's run history table both badged the former: a fire's raw workflow_run.status, which warm-keep deliberately leaves at "running" once the delivery agent stays deployed after replying. That column never settles back down on its own, so every fire (and the routine's own health pill) read as "RUNNING NOW" forever, matching CL-6595's Mission Control fix in spirit (the fires feed's job is separating a real deployment from a fired run) but not in mechanism, since a routine's run history has no feed to re-point at — it is already the platform's own fire ledger. health.ts's new fireOutcomeStatus is the one place that tells a fire still doing work apart from one merely staying warm: a "running" status past FIRE_RUNNING_WINDOW_MS since the fire started reads as "completed" instead of taken literally. routineHealth, cleanFireStreak, and the health pill's own "latest fire is running" check all route through it, and so does the Routines page's RunStatusCell (which feeds both the global list's Last run column and the detail page's Run history table) — one function decides for every surface that badges a fire's status, the same rule health.ts already claims for itself. --- apps/web/src/pages/routine-detail-page.tsx | 4 +- apps/web/src/pages/routines-page.tsx | 33 +++++++++---- packages/routines/src/client.ts | 2 + packages/routines/src/health.ts | 54 ++++++++++++++++++---- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/apps/web/src/pages/routine-detail-page.tsx b/apps/web/src/pages/routine-detail-page.tsx index 54096740d..0dfc305d6 100644 --- a/apps/web/src/pages/routine-detail-page.tsx +++ b/apps/web/src/pages/routine-detail-page.tsx @@ -309,7 +309,7 @@ export function RoutineRunHistory({ - + {formatRelativeTime(run.createdAt, now)} @@ -349,7 +349,7 @@ export function RoutineDetailPage({ readonly onToggleEnabled: (enabled: boolean) => void; readonly onSaveSchedule: (expression: string) => Promise; }) { - const health = routineHealth(row.routine, row.runs); + const health = routineHealth(row.routine, row.runs, now); const latestRunId = row.runs.find((run) => !fireNeverStarted(run.triggeredBy))?.runId ?? null; return ( diff --git a/apps/web/src/pages/routines-page.tsx b/apps/web/src/pages/routines-page.tsx index e6961e2d8..f7e00e85f 100644 --- a/apps/web/src/pages/routines-page.tsx +++ b/apps/web/src/pages/routines-page.tsx @@ -36,6 +36,7 @@ import type { BadgeTone, RunStatus } from "@corbits/react-ui"; import { Clock, PlayCircle, Plus } from "@corbits/icons"; import type { KeyboardEvent } from "react"; import { + fireOutcomeStatus, routineHealth, routineScheduleSentence, runStatusLabel, @@ -145,7 +146,7 @@ export function RunsTable({ - + {formatRelativeTime(run.createdAt, now)} @@ -176,11 +177,20 @@ export function TriggeredByCell({ run }: { readonly run: RoutineRun }) { ); } -/** A fire's settled run status in words, or a dash when the platform has - * no run to report (a launch that never got that far). */ -export function RunStatusCell({ run }: { readonly run: RoutineRun }) { - const status = run.run?.status; - if (typeof status !== "string") { +/** A fire's outcome in words, or a dash when the platform has no run to + * report (a launch that never got that far). Reads `fireOutcomeStatus`, + * not the raw `run.status` — warm-keep (CL-6681) leaves a fire's delivery + * agent deployed after it replies, so the raw column never settles out of + * `running` on its own. */ +export function RunStatusCell({ + run, + now, +}: { + readonly run: RoutineRun; + readonly now: number; +}) { + const status = fireOutcomeStatus(run, now); + if (status === null) { return —; } return {runStatusLabel(status)}; @@ -189,8 +199,11 @@ export function RunStatusCell({ run }: { readonly run: RoutineRun }) { /** A routine's health, from the telemetry the scheduler already records — * the same reading the detail page's health rail shows, never a second * opinion. */ -export function routineRowHealth(row: GlobalRoutineRow): RoutineHealth { - return routineHealth(row.routine, row.runs); +export function routineRowHealth( + row: GlobalRoutineRow, + now: number, +): RoutineHealth { + return routineHealth(row.routine, row.runs, now); } /** "At 09:00, Monday through Friday (UTC)" — the schedule as a sentence, @@ -275,7 +288,7 @@ export function GlobalRoutinesList({ {rows.map((row) => { - const health = routineRowHealth(row); + const health = routineRowHealth(row, now); const lastRun = latestFire(row); return ( @@ -311,7 +324,7 @@ export function GlobalRoutinesList({ {formatRelativeTime(lastRun.createdAt, now)} - + )} diff --git a/packages/routines/src/client.ts b/packages/routines/src/client.ts index e17d243f3..8e462e4c8 100644 --- a/packages/routines/src/client.ts +++ b/packages/routines/src/client.ts @@ -26,7 +26,9 @@ export { } from "./run-language"; export { cleanFireStreak, + FIRE_RUNNING_WINDOW_MS, fireFailed, + fireOutcomeStatus, lastFailedFire, medianFireDurationMs, routineHealth, diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index dd813ee99..0fa8691a1 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -70,6 +70,37 @@ function statusOf(fire: RoutineFire): string | null { return typeof status === "string" ? status : null; } +/** + * How long a fire may credibly still be doing work before its lingering + * `running` status is read as warm-keep (CL-6681) — a routine's delivery + * agent deliberately stays deployed after it replies, so + * `workflow_run.status` never settles out of `running` on its own. Past + * this window a `running` fire is presumed to have already delivered its + * reply, so every surface badging its status reads it through + * `fireOutcomeStatus` rather than the raw column. + */ +export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; + +/** + * A fire's status the way this build should show it: the raw `run.status` + * for every terminal value, but a `running` status older than + * `FIRE_RUNNING_WINDOW_MS` is read as `completed` instead of taken + * literally — see that constant's own comment. This is the one place that + * tells a fire still doing work apart from one merely staying warm; every + * caller that needs a fire's displayed status goes through here, never + * `fire.run?.status` directly. + */ +export function fireOutcomeStatus( + fire: RoutineFire, + now: number, +): string | null { + const status = statusOf(fire); + if (status !== "running") return status; + const startedAt = Date.parse(fire.createdAt); + if (Number.isNaN(startedAt)) return status; + return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; +} + /** A fire failed when it recorded a launch error (the synthetic * `schedule-failed` row) or its run settled as failed. */ export function fireFailed(fire: RoutineFire): boolean { @@ -77,17 +108,20 @@ export function fireFailed(fire: RoutineFire): boolean { return statusOf(fire) === "failed"; } -function fireSucceeded(fire: RoutineFire): boolean { - return !fireFailed(fire) && statusOf(fire) === "completed"; +function fireSucceeded(fire: RoutineFire, now: number): boolean { + return !fireFailed(fire) && fireOutcomeStatus(fire, now) === "completed"; } /** Successful fires from the newest backwards, stopping at the first * failure — the "N clean runs" streak, not a lifetime total. */ -export function cleanFireStreak(fires: readonly RoutineFire[]): number { +export function cleanFireStreak( + fires: readonly RoutineFire[], + now: number, +): number { let streak = 0; for (const fire of fires) { if (fireFailed(fire)) break; - if (fireSucceeded(fire)) streak += 1; + if (fireSucceeded(fire, now)) streak += 1; } return streak; } @@ -142,6 +176,7 @@ function stateAndWords( routine: RoutineHealthSubject, fires: readonly RoutineFire[], streak: number, + now: number, ): { readonly state: RoutineHealthState; readonly label: string; @@ -163,7 +198,7 @@ function stateAndWords( }; } const latest = fires[0]; - if (latest !== undefined && statusOf(latest) === "running") { + if (latest !== undefined && fireOutcomeStatus(latest, now) === "running") { return { state: "running", label: "Running now", @@ -203,14 +238,17 @@ function stateAndWords( /** * A routine's health from its own row plus its fire history (newest - * first, as `GET /routines/:id/runs` returns it). + * first, as `GET /routines/:id/runs` returns it). `now` defaults to the + * wall clock — callers that render a ticking page pass their own shared + * clock so this agrees with the rest of that page's relative times. */ export function routineHealth( routine: RoutineHealthSubject, fires: readonly RoutineFire[], + now: number = Date.now(), ): RoutineHealth { - const cleanStreak = cleanFireStreak(fires); - const words = stateAndWords(routine, fires, cleanStreak); + const cleanStreak = cleanFireStreak(fires, now); + const words = stateAndWords(routine, fires, cleanStreak, now); return { state: words.state, label: words.label,