diff --git a/apps/web/src/pages/routine-detail-page.tsx b/apps/web/src/pages/routine-detail-page.tsx index 54096740..0dfc305d 100644 --- a/apps/web/src/pages/routine-detail-page.tsx +++ b/apps/web/src/pages/routine-detail-page.tsx @@ -309,7 +309,7 @@ export function RoutineRunHistory({ - + {formatRelativeTime(run.createdAt, now)} @@ -349,7 +349,7 @@ export function RoutineDetailPage({ readonly onToggleEnabled: (enabled: boolean) => void; readonly onSaveSchedule: (expression: string) => Promise; }) { - const health = routineHealth(row.routine, row.runs); + const health = routineHealth(row.routine, row.runs, now); const latestRunId = row.runs.find((run) => !fireNeverStarted(run.triggeredBy))?.runId ?? null; return ( diff --git a/apps/web/src/pages/routines-page.tsx b/apps/web/src/pages/routines-page.tsx index e6961e2d..f7e00e85 100644 --- a/apps/web/src/pages/routines-page.tsx +++ b/apps/web/src/pages/routines-page.tsx @@ -36,6 +36,7 @@ import type { BadgeTone, RunStatus } from "@corbits/react-ui"; import { Clock, PlayCircle, Plus } from "@corbits/icons"; import type { KeyboardEvent } from "react"; import { + fireOutcomeStatus, routineHealth, routineScheduleSentence, runStatusLabel, @@ -145,7 +146,7 @@ export function RunsTable({ - + {formatRelativeTime(run.createdAt, now)} @@ -176,11 +177,20 @@ export function TriggeredByCell({ run }: { readonly run: RoutineRun }) { ); } -/** A fire's settled run status in words, or a dash when the platform has - * no run to report (a launch that never got that far). */ -export function RunStatusCell({ run }: { readonly run: RoutineRun }) { - const status = run.run?.status; - if (typeof status !== "string") { +/** A fire's outcome in words, or a dash when the platform has no run to + * report (a launch that never got that far). Reads `fireOutcomeStatus`, + * not the raw `run.status` — warm-keep (CL-6681) leaves a fire's delivery + * agent deployed after it replies, so the raw column never settles out of + * `running` on its own. */ +export function RunStatusCell({ + run, + now, +}: { + readonly run: RoutineRun; + readonly now: number; +}) { + const status = fireOutcomeStatus(run, now); + if (status === null) { return ; } return {runStatusLabel(status)}; @@ -189,8 +199,11 @@ export function RunStatusCell({ run }: { readonly run: RoutineRun }) { /** A routine's health, from the telemetry the scheduler already records — * the same reading the detail page's health rail shows, never a second * opinion. */ -export function routineRowHealth(row: GlobalRoutineRow): RoutineHealth { - return routineHealth(row.routine, row.runs); +export function routineRowHealth( + row: GlobalRoutineRow, + now: number, +): RoutineHealth { + return routineHealth(row.routine, row.runs, now); } /** "At 09:00, Monday through Friday (UTC)" — the schedule as a sentence, @@ -275,7 +288,7 @@ export function GlobalRoutinesList({ {rows.map((row) => { - const health = routineRowHealth(row); + const health = routineRowHealth(row, now); const lastRun = latestFire(row); return ( @@ -311,7 +324,7 @@ export function GlobalRoutinesList({ {formatRelativeTime(lastRun.createdAt, now)} - + )} diff --git a/apps/web/test/routines-page.test.tsx b/apps/web/test/routines-page.test.tsx index cafc6bab..a225f606 100644 --- a/apps/web/test/routines-page.test.tsx +++ b/apps/web/test/routines-page.test.tsx @@ -75,6 +75,7 @@ describe("routineRowHealth", () => { test("Off for a disabled routine, regardless of run history", () => { const health = routineRowHealth( row({ routine: { ...routine, enabled: false } }), + listProps.now, ); expect(health.state).toBe("off"); expect(health.label).toBe("Off"); @@ -85,6 +86,7 @@ describe("routineRowHealth", () => { row({ routine: { ...routine, deadLetteredAt: "2026-01-02T00:00:00.000Z" }, }), + listProps.now, ); expect(health.state).toBe("paused"); }); @@ -96,7 +98,10 @@ describe("routineRowHealth", () => { createdAt: "2026-01-01T00:00:00.000Z", run: { status: "completed" }, }; - const health = routineRowHealth(row({ runs: [finished, finished] })); + const health = routineRowHealth( + row({ runs: [finished, finished] }), + listProps.now, + ); expect(health.state).toBe("ok"); expect(health.cleanStreak).toBe(2); }); @@ -206,7 +211,7 @@ describe("GlobalRoutinesList", () => { { runId: "run_1", triggeredBy: "schedule", - createdAt: "2026-01-01T09:00:00.000Z", + createdAt: "2026-01-01T11:55:00.000Z", run: { status: "running" }, }, ], @@ -216,6 +221,23 @@ describe("GlobalRoutinesList", () => { expect(markup).not.toContain(">running<"); }); + test("warm-keep (CL-6681): a fire whose 'running' status is stale reads as finished, not stuck Running now forever", () => { + const markup = renderList([ + row({ + runs: [ + { + runId: "run_1", + triggeredBy: "schedule", + createdAt: "2026-01-01T09:00:00.000Z", + run: { status: "running" }, + }, + ], + }), + ]); + expect(markup).not.toContain("Running now"); + expect(markup).toContain("Finished"); + }); + test("a failing routine states its failure count in words, not only in colour", () => { const markup = renderList([ row({ routine: { ...routine, consecutiveFailures: 2 } }), diff --git a/packages/routines/src/client.ts b/packages/routines/src/client.ts index e17d243f..8e462e4c 100644 --- a/packages/routines/src/client.ts +++ b/packages/routines/src/client.ts @@ -26,7 +26,9 @@ export { } from "./run-language"; export { cleanFireStreak, + FIRE_RUNNING_WINDOW_MS, fireFailed, + fireOutcomeStatus, lastFailedFire, medianFireDurationMs, routineHealth, diff --git a/packages/routines/src/health.test.ts b/packages/routines/src/health.test.ts index 2fa67cc1..48c51844 100644 --- a/packages/routines/src/health.test.ts +++ b/packages/routines/src/health.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test"; import { cleanFireStreak, + FIRE_RUNNING_WINDOW_MS, + fireOutcomeStatus, lastFailedFire, medianFireDurationMs, routineHealth, @@ -14,6 +16,9 @@ const healthy: RoutineHealthSubject = { deadLetteredAt: null, }; +const FIRE_CREATED_AT = "2026-01-01T00:00:00.000Z"; +const NOW = Date.parse(FIRE_CREATED_AT) + 60_000; + function fire( runId: string, overrides: Partial = {}, @@ -22,32 +27,74 @@ function fire( return { runId, triggeredBy: "schedule", - createdAt: "2026-01-01T00:00:00.000Z", + createdAt: FIRE_CREATED_AT, run, ...overrides, }; } +describe("fireOutcomeStatus", () => { + test("a running fire still inside its window reads as running", () => { + expect(fireOutcomeStatus(fire("r1", {}, { status: "running" }), NOW)).toBe( + "running", + ); + }); + + test("warm-keep (CL-6681): a running fire past its window reads as completed", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "running" }), staleNow), + ).toBe("completed"); + }); + + test("every terminal status passes through unchanged, however old", () => { + const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10; + expect( + fireOutcomeStatus(fire("r1", {}, { status: "failed" }), longAfter), + ).toBe("failed"); + expect( + fireOutcomeStatus(fire("r1", {}, { status: "cancelled" }), longAfter), + ).toBe("cancelled"); + }); + + test("null when the platform has no run to report", () => { + expect(fireOutcomeStatus(fire("r1", {}, {}), NOW)).toBeNull(); + }); +}); + describe("cleanFireStreak", () => { test("counts successes from the newest fire and stops at the first failure", () => { expect( - cleanFireStreak([ - fire("r5"), - fire("r4"), - fire("r3", {}, { status: "failed" }), - fire("r2"), - ]), + cleanFireStreak( + [ + fire("r5"), + fire("r4"), + fire("r3", {}, { status: "failed" }), + fire("r2"), + ], + NOW, + ), ).toBe(2); }); test("an in-flight run neither breaks nor extends the streak", () => { expect( - cleanFireStreak([fire("r2", {}, { status: "running" }), fire("r1")]), + cleanFireStreak([fire("r2", {}, { status: "running" }), fire("r1")], NOW), ).toBe(1); }); + test("a stale running fire (warm-keep) counts as a success, not in-flight forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + expect( + cleanFireStreak( + [fire("r2", {}, { status: "running" }), fire("r1")], + staleNow, + ), + ).toBe(2); + }); + test("no history is a streak of zero, not a failure", () => { - expect(cleanFireStreak([])).toBe(0); + expect(cleanFireStreak([], NOW)).toBe(0); }); }); @@ -135,10 +182,22 @@ describe("routineHealth", () => { test("an in-flight latest run reports Running now", () => { expect( - routineHealth(healthy, [fire("r1", {}, { status: "running" })]).state, + routineHealth(healthy, [fire("r1", {}, { status: "running" })], NOW) + .state, ).toBe("running"); }); + test("warm-keep (CL-6681): a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => { + const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1; + const health = routineHealth( + healthy, + [fire("r1", {}, { status: "running" })], + staleNow, + ); + expect(health.state).toBe("ok"); + expect(health.label).not.toBe("Running now"); + }); + test("consecutive failures are stated in the caption, not just the pill", () => { const health = routineHealth({ ...healthy, consecutiveFailures: 2 }, [ fire("r1", { error: "boom" }, {}), diff --git a/packages/routines/src/health.ts b/packages/routines/src/health.ts index dd813ee9..0fa8691a 100644 --- a/packages/routines/src/health.ts +++ b/packages/routines/src/health.ts @@ -70,6 +70,37 @@ function statusOf(fire: RoutineFire): string | null { return typeof status === "string" ? status : null; } +/** + * How long a fire may credibly still be doing work before its lingering + * `running` status is read as warm-keep (CL-6681) — a routine's delivery + * agent deliberately stays deployed after it replies, so + * `workflow_run.status` never settles out of `running` on its own. Past + * this window a `running` fire is presumed to have already delivered its + * reply, so every surface badging its status reads it through + * `fireOutcomeStatus` rather than the raw column. + */ +export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000; + +/** + * A fire's status the way this build should show it: the raw `run.status` + * for every terminal value, but a `running` status older than + * `FIRE_RUNNING_WINDOW_MS` is read as `completed` instead of taken + * literally — see that constant's own comment. This is the one place that + * tells a fire still doing work apart from one merely staying warm; every + * caller that needs a fire's displayed status goes through here, never + * `fire.run?.status` directly. + */ +export function fireOutcomeStatus( + fire: RoutineFire, + now: number, +): string | null { + const status = statusOf(fire); + if (status !== "running") return status; + const startedAt = Date.parse(fire.createdAt); + if (Number.isNaN(startedAt)) return status; + return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running"; +} + /** A fire failed when it recorded a launch error (the synthetic * `schedule-failed` row) or its run settled as failed. */ export function fireFailed(fire: RoutineFire): boolean { @@ -77,17 +108,20 @@ export function fireFailed(fire: RoutineFire): boolean { return statusOf(fire) === "failed"; } -function fireSucceeded(fire: RoutineFire): boolean { - return !fireFailed(fire) && statusOf(fire) === "completed"; +function fireSucceeded(fire: RoutineFire, now: number): boolean { + return !fireFailed(fire) && fireOutcomeStatus(fire, now) === "completed"; } /** Successful fires from the newest backwards, stopping at the first * failure — the "N clean runs" streak, not a lifetime total. */ -export function cleanFireStreak(fires: readonly RoutineFire[]): number { +export function cleanFireStreak( + fires: readonly RoutineFire[], + now: number, +): number { let streak = 0; for (const fire of fires) { if (fireFailed(fire)) break; - if (fireSucceeded(fire)) streak += 1; + if (fireSucceeded(fire, now)) streak += 1; } return streak; } @@ -142,6 +176,7 @@ function stateAndWords( routine: RoutineHealthSubject, fires: readonly RoutineFire[], streak: number, + now: number, ): { readonly state: RoutineHealthState; readonly label: string; @@ -163,7 +198,7 @@ function stateAndWords( }; } const latest = fires[0]; - if (latest !== undefined && statusOf(latest) === "running") { + if (latest !== undefined && fireOutcomeStatus(latest, now) === "running") { return { state: "running", label: "Running now", @@ -203,14 +238,17 @@ function stateAndWords( /** * A routine's health from its own row plus its fire history (newest - * first, as `GET /routines/:id/runs` returns it). + * first, as `GET /routines/:id/runs` returns it). `now` defaults to the + * wall clock — callers that render a ticking page pass their own shared + * clock so this agrees with the rest of that page's relative times. */ export function routineHealth( routine: RoutineHealthSubject, fires: readonly RoutineFire[], + now: number = Date.now(), ): RoutineHealth { - const cleanStreak = cleanFireStreak(fires); - const words = stateAndWords(routine, fires, cleanStreak); + const cleanStreak = cleanFireStreak(fires, now); + const words = stateAndWords(routine, fires, cleanStreak, now); return { state: words.state, label: words.label,