Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/web/src/pages/routine-detail-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ export function RoutineRunHistory({
<TriggeredByCell run={run} />
</TableCell>
<TableCell>
<RunStatusCell run={run} />
<RunStatusCell run={run} now={now} />
</TableCell>
<TableCell>{formatRelativeTime(run.createdAt, now)}</TableCell>
<TableCell>
Expand Down Expand Up @@ -349,7 +349,7 @@ export function RoutineDetailPage({
readonly onToggleEnabled: (enabled: boolean) => void;
readonly onSaveSchedule: (expression: string) => Promise<void>;
}) {
const health = routineHealth(row.routine, row.runs);
const health = routineHealth(row.routine, row.runs, now);
const latestRunId =
row.runs.find((run) => !fireNeverStarted(run.triggeredBy))?.runId ?? null;
return (
Expand Down
33 changes: 23 additions & 10 deletions apps/web/src/pages/routines-page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import type { BadgeTone, RunStatus } from "@corbits/react-ui";
import { Clock, PlayCircle, Plus } from "@corbits/icons";
import type { KeyboardEvent } from "react";
import {
fireOutcomeStatus,
routineHealth,
routineScheduleSentence,
runStatusLabel,
Expand Down Expand Up @@ -145,7 +146,7 @@ export function RunsTable({
<TriggeredByCell run={run} />
</TableCell>
<TableCell>
<RunStatusCell run={run} />
<RunStatusCell run={run} now={now} />
</TableCell>
<TableCell>{formatRelativeTime(run.createdAt, now)}</TableCell>
</TableRow>
Expand Down Expand Up @@ -176,11 +177,20 @@ export function TriggeredByCell({ run }: { readonly run: RoutineRun }) {
);
}

/** A fire's settled run status in words, or a dash when the platform has
* no run to report (a launch that never got that far). */
export function RunStatusCell({ run }: { readonly run: RoutineRun }) {
const status = run.run?.status;
if (typeof status !== "string") {
/** A fire's outcome in words, or a dash when the platform has no run to
* report (a launch that never got that far). Reads `fireOutcomeStatus`,
* not the raw `run.status` — warm-keep (CL-6681) leaves a fire's delivery
* agent deployed after it replies, so the raw column never settles out of
* `running` on its own. */
export function RunStatusCell({
run,
now,
}: {
readonly run: RoutineRun;
readonly now: number;
}) {
const status = fireOutcomeStatus(run, now);
if (status === null) {
return <span className="text-[var(--ui-fg-muted)]">—</span>;
}
return <Badge tone={runStatusTone(status)}>{runStatusLabel(status)}</Badge>;
Expand All @@ -189,8 +199,11 @@ export function RunStatusCell({ run }: { readonly run: RoutineRun }) {
/** A routine's health, from the telemetry the scheduler already records —
* the same reading the detail page's health rail shows, never a second
* opinion. */
export function routineRowHealth(row: GlobalRoutineRow): RoutineHealth {
return routineHealth(row.routine, row.runs);
export function routineRowHealth(
row: GlobalRoutineRow,
now: number,
): RoutineHealth {
return routineHealth(row.routine, row.runs, now);
}

/** "At 09:00, Monday through Friday (UTC)" — the schedule as a sentence,
Expand Down Expand Up @@ -275,7 +288,7 @@ export function GlobalRoutinesList({
</TableHeader>
<TableBody>
{rows.map((row) => {
const health = routineRowHealth(row);
const health = routineRowHealth(row, now);
const lastRun = latestFire(row);
return (
<TableRow key={row.routine.id}>
Expand Down Expand Up @@ -311,7 +324,7 @@ export function GlobalRoutinesList({
<span className="text-sm">
{formatRelativeTime(lastRun.createdAt, now)}
</span>
<RunStatusCell run={lastRun} />
<RunStatusCell run={lastRun} now={now} />
</span>
)}
</TableCell>
Expand Down
26 changes: 24 additions & 2 deletions apps/web/test/routines-page.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ describe("routineRowHealth", () => {
test("Off for a disabled routine, regardless of run history", () => {
const health = routineRowHealth(
row({ routine: { ...routine, enabled: false } }),
listProps.now,
);
expect(health.state).toBe("off");
expect(health.label).toBe("Off");
Expand All @@ -85,6 +86,7 @@ describe("routineRowHealth", () => {
row({
routine: { ...routine, deadLetteredAt: "2026-01-02T00:00:00.000Z" },
}),
listProps.now,
);
expect(health.state).toBe("paused");
});
Expand All @@ -96,7 +98,10 @@ describe("routineRowHealth", () => {
createdAt: "2026-01-01T00:00:00.000Z",
run: { status: "completed" },
};
const health = routineRowHealth(row({ runs: [finished, finished] }));
const health = routineRowHealth(
row({ runs: [finished, finished] }),
listProps.now,
);
expect(health.state).toBe("ok");
expect(health.cleanStreak).toBe(2);
});
Expand Down Expand Up @@ -206,7 +211,7 @@ describe("GlobalRoutinesList", () => {
{
runId: "run_1",
triggeredBy: "schedule",
createdAt: "2026-01-01T09:00:00.000Z",
createdAt: "2026-01-01T11:55:00.000Z",
run: { status: "running" },
},
],
Expand All @@ -216,6 +221,23 @@ describe("GlobalRoutinesList", () => {
expect(markup).not.toContain(">running<");
});

test("warm-keep (CL-6681): a fire whose 'running' status is stale reads as finished, not stuck Running now forever", () => {
const markup = renderList([
row({
runs: [
{
runId: "run_1",
triggeredBy: "schedule",
createdAt: "2026-01-01T09:00:00.000Z",
run: { status: "running" },
},
],
}),
]);
expect(markup).not.toContain("Running now");
expect(markup).toContain("Finished");
});

test("a failing routine states its failure count in words, not only in colour", () => {
const markup = renderList([
row({ routine: { ...routine, consecutiveFailures: 2 } }),
Expand Down
2 changes: 2 additions & 0 deletions packages/routines/src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ export {
} from "./run-language";
export {
cleanFireStreak,
FIRE_RUNNING_WINDOW_MS,
fireFailed,
fireOutcomeStatus,
lastFailedFire,
medianFireDurationMs,
routineHealth,
Expand Down
79 changes: 69 additions & 10 deletions packages/routines/src/health.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test";

import {
cleanFireStreak,
FIRE_RUNNING_WINDOW_MS,
fireOutcomeStatus,
lastFailedFire,
medianFireDurationMs,
routineHealth,
Expand All @@ -14,6 +16,9 @@ const healthy: RoutineHealthSubject = {
deadLetteredAt: null,
};

const FIRE_CREATED_AT = "2026-01-01T00:00:00.000Z";
const NOW = Date.parse(FIRE_CREATED_AT) + 60_000;

function fire(
runId: string,
overrides: Partial<RoutineFire> = {},
Expand All @@ -22,32 +27,74 @@ function fire(
return {
runId,
triggeredBy: "schedule",
createdAt: "2026-01-01T00:00:00.000Z",
createdAt: FIRE_CREATED_AT,
run,
...overrides,
};
}

describe("fireOutcomeStatus", () => {
test("a running fire still inside its window reads as running", () => {
expect(fireOutcomeStatus(fire("r1", {}, { status: "running" }), NOW)).toBe(
"running",
);
});

test("warm-keep (CL-6681): a running fire past its window reads as completed", () => {
const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1;
expect(
fireOutcomeStatus(fire("r1", {}, { status: "running" }), staleNow),
).toBe("completed");
});

test("every terminal status passes through unchanged, however old", () => {
const longAfter = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS * 10;
expect(
fireOutcomeStatus(fire("r1", {}, { status: "failed" }), longAfter),
).toBe("failed");
expect(
fireOutcomeStatus(fire("r1", {}, { status: "cancelled" }), longAfter),
).toBe("cancelled");
});

test("null when the platform has no run to report", () => {
expect(fireOutcomeStatus(fire("r1", {}, {}), NOW)).toBeNull();
});
});

describe("cleanFireStreak", () => {
test("counts successes from the newest fire and stops at the first failure", () => {
expect(
cleanFireStreak([
fire("r5"),
fire("r4"),
fire("r3", {}, { status: "failed" }),
fire("r2"),
]),
cleanFireStreak(
[
fire("r5"),
fire("r4"),
fire("r3", {}, { status: "failed" }),
fire("r2"),
],
NOW,
),
).toBe(2);
});

test("an in-flight run neither breaks nor extends the streak", () => {
expect(
cleanFireStreak([fire("r2", {}, { status: "running" }), fire("r1")]),
cleanFireStreak([fire("r2", {}, { status: "running" }), fire("r1")], NOW),
).toBe(1);
});

test("a stale running fire (warm-keep) counts as a success, not in-flight forever", () => {
const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1;
expect(
cleanFireStreak(
[fire("r2", {}, { status: "running" }), fire("r1")],
staleNow,
),
).toBe(2);
});

test("no history is a streak of zero, not a failure", () => {
expect(cleanFireStreak([])).toBe(0);
expect(cleanFireStreak([], NOW)).toBe(0);
});
});

Expand Down Expand Up @@ -135,10 +182,22 @@ describe("routineHealth", () => {

test("an in-flight latest run reports Running now", () => {
expect(
routineHealth(healthy, [fire("r1", {}, { status: "running" })]).state,
routineHealth(healthy, [fire("r1", {}, { status: "running" })], NOW)
.state,
).toBe("running");
});

test("warm-keep (CL-6681): a latest run stuck 'running' past its window reads as healthy, not stuck Running now forever", () => {
const staleNow = Date.parse(FIRE_CREATED_AT) + FIRE_RUNNING_WINDOW_MS + 1;
const health = routineHealth(
healthy,
[fire("r1", {}, { status: "running" })],
staleNow,
);
expect(health.state).toBe("ok");
expect(health.label).not.toBe("Running now");
});

test("consecutive failures are stated in the caption, not just the pill", () => {
const health = routineHealth({ ...healthy, consecutiveFailures: 2 }, [
fire("r1", { error: "boom" }, {}),
Expand Down
54 changes: 46 additions & 8 deletions packages/routines/src/health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,24 +70,58 @@ function statusOf(fire: RoutineFire): string | null {
return typeof status === "string" ? status : null;
}

/**
* How long a fire may credibly still be doing work before its lingering
* `running` status is read as warm-keep (CL-6681) — a routine's delivery
* agent deliberately stays deployed after it replies, so
* `workflow_run.status` never settles out of `running` on its own. Past
* this window a `running` fire is presumed to have already delivered its
* reply, so every surface badging its status reads it through
* `fireOutcomeStatus` rather than the raw column.
*/
export const FIRE_RUNNING_WINDOW_MS = 10 * 60 * 1000;

/**
* A fire's status the way this build should show it: the raw `run.status`
* for every terminal value, but a `running` status older than
* `FIRE_RUNNING_WINDOW_MS` is read as `completed` instead of taken
* literally — see that constant's own comment. This is the one place that
* tells a fire still doing work apart from one merely staying warm; every
* caller that needs a fire's displayed status goes through here, never
* `fire.run?.status` directly.
*/
export function fireOutcomeStatus(
fire: RoutineFire,
now: number,
): string | null {
const status = statusOf(fire);
if (status !== "running") return status;
const startedAt = Date.parse(fire.createdAt);
if (Number.isNaN(startedAt)) return status;
return now - startedAt > FIRE_RUNNING_WINDOW_MS ? "completed" : "running";
}

/** A fire failed when it recorded a launch error (the synthetic
* `schedule-failed` row) or its run settled as failed. */
export function fireFailed(fire: RoutineFire): boolean {
if (fire.error !== undefined && fire.error !== null) return true;
return statusOf(fire) === "failed";
}

function fireSucceeded(fire: RoutineFire): boolean {
return !fireFailed(fire) && statusOf(fire) === "completed";
function fireSucceeded(fire: RoutineFire, now: number): boolean {
return !fireFailed(fire) && fireOutcomeStatus(fire, now) === "completed";
}

/** Successful fires from the newest backwards, stopping at the first
* failure — the "N clean runs" streak, not a lifetime total. */
export function cleanFireStreak(fires: readonly RoutineFire[]): number {
export function cleanFireStreak(
fires: readonly RoutineFire[],
now: number,
): number {
let streak = 0;
for (const fire of fires) {
if (fireFailed(fire)) break;
if (fireSucceeded(fire)) streak += 1;
if (fireSucceeded(fire, now)) streak += 1;
}
return streak;
}
Expand Down Expand Up @@ -142,6 +176,7 @@ function stateAndWords(
routine: RoutineHealthSubject,
fires: readonly RoutineFire[],
streak: number,
now: number,
): {
readonly state: RoutineHealthState;
readonly label: string;
Expand All @@ -163,7 +198,7 @@ function stateAndWords(
};
}
const latest = fires[0];
if (latest !== undefined && statusOf(latest) === "running") {
if (latest !== undefined && fireOutcomeStatus(latest, now) === "running") {
return {
state: "running",
label: "Running now",
Expand Down Expand Up @@ -203,14 +238,17 @@ function stateAndWords(

/**
* A routine's health from its own row plus its fire history (newest
* first, as `GET /routines/:id/runs` returns it).
* first, as `GET /routines/:id/runs` returns it). `now` defaults to the
* wall clock — callers that render a ticking page pass their own shared
* clock so this agrees with the rest of that page's relative times.
*/
export function routineHealth(
routine: RoutineHealthSubject,
fires: readonly RoutineFire[],
now: number = Date.now(),
): RoutineHealth {
const cleanStreak = cleanFireStreak(fires);
const words = stateAndWords(routine, fires, cleanStreak);
const cleanStreak = cleanFireStreak(fires, now);
const words = stateAndWords(routine, fires, cleanStreak, now);
return {
state: words.state,
label: words.label,
Expand Down
Loading