Merge pull request #551 from corbitsdev/cl-6935-stop-labeling-short-xai-rate-limits-as-quota-exhaustion

Stop labeling short xAI rate limits as quota exhaustion

Author: TheGreatAxios

CHANGELOG.md

@@ -17,6 +17,11 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
 
 ### Agent
 
+- **xAI short HTTP 429s are rate limits, not quota exhaustion.** Bare 429s from
+  known xAI/Grok providers remapped to retryable so moderate Retry-After no
+  longer aborts as a long-window quota. Clear usage/quota body markers still
+  abort. Transcript shows "Rate limited — retrying…" instead of "Quota exhausted".
+
 - **`apply_patch` can update files again.** Its Update operation read the target
   through the line-numbered `read_file` view and then tried to match the patch's
   raw context lines against it, so every context-bearing update failed with

src/agent/director.ts

@@ -8,6 +8,7 @@ import type {
   ReactorAction,
   ToolDefinition,
   ConversationTurn,
+  RetryPolicy,
 } from "@intx/types/runtime";
 import { type SessionMetadata, type TaskBoundary } from "../session/compactor.js";
 import type { WorkflowCoordinator } from "../workflows/coordinator.js";
@@ -33,8 +34,6 @@ import { isOperatorOriginated } from "./message-provenance.js";
 import { classifyBriefSalvage, isHardBlockSalvage } from "../subagent/brief-dispatch.js";
 import { PRIMARY_SALVAGE_NUDGE } from "./look-tour.js";
 
-const RETRY_POLICY = createCorbitsRetryPolicy();
-
 // Fired when turnsSinceUserMessage reaches TURNS_SINCE_USER_MESSAGE_BACKSTOP.
 // A nudge, not a pause — the operator explicitly wants long autonomous runs
 // to keep going, so silence alone (with no detected cycle) is not
@@ -356,12 +355,22 @@ export interface ChatDirectorOptions {
   onTasksChange: (tasks: Task[]) => void;
   requestContinuation?: (() => void) | undefined;
   provider?: { providerName: string; model?: string } | undefined;
+  /**
+   * Live catalog provider id for retry stamping. Resolved on each retry
+   * decision so mid-session `/model` switches remapping without rebuilding
+   * the agent. When set, preferred over static `provider.providerName`.
+   */
+  getProviderId?: (() => string | undefined) | undefined;
+  /** Explicit retry policy; when set, skips the default Corbits policy. */
+  retryPolicy?: RetryPolicy | undefined;
 }
 
 // The constructor takes the resolved ModelFamilyPolicy rather than the raw
 // `provider` input the factory function accepts and resolves on its behalf.
 type ChatDirectorImplOptions = Omit<ChatDirectorOptions, "provider"> & {
   modelFamilyPolicy?: ModelFamilyPolicy | undefined;
+  /** Provider-stamped retry policy (xAI short 429 remapping needs providerId). */
+  retryPolicy?: RetryPolicy | undefined;
 };
 
 class ChatDirectorImpl extends DefaultDirector {
@@ -390,6 +399,7 @@ class ChatDirectorImpl extends DefaultDirector {
   private startedAt = Date.now();
   private readonly compaction: CompactionGovernor;
   private readonly modelFamilyPolicy: ModelFamilyPolicy;
+  private readonly retryPolicy: RetryPolicy;
   // Consecutive assistant turns that contain tool calls and no text. Reset on
   // any turn with text and on every fresh user message — a weak model that
   // spins in place on one thread of tool calls still converges to the
@@ -476,6 +486,7 @@ class ChatDirectorImpl extends DefaultDirector {
     );
     this.modelFamilyPolicy =
       options.modelFamilyPolicy ?? resolveModelFamilyPolicy({ providerName: "" });
+    this.retryPolicy = options.retryPolicy ?? createCorbitsRetryPolicy();
   }
 
   setWorkflowCoordinator(coordinator: WorkflowCoordinator | undefined): void {
@@ -618,7 +629,7 @@ class ChatDirectorImpl extends DefaultDirector {
       const options = {
         ...action.options,
         tools,
-        retryPolicy: action.options?.retryPolicy ?? RETRY_POLICY,
+        retryPolicy: action.options?.retryPolicy ?? this.retryPolicy,
       };
       if (this.inactivityTimeoutMs !== undefined)
         options.inactivityTimeoutMs = this.inactivityTimeoutMs;
@@ -1064,12 +1075,24 @@ export function createChatDirector(
   toolDefinitions: ToolDefinition[],
   options: ChatDirectorOptions,
 ): ChatDirector {
-  const { provider, ...rest } = options;
+  const { provider, getProviderId, retryPolicy, ...rest } = options;
   return new ChatDirectorImpl(systemPrompt, toolDefinitions, {
     ...rest,
     // `provider` is raw {providerName, model} input; the constructor wants
     // the resolved ModelFamilyPolicy, not the input it was resolved from.
     modelFamilyPolicy: provider !== undefined ? resolveModelFamilyPolicy(provider) : undefined,
+    // Stamp provider id onto retry errors so known-xAI short 429s remap.
+    // Prefer an explicit policy, then a live getter (mid-session `/model`),
+    // then the bootstrap providerName.
+    retryPolicy:
+      retryPolicy ??
+      createCorbitsRetryPolicy(
+        getProviderId !== undefined
+          ? { providerId: getProviderId }
+          : provider !== undefined
+            ? { providerId: provider.providerName }
+            : undefined,
+      ),
   });
 }
 

src/agent/retry-policy.test.ts

@@ -55,4 +55,96 @@ describe("createCorbitsRetryPolicy", () => {
     });
     expect(decision).toEqual({ kind: "abort" });
   });
+
+  test("stamped xAI bare 429 retries as retryable, not long-quota abort", async () => {
+    const policy = createCorbitsRetryPolicy({ providerId: "xai/thegreataxios" });
+    const decision = await policy({
+      attempt: 1,
+      elapsedMs: 0,
+      error: {
+        category: "quota_exhausted",
+        message: "Too Many Requests",
+        statusCode: 429,
+        retryAfterMs: 45_000,
+        raw: { error: { message: "Too Many Requests" } },
+      },
+    });
+    // Remapped to retryable → default backoff, not abort on moderate Retry-After.
+    expect(decision).toEqual({ kind: "retry", delayMs: 500 });
+  });
+
+  test("stamped xAI usage/quota body still aborts on long retryAfterMs", async () => {
+    const policy = createCorbitsRetryPolicy({ providerId: "xai/thegreataxios" });
+    const decision = await policy({
+      attempt: 1,
+      elapsedMs: 0,
+      error: {
+        category: "quota_exhausted",
+        message: "You exceeded your current quota",
+        statusCode: 429,
+        retryAfterMs: 86_400_000,
+        raw: {
+          error: {
+            message: "You exceeded your current quota",
+            code: "insufficient_quota",
+          },
+        },
+      },
+    });
+    expect(decision).toEqual({ kind: "abort" });
+  });
+
+  test("unknown provider bare 429 with moderate Retry-After still aborts as quota", async () => {
+    const policy = createCorbitsRetryPolicy({ providerId: "openai" });
+    const decision = await policy({
+      attempt: 1,
+      elapsedMs: 0,
+      error: {
+        category: "quota_exhausted",
+        message: "Too Many Requests",
+        statusCode: 429,
+        retryAfterMs: 45_000,
+        raw: { error: { message: "Too Many Requests" } },
+      },
+    });
+    expect(decision).toEqual({ kind: "abort" });
+  });
+
+  test("live providerId getter: non-xAI → xAI starts remapping bare 429", async () => {
+    let current: string | undefined = "openai";
+    const policy = createCorbitsRetryPolicy({ providerId: () => current });
+    const bare429 = {
+      attempt: 1,
+      elapsedMs: 0,
+      error: {
+        category: "quota_exhausted" as const,
+        message: "Too Many Requests",
+        statusCode: 429,
+        retryAfterMs: 45_000,
+        raw: { error: { message: "Too Many Requests" } },
+      },
+    };
+    expect(await policy(bare429)).toEqual({ kind: "abort" });
+    current = "xai/thegreataxios";
+    expect(await policy(bare429)).toEqual({ kind: "retry", delayMs: 500 });
+  });
+
+  test("live providerId getter: xAI → non-xAI stops remapping bare 429", async () => {
+    let current: string | undefined = "xai/thegreataxios";
+    const policy = createCorbitsRetryPolicy({ providerId: () => current });
+    const bare429 = {
+      attempt: 1,
+      elapsedMs: 0,
+      error: {
+        category: "quota_exhausted" as const,
+        message: "Too Many Requests",
+        statusCode: 429,
+        retryAfterMs: 45_000,
+        raw: { error: { message: "Too Many Requests" } },
+      },
+    };
+    expect(await policy(bare429)).toEqual({ kind: "retry", delayMs: 500 });
+    current = "openai";
+    expect(await policy(bare429)).toEqual({ kind: "abort" });
+  });
 });

src/agent/retry-policy.ts

@@ -1,6 +1,9 @@
 import { createDefaultRetryPolicy } from "@intx/inference";
 import type { RetryDecision, RetryPolicy, RetrySituation } from "@intx/types/runtime";
-import { normalizeInferenceErrorForRetry } from "../inference-gateway-error.js";
+import {
+  normalizeInferenceErrorForRetry,
+  type InferenceErrorWithGoContext,
+} from "../inference-gateway-error.js";
 
 // Providers that enforce long-window quotas (e.g. monthly limits) set
 // Retry-After to days or weeks. The default policy trusts that value and
@@ -9,10 +12,32 @@ import { normalizeInferenceErrorForRetry } from "../inference-gateway-error.js";
 // so the user can switch providers or decide when to retry manually.
 const MAX_BLIND_WAIT_MS = 30_000;
 
-export function createCorbitsRetryPolicy(): RetryPolicy {
+export interface CorbitsRetryPolicyOptions {
+  /**
+   * Catalog provider id (e.g. xai/thegreataxios) stamped onto errors before
+   * normalize. Pass a getter when the live provider can change mid-session
+   * (e.g. `/model`); it is resolved on each retry decision.
+   */
+  providerId?: string | (() => string | undefined);
+}
+
+/**
+ * Corbits retry policy. When `providerId` is set, merges it onto the error
+ * before `normalizeInferenceErrorForRetry` so known-provider remappers (xAI
+ * short 429 → retryable, Go, Codex) can gate on context the harness does not
+ * attach to InferenceError today.
+ */
+export function createCorbitsRetryPolicy(options?: CorbitsRetryPolicyOptions): RetryPolicy {
   const defaultPolicy = createDefaultRetryPolicy();
   return (situation: RetrySituation): RetryDecision | Promise<RetryDecision> => {
-    const error = normalizeInferenceErrorForRetry(situation.error);
+    const raw = options?.providerId;
+    const stampedProviderId = typeof raw === "function" ? raw() : raw;
+    const incoming = situation.error as InferenceErrorWithGoContext;
+    const withProvider: InferenceErrorWithGoContext =
+      stampedProviderId !== undefined && incoming.providerId === undefined
+        ? { ...incoming, providerId: stampedProviderId }
+        : incoming;
+    const error = normalizeInferenceErrorForRetry(withProvider);
     if (
       error.category === "quota_exhausted" &&
       error.retryAfterMs !== undefined &&

src/inference-error-message.test.ts

@@ -37,4 +37,32 @@ describe("inferenceErrorMessage", () => {
     expect(line).not.toContain("Codex");
     expect(line).toBe("Quota exhausted — usage limit reached.");
   });
+
+  test("known-xAI short 429 shows rate-limit line, not Quota exhausted", () => {
+    const line = inferenceErrorMessage({
+      category: "quota_exhausted",
+      message: "Too Many Requests",
+      statusCode: 429,
+      providerId: "xai/thegreataxios",
+      raw: { error: { message: "Too Many Requests" } },
+    });
+    expect(line.toLowerCase()).toMatch(/rate limit/);
+    expect(line).not.toContain("Quota exhausted");
+  });
+
+  test("known-xAI quota body still shows Quota exhausted", () => {
+    const line = inferenceErrorMessage({
+      category: "quota_exhausted",
+      message: "You exceeded your current quota",
+      statusCode: 429,
+      providerId: "xai/thegreataxios",
+      raw: {
+        error: {
+          message: "You exceeded your current quota",
+          code: "insufficient_quota",
+        },
+      },
+    });
+    expect(line).toBe("Quota exhausted — usage limit reached.");
+  });
 });

src/inference-error-message.ts

@@ -14,6 +14,8 @@ import { codexProfileFromProviderName, isCodexProviderName } from "./config/code
 import {
   gatewayOverloadUserMessage,
   isGatewayOverloadInferenceError,
+  isXaiShortRateLimitInferenceError,
+  XAI_RATE_LIMIT_USER_MESSAGE,
   type InferenceErrorLike,
 } from "./inference-gateway-error.js";
 
@@ -90,6 +92,9 @@ function codexUsageLimitLine(error: InferenceErrorLike): string | undefined {
 /** One line describing the failure, falling back to the provider's own message. */
 export function inferenceErrorMessage(error: InferenceErrorLike): string {
   if (isGatewayOverloadInferenceError(error)) return gatewayOverloadUserMessage(error);
+  // Dual-path: harness may still emit intx's quota_exhausted for a known-xAI
+  // short 429; FRIENDLY_BY_CATEGORY would otherwise say "Quota exhausted".
+  if (isXaiShortRateLimitInferenceError(error)) return XAI_RATE_LIMIT_USER_MESSAGE;
 
   const category = classifyInferenceErrorCategory(error);
   if (category === "quota_exhausted") {

src/inference-gateway-error.test.ts

@@ -266,4 +266,56 @@ describe("normalizeInferenceErrorForRetry", () => {
     const normalized = normalizeInferenceErrorForRetry(error);
     expect(normalized).toBe(error);
   });
+
+  test("known-xAI bare 429 reclassifies as retryable", () => {
+    const bare = {
+      category: "quota_exhausted" as const,
+      message: "Too Many Requests",
+      statusCode: 429,
+      retryAfterMs: 45_000,
+      raw: { error: { message: "Too Many Requests" } },
+    };
+
+    // Without xAI context, leave intx's classification alone.
+    expect(normalizeInferenceErrorForRetry(bare)).toEqual(bare);
+
+    const viaProviderId = normalizeInferenceErrorForRetry({
+      ...bare,
+      providerId: "xai/thegreataxios",
+    });
+    expect(viaProviderId.category).toBe("retryable");
+    expect(viaProviderId.retryAfterMs).toBe(45_000);
+    expect(viaProviderId.message.toLowerCase()).toMatch(/rate limit/);
+  });
+
+  test("known-xAI 429 with usage/quota body stays quota_exhausted", () => {
+    const normalized = normalizeInferenceErrorForRetry({
+      category: "quota_exhausted",
+      message: "Too Many Requests",
+      statusCode: 429,
+      providerId: "xai/thegreataxios",
+      retryAfterMs: 86_400_000,
+      raw: {
+        error: {
+          message: "You exceeded your current quota, please check your plan and billing details.",
+          type: "insufficient_quota",
+          code: "insufficient_quota",
+        },
+      },
+    });
+    expect(normalized.category).toBe("quota_exhausted");
+    expect(normalized.retryAfterMs).toBe(86_400_000);
+  });
+
+  test("unknown provider bare 429 stays quota_exhausted", () => {
+    const err = {
+      category: "quota_exhausted" as const,
+      message: "Too Many Requests",
+      statusCode: 429,
+      providerId: "openai",
+      retryAfterMs: 5_000,
+      raw: { error: { message: "Too Many Requests" } },
+    };
+    expect(normalizeInferenceErrorForRetry(err)).toEqual(err);
+  });
 });