fix(indexer): classify storage read failures

reference-implementations/dacs-directory/README.md

@@ -167,6 +167,13 @@ means the scanner could not read enough data to establish that the locator conta
 DACS artifact; it does not attribute a publishing failure to an agent. Raw exceptions,
 payloads, internal URLs and stack traces are never returned.
 
+Storage-read diagnostics distinguish `STORAGE_NOT_FOUND`, `STORAGE_NOT_PUBLIC`,
+`STORAGE_RPC_UNAVAILABLE`, and `STORAGE_INVALID_RESPONSE`. Missing and non-public
+locators are terminal until a later chain replay observes them again; transient node
+and response failures retain bounded retries. `STORAGE_NOT_FOUND` is operational
+diagnostic evidence only: under the current Demos mapping it is never authoritative
+DACS-5 absence evidence and cannot satisfy BB-8.
+
 ## Discovery — three channels
 
 1. **Registration** (`/register` UI or `POST /api/dacs/register`): bounded pointer sets,

reference-implementations/dacs-directory/src/catalog/reindexCore.ts

@@ -76,7 +76,7 @@ export async function reindexAll(opts: ReindexOptions = {}): Promise<ReindexSumm
     const previousCursor = state.lastSeenTxId;
     clearChainDerivedArtifacts();
     state = {
-      schemaVersion: 7,
+      schemaVersion: 8,
       lastSeenTxId: 0,
       lastChainTip: observedChainTip,
       listings: {},
@@ -94,14 +94,15 @@ export async function reindexAll(opts: ReindexOptions = {}): Promise<ReindexSumm
         `behind cursor ${previousCursor}; cleared chain-derived cache and restarting from genesis`,
     );
   }
-  // v7 replays history to discover BB-4-valid DACS-5 BundleBindings. It also
-  // retains v6's consensus-time and v5's revocation-marker binding replays.
-  const needsBindingBackfill = state.schemaVersion !== 7;
+  // v8 replays history to classify stable storage failures without the legacy
+  // STORAGE_UNREADABLE bucket. It retains v7's BundleBinding, v6's consensus-
+  // time and v5's revocation-marker binding replays.
+  const needsHistoryReplay = state.schemaVersion !== 8;
   const configuredMax = Number(process.env.DACS_SCAN_MAX_TXS ?? 100000);
   const maxTxs = Number.isSafeInteger(configuredMax) && configuredMax > 0 ? configuredMax : 100000;
   const configuredOverlap = Number(process.env.DACS_SCAN_REPLAY_DEPTH ?? 2);
   const overlap = Number.isSafeInteger(configuredOverlap) && configuredOverlap >= 0 ? configuredOverlap : 2;
-  const sinceTxId = needsBindingBackfill ? 0 : Math.max(0, state.lastSeenTxId - overlap);
+  const sinceTxId = needsHistoryReplay ? 0 : Math.max(0, state.lastSeenTxId - overlap);
   state.verifiedRevocations ??= {};
   state.bundleBindings ??= {};
   state.bundleBindingOverflow ??= [];
@@ -150,7 +151,7 @@ export async function reindexAll(opts: ReindexOptions = {}): Promise<ReindexSumm
   for (const [jobId, deal] of scan.deals) state.deals[jobId] = deal;
   state.programs ??= {};
   for (const [key, address] of scan.programs) state.programs[key] = address;
-  if (needsBindingBackfill) state.revocations = {};
+  if (needsHistoryReplay) state.revocations = {};
   state.revocations ??= {};
   let revocationCandidatesTruncated = scan.revocationCandidatesTruncated;
   for (const [hash, addresses] of scan.revocations) {
@@ -186,7 +187,10 @@ export async function reindexAll(opts: ReindexOptions = {}): Promise<ReindexSumm
     revocationCandidatesTruncated += bounded.truncated;
   }
   for (const observation of scan.observations) recordArtifact(observation);
-  for (const failure of scan.failures) recordArtifactFailure(failure.locator, failure.kind, failure.code, failure.message);
+  for (const failure of scan.failures) {
+    const stable = failure.code === "STORAGE_NOT_FOUND" || failure.code === "STORAGE_NOT_PUBLIC";
+    recordArtifactFailure(failure.locator, failure.kind, failure.code, failure.message, stable ? 1 : 5);
+  }
   // One bounded age-prune batch per pass keeps failure telemetry from growing
   // without limit (issue #51) while never becoming a blocking maintenance job.
   pruneFailureHistory();
@@ -238,7 +242,7 @@ export async function reindexAll(opts: ReindexOptions = {}): Promise<ReindexSumm
   else state.cursorAdvancedAt ??= Date.now();
   state.lastSeenTxId = nextCursor;
   state.lastChainTip = scan.chainTip;
-  state.schemaVersion = 7;
+  state.schemaVersion = 8;
   saveScanState(state);
   finishScanRun(runId, { toTx: state.lastSeenTxId, chainTip: scan.chainTip, txs: scan.txsScanned,
     artifacts: scan.observations.length, rejected: scan.failures.length });

reference-implementations/dacs-directory/src/catalog/scan.ts

@@ -60,17 +60,65 @@ interface StorageRead {
   owner?: string;
   programName?: string;
   data?: Record<string, unknown>;
+  errorCode?: string;
+  failureCode?: StorageReadFailureCode;
 }
 
-async function readStorage(address: string, attempts = 3): Promise<StorageRead | null> {
-  for (let attempt = 1; attempt <= attempts; attempt++) try {
-    const res = await fetch(`${RPC}/storage-program/${address}`, {
-      signal: AbortSignal.timeout(15_000),
-    });
-    if (!res.ok) throw new Error(`HTTP ${res.status}`);
-    return (await res.json()) as StorageRead;
-  } catch { if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, 100 * 2 ** (attempt - 1))); }
-  return null;
+export type StorageReadFailureCode =
+  | "STORAGE_NOT_FOUND"
+  | "STORAGE_NOT_PUBLIC"
+  | "STORAGE_RPC_UNAVAILABLE"
+  | "STORAGE_INVALID_RESPONSE";
+
+/** Safe cause classification; response bodies and upstream text are never persisted. */
+export function storageReadFailureCode(status: number, errorCode?: string): StorageReadFailureCode {
+  const normalized = typeof errorCode === "string" ? errorCode.trim().toUpperCase() : undefined;
+  if (status === 404) {
+    return "STORAGE_NOT_FOUND";
+  }
+  if (status === 401 || status === 403) {
+    return "STORAGE_NOT_PUBLIC";
+  }
+  // A server-side or transport-class status remains retryable even if an
+  // untrusted error body happens to carry a terminal-looking code.
+  if (status >= 500 || status < 100) return "STORAGE_RPC_UNAVAILABLE";
+  if (normalized === "NOT_FOUND" || normalized === "STORAGE_NOT_FOUND" || normalized === "PROGRAM_NOT_FOUND") {
+    return "STORAGE_NOT_FOUND";
+  }
+  if (normalized === "PERMISSION_DENIED" || normalized === "UNAUTHORIZED") return "STORAGE_NOT_PUBLIC";
+  return status >= 200 && status < 300 ? "STORAGE_INVALID_RESPONSE" : "STORAGE_RPC_UNAVAILABLE";
+}
+
+export async function readStorage(address: string, attempts = 3): Promise<StorageRead> {
+  let lastFailure: StorageReadFailureCode = "STORAGE_RPC_UNAVAILABLE";
+  const boundedAttempts = Number.isSafeInteger(attempts) && attempts >= 1 && attempts <= 5 ? attempts : 3;
+  for (let attempt = 1; attempt <= boundedAttempts; attempt++) {
+    let res: Response;
+    try {
+      res = await fetch(`${RPC}/storage-program/${address}`, {
+        signal: AbortSignal.timeout(15_000),
+      });
+    } catch {
+      lastFailure = "STORAGE_RPC_UNAVAILABLE";
+      if (attempt < boundedAttempts) await new Promise((resolve) => setTimeout(resolve, 100 * 2 ** (attempt - 1)));
+      continue;
+    }
+
+    const statusFailure = storageReadFailureCode(res.status);
+    if (statusFailure === "STORAGE_NOT_FOUND" || statusFailure === "STORAGE_NOT_PUBLIC") {
+      return { success: false, failureCode: statusFailure };
+    }
+    let body: StorageRead | null = null;
+    try { body = (await res.json()) as StorageRead; } catch { /* classified below */ }
+    const failure = storageReadFailureCode(res.status, body?.errorCode);
+    if (failure === "STORAGE_NOT_FOUND" || failure === "STORAGE_NOT_PUBLIC") {
+      return { success: false, failureCode: failure };
+    }
+    if (res.ok && body?.success && body.programName && body.owner) return body;
+    lastFailure = failure;
+    if (attempt < boundedAttempts) await new Promise((resolve) => setTimeout(resolve, 100 * 2 ** (attempt - 1)));
+  }
+  return { success: false, failureCode: lastFailure };
 }
 
 /**
@@ -421,7 +469,15 @@ export async function scanChain(
 
   for (const address of addresses) {
     const read = await readStorage(address);
-    if (!read?.success || !read.programName || !read.owner) { failures.push({ locator: address, kind: "unknown", code: "STORAGE_UNREADABLE", message: "storage program could not be read after retries" }); continue; }
+    if (!read?.success || !read.programName || !read.owner) {
+      failures.push({
+        locator: address,
+        kind: "unknown",
+        code: read?.failureCode ?? "STORAGE_INVALID_RESPONSE",
+        message: "storage program could not be read under the public indexer policy",
+      });
+      continue;
+    }
     const name = read.programName;
     programs.set(programBindingKey(read.owner, name), address);
     const data = read.data as Record<string, unknown> | undefined;

reference-implementations/dacs-directory/src/catalog/store.ts

@@ -423,6 +423,10 @@ export function finishScanRun(id: number, values: { toTx: number; chainTip?: num
 }
 const PUBLIC_FAILURES: Record<string, string> = {
   STORAGE_UNREADABLE: "The storage program could not be read after repeated attempts. Confirm the locator exists and is publicly readable before retrying.",
+  STORAGE_NOT_FOUND: "No storage program was found at this locator. The response is operational evidence only and is not authoritative DACS absence evidence.",
+  STORAGE_NOT_PUBLIC: "A storage program exists at this locator but is not publicly readable. Publish it for unauthenticated reads before retrying.",
+  STORAGE_RPC_UNAVAILABLE: "The storage program could not be read because the public node was unavailable after bounded retries.",
+  STORAGE_INVALID_RESPONSE: "The public node returned a response that did not satisfy the storage-read contract after bounded retries.",
 };
 const DACS_ARTIFACT_KINDS = new Set(["listing", "listing-revocation", "bundle", "agreement", "evidence", "verify-result", "composite", "rating"]);
 

reference-implementations/dacs-directory/test/dead-letter-diagnostics.test.ts

@@ -88,6 +88,26 @@ test("dead-letter diagnostics are safe, bounded, filterable, and recoverable", (
   assert.equal(failedAgain.classification, "dacs-artifact");
 });
 
+test("storage cause diagnostics are actionable without claiming DACS absence", () => {
+  const missing = locator("a");
+  const privateLocator = locator("b");
+  store.recordArtifactFailure(missing, "unknown", "STORAGE_NOT_FOUND", "raw 404 response", 1);
+  store.recordArtifactFailure(privateLocator, "unknown", "STORAGE_NOT_PUBLIC", "raw 403 response", 1);
+
+  const missingDiagnostic = store.indexerDiagnostics({ deadLetterLocator: missing })
+    .deadLetterDiagnostics.items[0];
+  assert.equal(missingDiagnostic.code, "STORAGE_NOT_FOUND");
+  assert.match(missingDiagnostic.message, /operational evidence only/);
+  assert.match(missingDiagnostic.message, /not authoritative DACS absence evidence/);
+  assert.doesNotMatch(JSON.stringify(missingDiagnostic), /raw 404 response/);
+
+  const privateDiagnostic = store.indexerDiagnostics({ deadLetterLocator: privateLocator })
+    .deadLetterDiagnostics.items[0];
+  assert.equal(privateDiagnostic.code, "STORAGE_NOT_PUBLIC");
+  assert.match(privateDiagnostic.message, /not publicly readable/);
+  assert.doesNotMatch(JSON.stringify(privateDiagnostic), /raw 403 response/);
+});
+
 test("listing binding rejections are persistent, public-safe, filterable, and recoverable", () => {
   const target = locator("7");
   const claim = `did:demos:agent:${"7".repeat(64)}`;

reference-implementations/dacs-directory/test/reindex-chain-reset.test.ts

@@ -55,6 +55,7 @@ test("reindex clears replaced-chain discoveries and rescans from genesis in one
   const result = await reindexAll({ log: (line) => logs.push(line) });
 
   assert.equal(result.cursor, 3, "finality depth leaves the newest two transactions for replay");
+  assert.equal(store.loadScanState().schemaVersion, 8);
   assert.deepEqual(store.loadScanState().listings, {});
   assert.equal(store.indexerDiagnostics().deadLetters, 0);
   assert.match(logs[0] ?? "", /chain replacement detected/);

reference-implementations/dacs-directory/test/scan-addresses.test.ts

@@ -1,7 +1,11 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { collectNativeStorageAddresses } from "../src/catalog/scan.js";
+import {
+  collectNativeStorageAddresses,
+  readStorage,
+  storageReadFailureCode,
+} from "../src/catalog/scan.js";
 
 const native = `stor-${"a".repeat(40)}`;
 const logical = `stor-${"b".repeat(64)}`;
@@ -27,3 +31,41 @@ test("scanner requires a hex boundary after the native locator", () => {
 
   assert.deepEqual([...addresses], [native]);
 });
+
+test("storage failures are classified into public-safe operational causes", () => {
+  assert.equal(storageReadFailureCode(404), "STORAGE_NOT_FOUND");
+  assert.equal(storageReadFailureCode(403), "STORAGE_NOT_PUBLIC");
+  assert.equal(storageReadFailureCode(200, "PERMISSION_DENIED"), "STORAGE_NOT_PUBLIC");
+  assert.equal(storageReadFailureCode(503), "STORAGE_RPC_UNAVAILABLE");
+  assert.equal(storageReadFailureCode(503, "NOT_FOUND"), "STORAGE_RPC_UNAVAILABLE");
+  assert.equal(storageReadFailureCode(200), "STORAGE_INVALID_RESPONSE");
+});
+
+test("terminal storage failures skip retries while transient failures remain bounded", async () => {
+  const originalFetch = globalThis.fetch;
+  let calls = 0;
+  try {
+    globalThis.fetch = async () => {
+      calls++;
+      return new Response(null, { status: 404 });
+    };
+    assert.deepEqual(await readStorage(native, 3), {
+      success: false,
+      failureCode: "STORAGE_NOT_FOUND",
+    });
+    assert.equal(calls, 1);
+
+    calls = 0;
+    globalThis.fetch = async () => {
+      calls++;
+      return new Response("temporarily unavailable", { status: 503 });
+    };
+    assert.deepEqual(await readStorage(native, 2), {
+      success: false,
+      failureCode: "STORAGE_RPC_UNAVAILABLE",
+    });
+    assert.equal(calls, 2);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});