From 457fccf85615d7fa423dbd6c422be2f9b5881e87 Mon Sep 17 00:00:00 2001 From: logseq Date: Mon, 3 Aug 2026 10:02:15 +0200 Subject: [PATCH] Cloudflare collector: degrade per-call, don't discard the whole run collect() threw on the first non-OK response from any account-level cf() listing call (routes, dns_records, kv, r2, d1, workers/scripts, workers/domains), discarding everything already gathered in that run - zones, DNS, workers, KV, R2, D1, all of it - the moment any ONE of those hit a scope gap. Found running this against a real Cloudflare account with a narrowly- scoped API token (zone/DNS read access, but not the account-level Workers/KV/R2/D1 APIs): the very first sync threw on the per-zone workers/routes call and produced zero Cloudflare assets, even though the account/zones/DNS enumeration had already succeeded moments before. A token scoped to only some of Cloudflare's many permission groups is a completely ordinary setup (least-privilege tokens created for a specific deploy pipeline, for instance) - not a misconfiguration this collector should treat as fatal. This contradicts the collector's own stated doctrine ("a partial page, an API error, a rate limit -> PARTIAL run, no sweep, prior state intact" - Store.ts's sweep gate exists precisely to make this safe), which the bare cf() throw never actually implemented for this failure class. Added a cfSoft() wrapper: catches a failure on a single listing call, logs it, marks the run `partial` (so Store.applyRun's sweep gate correctly withholds sweeping instead of expiring real assets over incomplete data), and returns an empty array so the rest of the enumeration proceeds. Applied to every account/zone-scoped listing call except /accounts and /zones themselves, which stay hard failures - without an account or the zone list, nothing else in the collector is meaningful anyway. Verified against the real account described above: before the fix, `atlas sync cloudflare` returned 0 assets on a fresh graph. After, it returns the full zone/DNS enumeration (PARTIAL, correctly not swept) while cleanly logging which specific calls degraded and why. --- .../LIFEOS/ATLAS/collectors/Cloudflare.ts | 50 ++++++++++++++++--- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/LifeOS/install/LIFEOS/ATLAS/collectors/Cloudflare.ts b/LifeOS/install/LIFEOS/ATLAS/collectors/Cloudflare.ts index 2e132a8f5a..17dcfa2593 100644 --- a/LifeOS/install/LIFEOS/ATLAS/collectors/Cloudflare.ts +++ b/LifeOS/install/LIFEOS/ATLAS/collectors/Cloudflare.ts @@ -52,6 +52,28 @@ async function cfOne(path: string, tok: string): Promise { return body.success ? body.result : null; } +/** + * Degrading wrapper around cf() for listing calls that can legitimately hit a + * token-scope gap (found 2026-08-03: a narrowly-scoped CLOUDFLARE_API_TOKEN + * threw on workers/routes 403, then storage/kv/namespaces 401, discarding + * everything already gathered each time). One missing permission on one + * resource type must degrade that signal only, never the whole collector — + * matches Atlas's own doctrine ("a partial page, an API error → PARTIAL run, + * no sweep, prior state intact"), which the bare cf() calls didn't actually + * implement for this failure class. Sets `partial` via the caller-owned flag + * so the run reports incomplete (never sweeps) instead of claiming success + * over data it couldn't fully see. + */ +async function cfSoft(path: string, tok: string, label: string, onFail: () => void): Promise { + try { + return await cf(path, tok); + } catch (err) { + onFail(); + console.error(`[cloudflare] ${label} failed: ${(err as Error).message}`); + return []; + } +} + /** Bounded-concurrency map — keeps the per-worker fan-out from hammering the API. */ async function mapPool(items: T[], limit: number, fn: (t: T) => Promise): Promise { const out: R[] = new Array(items.length); @@ -103,6 +125,15 @@ export const cloudflare: Collector = { if (tok === null) return { complete: false, assets: [], edges: [] }; const assets: AssetObs[] = []; const edges: EdgeObs[] = []; + // Set when a per-zone/per-resource call fails on a token-scope gap rather than + // a wholesale outage — everything gathered so far (and after) is still real + // and worth writing, but the run can't claim enumeration-complete, so it must + // never sweep (Store.applyRun's gate). A throw here would discard the WHOLE + // collector's output (zones/DNS/workers/KV/R2/D1) over one missing permission + // on one zone — the 2026-08-03 finding: a narrowly-scoped CLOUDFLARE_API_TOKEN + // lacking "Workers Routes:Read" turned a single zone's routes lookup into a + // total loss instead of a partial one. + let partial = false; const accounts = (await cf("/accounts", tok)) as Array<{ id: string; name: string }>; const acct = accounts[0]; @@ -116,13 +147,16 @@ export const cloudflare: Collector = { for (const z of zones) { assets.push({ kind: "domain", key: `domain:${z.name}`, name: z.name, attrs: { is_zone: true, zone_status: z.status } }); // Zone worker ROUTES (pattern-based, not custom domains) — a real serving path. - const routes = ((await cf(`/zones/${z.id}/workers/routes`, tok)) as Array<{ pattern: string; script?: string }>) ?? []; + // Token-scope gaps are per-zone (a token can be scoped to some zones' Workers + // Routes permission and not others), so one zone's 403 degrades only this + // signal for that zone, never the rest of the collector. + const routes = (await cfSoft(`/zones/${z.id}/workers/routes`, tok, `routes lookup for zone ${z.name}`, () => { partial = true; })) as Array<{ pattern: string; script?: string }>; for (const rt of routes) { if (!rt.script) continue; const host = rt.pattern.replace(/^https?:\/\//, "").split("/")[0].replace(/^\*\./, ""); if (host) edges.push({ kind: "ROUTE", srcKey: `cloudflare:worker:${rt.script}`, dstKey: `domain:${host}`, srcKind: "worker", dstKind: "domain" }); } - const records = (await cf(`/zones/${z.id}/dns_records`, tok)) as Array<{ type: string; name: string; content: string; proxied?: boolean }>; + const records = (await cfSoft(`/zones/${z.id}/dns_records`, tok, `DNS records for zone ${z.name}`, () => { partial = true; })) as Array<{ type: string; name: string; content: string; proxied?: boolean }>; for (const r of records) { const key = `dns:${z.name}/${r.type}/${r.name}`; assets.push({ kind: "dns_record", key, name: `${r.type} ${r.name}`, attrs: { content: r.content, proxied: r.proxied ?? false } }); @@ -134,22 +168,22 @@ export const cloudflare: Collector = { } } - const kv = (await cf(`/accounts/${acct.id}/storage/kv/namespaces`, tok)) as Array<{ id: string; title: string }>; + const kv = (await cfSoft(`/accounts/${acct.id}/storage/kv/namespaces`, tok, "KV namespaces", () => { partial = true; })) as Array<{ id: string; title: string }>; for (const ns of kv) assets.push({ kind: "kv_namespace", key: `cloudflare:kv:${ns.id}`, name: ns.title }); - const r2 = (await cf(`/accounts/${acct.id}/r2/buckets`, tok)) as Array<{ name?: string; buckets?: Array<{ name: string }> }>; + const r2 = (await cfSoft(`/accounts/${acct.id}/r2/buckets`, tok, "R2 buckets", () => { partial = true; })) as Array<{ name?: string; buckets?: Array<{ name: string }> }>; // Endpoint wraps the list: result = { buckets: [...] } → cf() returns [wrapper]. const buckets = r2.flatMap((item) => (item.buckets ? item.buckets : item.name ? [{ name: item.name }] : [])); for (const b of buckets) assets.push({ kind: "r2_bucket", key: `cloudflare:r2:${b.name}`, name: b.name }); - const d1 = (await cf(`/accounts/${acct.id}/d1/database`, tok)) as Array<{ uuid: string; name: string }>; + const d1 = (await cfSoft(`/accounts/${acct.id}/d1/database`, tok, "D1 databases", () => { partial = true; })) as Array<{ uuid: string; name: string }>; for (const db of d1) assets.push({ kind: "d1_database", key: `cloudflare:d1:${db.name}`, name: db.name, attrs: { uuid: db.uuid } }); // D1 assets are keyed by NAME but bindings reference the uuid — map it before the // worker loop so a binding can be resolved to the asset it actually points at. const d1ByUuid = new Map(d1.map((db) => [db.uuid, db.name])); - const workers = (await cf(`/accounts/${acct.id}/workers/scripts`, tok)) as Array<{ id: string; modified_on?: string }>; + const workers = (await cfSoft(`/accounts/${acct.id}/workers/scripts`, tok, "worker scripts", () => { partial = true; })) as Array<{ id: string; modified_on?: string }>; // Per-worker wiring: cron schedules, workers.dev enablement, and bindings // (service → CALLS edge, queue → consumer flag). Bounded fan-out. await mapPool(workers, 8, async (w) => { @@ -210,7 +244,7 @@ export const cloudflare: Collector = { } }); - const wDomains = (await cf(`/accounts/${acct.id}/workers/domains`, tok)) as Array<{ hostname: string; service: string; zone_name: string }>; + const wDomains = (await cfSoft(`/accounts/${acct.id}/workers/domains`, tok, "worker custom domains", () => { partial = true; })) as Array<{ hostname: string; service: string; zone_name: string }>; for (const d of wDomains) { assets.push({ kind: "domain", key: `domain:${d.hostname}`, name: d.hostname, attrs: { is_zone: false, worker_custom_domain: true } }); edges.push({ kind: "SERVES", srcKey: `cloudflare:worker:${d.service}`, dstKey: `domain:${d.hostname}`, srcKind: "worker", dstKind: "domain" }); @@ -219,6 +253,6 @@ export const cloudflare: Collector = { } } - return { complete: true, assets, edges }; + return { complete: !partial, assets, edges }; }, };