From a1c929e39c358fc78e50cd418b05850d67e0e8ac Mon Sep 17 00:00:00 2001 From: random block Date: Wed, 5 Aug 2026 10:30:52 +0100 Subject: [PATCH] feat(directory): publish safe reachability hints --- .../dacs-directory/README.md | 2 + .../dacs-directory/app/llms.txt/route.ts | 2 +- .../[seller]/[listingId]/[version]/page.tsx | 3 +- .../src/catalog/boundedHttps.ts | 229 ++++++++++++++++++ .../dacs-directory/src/catalog/contracts.ts | 10 + .../src/catalog/reachability.ts | 115 +++++++++ .../src/catalog/reachabilityStatus.ts | 17 ++ .../dacs-directory/src/catalog/reindexCore.ts | 4 + .../dacs-directory/src/catalog/types.ts | 10 + .../dacs-directory/src/catalog/wellknown.ts | 193 +-------------- .../components/directory-evidence-state.ts | 26 +- .../test/directory-evidence-state.test.ts | 30 +++ .../dacs-directory/test/reachability.test.ts | 102 ++++++++ 13 files changed, 550 insertions(+), 193 deletions(-) create mode 100644 reference-implementations/dacs-directory/src/catalog/boundedHttps.ts create mode 100644 reference-implementations/dacs-directory/src/catalog/reachability.ts create mode 100644 reference-implementations/dacs-directory/src/catalog/reachabilityStatus.ts create mode 100644 reference-implementations/dacs-directory/test/reachability.test.ts diff --git a/reference-implementations/dacs-directory/README.md b/reference-implementations/dacs-directory/README.md index eb16351..02687de 100644 --- a/reference-implementations/dacs-directory/README.md +++ b/reference-implementations/dacs-directory/README.md @@ -118,6 +118,8 @@ vendor directory. | `DACS_SCAN_REPLAY_DEPTH` | No | Finalized transaction overlap replayed on every pass; defaults to `2` | | `DACS_INDEX_INTERVAL_SECONDS` | No | Seconds between production reindex passes; defaults to `900` | | `DACS_CURSOR_STALL_SECONDS` | No | Cursor-stall alert threshold; defaults to twice the valid index interval (minimum `300`, default `1800`, maximum `86400`) | +| `DACS_REACHABILITY_MAX_PROBES` | No | Maximum due listing surfaces probed per reindex; defaults to `20` (bounded to `1..100`) | +| `DACS_REACHABILITY_CONCURRENCY` | No | Concurrent pinned HTTPS reachability probes; defaults to `5` (bounded to `1..10`) | | `DACS_RECIPE_POLICIES` | For tier elevation | JSON array of version-pinned DACS-2 recipe policies (`scheme`, `recipeVersion`, `methods`, `defaultMaxAgeSec`, `availability`, `trustedResultSigners`); absent/invalid policy fails closed to `self-declared` | | `DACS_TRUST_PROXY` | No | Set to `1` only behind a trusted proxy that overwrites client-IP headers; otherwise the in-process rate limiter is disabled and the deployment must enforce its edge limit | | `NEXT_PUBLIC_DIRECTORY_URL` | Production | Public origin used by canonical URLs, sitemap, `llms.txt`, and machine-discovery documents; defaults to `http://localhost:3400`, which silently poisons production canonical URLs and the sitemap — the server logs a warning when unset in production | diff --git a/reference-implementations/dacs-directory/app/llms.txt/route.ts b/reference-implementations/dacs-directory/app/llms.txt/route.ts index fa2ba73..c65ea8a 100644 --- a/reference-implementations/dacs-directory/app/llms.txt/route.ts +++ b/reference-implementations/dacs-directory/app/llms.txt/route.ts @@ -21,7 +21,7 @@ DACS Directory discovers signed, chain-anchored agent services for humans and so GET ${base}/api/dacs/listings supports category, repeated tag, credential, primaryClaim, identityTier, rail, priceMax, minCompletionRate, minRating, cursor and limit. Directory extensions are q and profile. Every result includes an anchor and contentHash; dereference the listing-detail URL before engaging. identityTier is derived only from fresh, passing, version-pinned DACS-2 verifiedBy evidence; missing recipe policy fails closed to self-declared. -artifactProfile=dacs-v0.1 identifies a current structured Listing. artifactProfile=legacy-sdk-v0.1 identifies the pinned SDK compatibility shape. artifactProfile=fixture-listing identifies a local fixture that is not signed or chain-anchored. Treat a missing profile as legacy for backward compatibility. A publicEndpoint, when present, is an advertised engagement route rather than a trust anchor. +artifactProfile=dacs-v0.1 identifies a current structured Listing. artifactProfile=legacy-sdk-v0.1 identifies the pinned SDK compatibility shape. artifactProfile=fixture-listing identifies a local fixture that is not signed or chain-anchored. Treat a missing profile as legacy for backward compatibility. A publicEndpoint, when present, is an advertised engagement route rather than a trust anchor. reachabilityHint is a time-stamped, non-authoritative catalog probe; treat stale hints as unknown and never use them for validity, identity, revocation, or reputation decisions. ## Publication diagnostics diff --git a/reference-implementations/dacs-directory/app/service/[seller]/[listingId]/[version]/page.tsx b/reference-implementations/dacs-directory/app/service/[seller]/[listingId]/[version]/page.tsx index fd6e883..7ff1d04 100644 --- a/reference-implementations/dacs-directory/app/service/[seller]/[listingId]/[version]/page.tsx +++ b/reference-implementations/dacs-directory/app/service/[seller]/[listingId]/[version]/page.tsx @@ -84,6 +84,7 @@ export default async function ServicePage({ params }: { params: Promise {identity.label} {evidence.listing.label} {seller.ownerRegistered && owner-registered} + {engagementEndpoint && {evidence.reachability.label}} {!isFixtureListing && !seller.ownerRegistered && seller.discovered && discovered on-chain} {isFixtureListing && not chain anchored} {!isFixtureListing && !seller.ownerRegistered && !seller.discovered && ( @@ -124,7 +125,7 @@ export default async function ServicePage({ params }: { params: Promise
  • {seller.cci.length ? "✓" : "–"}
    Identity links

    {seller.cci.length ? `${seller.cci.length} GCR identity link${seller.cci.length === 1 ? "" : "s"}; no fresh DACS-2 verification resolved` : "No linked identities beyond the signing key"}

  • {isFixtureListing ? "–" : "✓"}
    Listing

    {isFixtureListing ? "Fixture machine contract and content hash match; no chain anchor claimed" : "Signature and chain anchor verified"}

  • {engagementEndpoint ? "•" : "–"}
    Endpoint declaration

    {evidence.endpoint.label}; this says nothing about availability

    • -
  • –
    Reachability

    {evidence.reachability.label}

    • +
  • {evidence.reachability.kind === "reachable" ? "✓" : "–"}
    Reachability

    {evidence.reachability.label}

  • {evidence.deals.completed ? "✓" : "–"}
    Two-sided deal evidence

    {evidence.deals.label}. {evidence.deals.explanation}.

    • diff --git a/reference-implementations/dacs-directory/src/catalog/boundedHttps.ts b/reference-implementations/dacs-directory/src/catalog/boundedHttps.ts new file mode 100644 index 0000000..0d9dc28 --- /dev/null +++ b/reference-implementations/dacs-directory/src/catalog/boundedHttps.ts @@ -0,0 +1,229 @@ +import { lookup } from "node:dns/promises"; +import { request as httpsRequest } from "node:https"; +import { isIP } from "node:net"; +import type { LookupFunction } from "node:net"; + +const DEFAULT_MAX_BYTES = 1024 * 1024; +const DEFAULT_MAX_REDIRECTS = 3; +const DEFAULT_TIMEOUT_MS = 15_000; + +function ipv6Words(address: string): number[] | null { + let input = address.toLowerCase().split("%", 1)[0]; + const dotted = input.match(/^(.*:)(\d{1,3}(?:\.\d{1,3}){3})$/); + if (dotted) { + const octets = dotted[2].split(".").map(Number); + if (octets.some((part) => part < 0 || part > 255)) return null; + input = `${dotted[1]}${((octets[0] << 8) | octets[1]).toString(16)}:${ + ((octets[2] << 8) | octets[3]).toString(16) + }`; + } + const halves = input.split("::"); + if (halves.length > 2) return null; + const parseHalf = (half: string) => half ? half.split(":").map((word) => Number.parseInt(word, 16)) : []; + const left = parseHalf(halves[0]); + const right = halves.length === 2 ? parseHalf(halves[1]) : []; + if ([...left, ...right].some((word) => !Number.isInteger(word) || word < 0 || word > 0xffff)) return null; + if (halves.length === 1) return left.length === 8 ? left : null; + const zeros = 8 - left.length - right.length; + return zeros >= 1 ? [...left, ...Array(zeros).fill(0), ...right] : null; +} + +/** Conservative global-unicast policy for untrusted outbound targets. */ +export function isPrivateAddress(address: string): boolean { + if (isIP(address) === 4) { + const p = address.split(".").map(Number); + return ( + p[0] === 0 || p[0] === 10 || p[0] === 127 || + (p[0] === 100 && p[1] >= 64 && p[1] <= 127) || + (p[0] === 169 && p[1] === 254) || + (p[0] === 172 && p[1] >= 16 && p[1] <= 31) || + (p[0] === 192 && (p[1] === 0 || p[1] === 168)) || + (p[0] === 198 && (p[1] === 18 || p[1] === 19 || (p[1] === 51 && p[2] === 100))) || + (p[0] === 203 && p[1] === 0 && p[2] === 113) || + p[0] >= 224 + ); + } + if (isIP(address) === 6) { + const words = ipv6Words(address); + if (!words) return true; + if (words.slice(0, 5).every((word) => word === 0) && words[5] === 0xffff) { + return isPrivateAddress( + `${words[6] >> 8}.${words[6] & 0xff}.${words[7] >> 8}.${words[7] & 0xff}`, + ); + } + // Accept only global-unicast 2000::/3, excluding reserved/documentation + // and transition assignments that can tunnel an unchecked IPv4 target. + if ((words[0] & 0xe000) !== 0x2000) return true; + if (words[0] === 0x2001 && (words[1] & 0xfe00) === 0) return true; + if (words[0] === 0x2001 && words[1] === 0x0db8) return true; + if (words[0] === 0x2002) return true; + if (words[0] === 0x3fff && (words[1] & 0xf000) === 0) return true; + return false; + } + return true; +} + +export class OutboundTargetError extends Error {} + +export interface VettedUrl { + url: URL; + /** The specific resolved address the caller MUST connect to. */ + ip: string; +} + +/** Resolve, reject every non-public answer, and pin one approved address. */ +export async function validatePublicHttpsUrl(raw: string): Promise { + let url: URL; + try { url = new URL(raw); } catch { throw new OutboundTargetError("invalid URL"); } + if (url.href.length > 2_048 || url.protocol !== "https:" || url.username || url.password || + (url.port && url.port !== "443")) { + throw new OutboundTargetError("outbound URLs must use public HTTPS on port 443"); + } + const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, ""); + if (!hostname || hostname === "localhost" || hostname.endsWith(".localhost") || hostname.endsWith(".local")) { + throw new OutboundTargetError("local hostnames are not allowed"); + } + let resolved: Array<{ address: string; family: number }>; + if (isIP(hostname)) { + resolved = [{ address: hostname, family: isIP(hostname) }]; + } else { + try { resolved = await lookup(hostname, { all: true, verbatim: true }); } + catch (error) { throw new Error("target DNS lookup failed", { cause: error }); } + } + if (resolved.length === 0 || resolved.some((record) => isPrivateAddress(record.address))) { + throw new OutboundTargetError("URL resolves to a non-public address"); + } + if (!isIP(hostname)) url.hostname = hostname; + return { url, ip: resolved[0].address }; +} + +function beforeDeadline(promise: Promise, deadline: number): Promise { + const remaining = deadline - Date.now(); + if (remaining <= 0) return Promise.reject(new Error("whole-request timeout")); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("whole-request timeout")), remaining); + timer.unref(); + promise.then( + (value) => { clearTimeout(timer); resolve(value); }, + (error) => { clearTimeout(timer); reject(error); }, + ); + }); +} + +function pinnedLookup(ip: string): LookupFunction { + const family = isIP(ip) || 4; + return function (_hostname: string, options: unknown, callback?: unknown) { + const cb = (typeof options === "function" ? options : callback) as ( + err: NodeJS.ErrnoException | null, + address: unknown, + family?: number, + ) => void; + const wantsAll = typeof options === "object" && options !== null && (options as { all?: boolean }).all; + if (wantsAll) cb(null, [{ address: ip, family }], undefined); + else cb(null, ip, family); + } as unknown as LookupFunction; +} + +export interface BoundedHttpsResponse { + status: number; + location: string | null; + contentEncoding: string | null; + body: Buffer; + finalUrl: string; +} + +interface RequestOptions { + method?: "GET" | "HEAD"; + accept?: string; + maxBytes?: number; + maxRedirects?: number; + timeoutMs?: number; +} + +function requestPinned(url: URL, ip: string, options: Required): Promise { + return new Promise((resolve, reject) => { + const signal = AbortSignal.timeout(options.timeoutMs); + const req = httpsRequest(url, { + agent: false, + method: options.method, + lookup: pinnedLookup(ip), + servername: isIP(url.hostname.replace(/^\[|\]$/g, "")) ? undefined : url.hostname, + signal, + headers: { + accept: options.accept, + "accept-encoding": "identity", + host: url.host, + }, + timeout: options.timeoutMs, + }, (res) => { + const announced = Number(res.headers["content-length"] ?? 0); + if (options.method !== "HEAD" && announced > options.maxBytes) { + res.destroy(new Error("response too large")); + return; + } + const encodingHeader = res.headers["content-encoding"]; + const contentEncoding = Array.isArray(encodingHeader) ? encodingHeader[0] ?? null : encodingHeader ?? null; + // Compression is disabled so the byte cap is also the post-decoding cap. + // A peer that ignores identity encoding is rejected rather than decoded. + if (options.method !== "HEAD" && contentEncoding && contentEncoding.toLowerCase() !== "identity") { + res.destroy(new Error("encoded responses are not accepted")); + return; + } + const chunks: Buffer[] = []; + let size = 0; + res.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > options.maxBytes) { + res.destroy(new Error("response too large")); + return; + } + chunks.push(chunk); + }); + res.on("end", () => { + const locationHeader = res.headers.location; + resolve({ + status: res.statusCode ?? 0, + location: Array.isArray(locationHeader) ? locationHeader[0] ?? null : locationHeader ?? null, + contentEncoding, + body: Buffer.concat(chunks), + finalUrl: url.href, + }); + }); + res.on("error", reject); + }); + req.on("timeout", () => req.destroy(new Error("timeout"))); + req.on("error", reject); + req.end(); + }); +} + +/** + * Credential-free bounded HTTPS request. DNS is resolved and pinned anew for + * every hop, so redirects cannot introduce a DNS-rebinding or private target. + */ +export async function boundedPublicHttpsRequest( + raw: string, + options: RequestOptions = {}, +): Promise { + const bounded: Required = { + method: options.method ?? "GET", + accept: options.accept ?? "application/json", + maxBytes: Math.max(0, Math.min(1024 * 1024, options.maxBytes ?? DEFAULT_MAX_BYTES)), + maxRedirects: Math.max(0, Math.min(5, options.maxRedirects ?? DEFAULT_MAX_REDIRECTS)), + timeoutMs: Math.max(250, Math.min(30_000, options.timeoutMs ?? DEFAULT_TIMEOUT_MS)), + }; + const deadline = Date.now() + bounded.timeoutMs; + let current = await beforeDeadline(validatePublicHttpsUrl(raw), deadline); + for (let redirects = 0; redirects <= bounded.maxRedirects; redirects++) { + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error("whole-request timeout"); + const response = await requestPinned(current.url, current.ip, { ...bounded, timeoutMs: remaining }); + if (response.status < 300 || response.status >= 400) return response; + if (!response.location || redirects === bounded.maxRedirects) throw new Error("redirect limit exceeded"); + current = await beforeDeadline( + validatePublicHttpsUrl(new URL(response.location, current.url).href), + deadline, + ); + } + throw new Error("redirect limit exceeded"); +} diff --git a/reference-implementations/dacs-directory/src/catalog/contracts.ts b/reference-implementations/dacs-directory/src/catalog/contracts.ts index e010a92..f970580 100644 --- a/reference-implementations/dacs-directory/src/catalog/contracts.ts +++ b/reference-implementations/dacs-directory/src/catalog/contracts.ts @@ -22,6 +22,16 @@ export const listingSummarySchema = { }, artifactProfile: { enum: artifactProfiles }, publicEndpoint: { type: "string", format: "uri" }, + reachabilityHint: { + type: "object", + required: ["status", "checkedAt"], + additionalProperties: false, + properties: { + status: { enum: ["reachable", "unreachable", "unknown"] }, + checkedAt: { type: "integer", minimum: 0 }, + surface: { type: "string", format: "uri" }, + }, + }, offering: { type: "object", required: ["title", "category", "tags"], properties: { diff --git a/reference-implementations/dacs-directory/src/catalog/reachability.ts b/reference-implementations/dacs-directory/src/catalog/reachability.ts new file mode 100644 index 0000000..bbed732 --- /dev/null +++ b/reference-implementations/dacs-directory/src/catalog/reachability.ts @@ -0,0 +1,115 @@ +import { boundedPublicHttpsRequest, OutboundTargetError } from "./boundedHttps.js"; +import { safePublicEndpoint } from "./publicEndpoint.js"; +import { + effectiveReachabilityStatus, + reachabilityHintIsFresh, + REACHABILITY_FRESH_MS, +} from "./reachabilityStatus.js"; +import type { ListingSummary, ReachabilityHint, SellerRecord } from "./types.js"; + +export { effectiveReachabilityStatus, REACHABILITY_FRESH_MS }; + +const boundedInt = (value: unknown, fallback: number, min: number, max: number): number => { + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback; +}; + +/** Probe only network reachability; an HTTP error response still proves reachability. */ +export async function probeReachabilitySurface(surface: string): Promise { + const checkedAt = Date.now(); + try { + const response = await boundedPublicHttpsRequest(surface, { + method: "HEAD", + accept: "*/*", + maxBytes: 1_024, + maxRedirects: 3, + timeoutMs: 5_000, + }); + return { + status: response.status >= 100 && response.status <= 599 ? "reachable" : "unreachable", + checkedAt, + surface, + }; + } catch (error) { + return { + // A target rejected by outbound policy was deliberately not contacted; + // do not claim it is unreachable or disclose internal network details. + status: error instanceof OutboundTargetError ? "unknown" : "unreachable", + checkedAt, + surface, + }; + } +} + +const listingKey = (seller: string, listing: ListingSummary): string => + `${seller}\n${listing.listingId}\n${listing.version}\n${listing.contentHash}`; + +export interface ReachabilityRefreshOptions { + now?: number; + cursor?: number; + maxProbes?: number; + concurrency?: number; + probe?: (surface: string) => Promise; +} + +/** + * Attach independently-derived hints after all listing verification. Results + * never participate in listing inclusion, revocation, identity or reputation. + * Returns the round-robin cursor for the next bounded refresh pass. + */ +export async function refreshReachabilityHints( + sellers: SellerRecord[], + priorSellers: SellerRecord[], + options: ReachabilityRefreshOptions = {}, +): Promise { + const now = options.now ?? Date.now(); + const maxProbes = boundedInt(options.maxProbes ?? process.env.DACS_REACHABILITY_MAX_PROBES, 20, 1, 100); + const concurrency = boundedInt(options.concurrency ?? process.env.DACS_REACHABILITY_CONCURRENCY, 5, 1, 10); + const probe = options.probe ?? probeReachabilitySurface; + const prior = new Map(); + for (const seller of priorSellers) { + for (const listing of seller.listings) prior.set(listingKey(seller.primaryClaim, listing), listing); + } + + const due: Array<{ key: string; listing: ListingSummary; surface: string }> = []; + for (const seller of sellers) { + for (const listing of seller.listings) { + const surface = safePublicEndpoint(listing.publicEndpoint); + if (!surface || listing.status !== "active") { + listing.reachabilityHint = undefined; + continue; + } + const previous = prior.get(listingKey(seller.primaryClaim, listing))?.reachabilityHint; + if (previous?.surface === surface && reachabilityHintIsFresh(previous, now)) { + listing.reachabilityHint = previous; + continue; + } + if (previous?.surface === surface) listing.reachabilityHint = { ...previous, status: "unknown" }; + due.push({ key: listingKey(seller.primaryClaim, listing), listing, surface }); + } + } + if (due.length === 0) return 0; + due.sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : 0); + const start = boundedInt(options.cursor, 0, 0, Number.MAX_SAFE_INTEGER) % due.length; + const selected = Array.from({ length: Math.min(maxProbes, due.length) }, (_, index) => due[(start + index) % due.length]); + let next = 0; + const bySurface = new Map>(); + const probeOnce = (surface: string): Promise => { + let pending = bySurface.get(surface); + if (!pending) { + pending = probe(surface); + bySurface.set(surface, pending); + } + return pending; + }; + const workers = Array.from({ length: Math.min(concurrency, selected.length) }, async () => { + for (;;) { + const index = next++; + if (index >= selected.length) return; + const candidate = selected[index]; + candidate.listing.reachabilityHint = await probeOnce(candidate.surface); + } + }); + await Promise.all(workers); + return (start + selected.length) % due.length; +} diff --git a/reference-implementations/dacs-directory/src/catalog/reachabilityStatus.ts b/reference-implementations/dacs-directory/src/catalog/reachabilityStatus.ts new file mode 100644 index 0000000..15cfbec --- /dev/null +++ b/reference-implementations/dacs-directory/src/catalog/reachabilityStatus.ts @@ -0,0 +1,17 @@ +import type { ReachabilityHint } from "./types.js"; + +export const REACHABILITY_FRESH_MS = 60 * 60_000; + +const REACHABILITY_STATUSES = new Set(["reachable", "unreachable", "unknown"]); + +export const reachabilityHintIsFresh = (hint: ReachabilityHint | undefined, now: number): boolean => + Boolean(hint && REACHABILITY_STATUSES.has(hint.status) && Number.isSafeInteger(hint.checkedAt) && + hint.checkedAt >= 0 && hint.checkedAt <= now && now - hint.checkedAt <= REACHABILITY_FRESH_MS); + +/** A missing or stale observation is explicitly unknown to rendering consumers. */ +export function effectiveReachabilityStatus( + hint: ReachabilityHint | undefined, + now = Date.now(), +): ReachabilityHint["status"] { + return reachabilityHintIsFresh(hint, now) ? hint!.status : "unknown"; +} diff --git a/reference-implementations/dacs-directory/src/catalog/reindexCore.ts b/reference-implementations/dacs-directory/src/catalog/reindexCore.ts index fcad4e3..b5b0cf9 100644 --- a/reference-implementations/dacs-directory/src/catalog/reindexCore.ts +++ b/reference-implementations/dacs-directory/src/catalog/reindexCore.ts @@ -15,6 +15,7 @@ import { import { chainResetRequired, chainResetThreshold } from "./chainContinuity"; import { crawlDomains } from "./wellknown"; import { upsertCounterpartyEvidenceSeller } from "./counterpartyEvidence"; +import { refreshReachabilityHints } from "./reachability"; import { loadCatalog, loadDomains, @@ -308,6 +309,9 @@ export async function reindexAll(opts: ReindexOptions = {}): Promise; /** listing content hash → bounded, deterministic revocation candidates. */ diff --git a/reference-implementations/dacs-directory/src/catalog/wellknown.ts b/reference-implementations/dacs-directory/src/catalog/wellknown.ts index 30ff653..6be5368 100644 --- a/reference-implementations/dacs-directory/src/catalog/wellknown.ts +++ b/reference-implementations/dacs-directory/src/catalog/wellknown.ts @@ -13,10 +13,9 @@ * must equal the claimed seller). Per-domain failures never poison the pass. */ import { sha256Hex } from "@kynesyslabs/dacs/canonical"; -import { lookup } from "node:dns/promises"; -import { request as httpsRequest } from "node:https"; -import { isIP } from "node:net"; -import type { LookupFunction } from "node:net"; +import { boundedPublicHttpsRequest, isPrivateAddress, validatePublicHttpsUrl } from "./boundedHttps.js"; + +export { isPrivateAddress } from "./boundedHttps.js"; export interface WellKnownAgent { domain: string; @@ -45,100 +44,6 @@ interface ListingIndex { }>; } -const MAX_RESPONSE_BYTES = 1024 * 1024; -const MAX_REDIRECTS = 3; - -function ipv6Words(address: string): number[] | null { - let input = address.toLowerCase().split("%", 1)[0]; - const dotted = input.match(/^(.*:)(\d{1,3}(?:\.\d{1,3}){3})$/); - if (dotted) { - const octets = dotted[2].split(".").map(Number); - if (octets.some((part) => part < 0 || part > 255)) return null; - input = `${dotted[1]}${((octets[0] << 8) | octets[1]).toString(16)}:${ - ((octets[2] << 8) | octets[3]).toString(16) - }`; - } - const halves = input.split("::"); - if (halves.length > 2) return null; - const parseHalf = (half: string) => half ? half.split(":").map((word) => Number.parseInt(word, 16)) : []; - const left = parseHalf(halves[0]); - const right = halves.length === 2 ? parseHalf(halves[1]) : []; - if ([...left, ...right].some((word) => !Number.isInteger(word) || word < 0 || word > 0xffff)) return null; - if (halves.length === 1) return left.length === 8 ? left : null; - const zeros = 8 - left.length - right.length; - return zeros >= 1 ? [...left, ...Array(zeros).fill(0), ...right] : null; -} - -/** Reject non-global address ranges before every outbound well-known fetch. */ -export function isPrivateAddress(address: string): boolean { - if (isIP(address) === 4) { - const p = address.split(".").map(Number); - return ( - p[0] === 0 || p[0] === 10 || p[0] === 127 || - (p[0] === 100 && p[1] >= 64 && p[1] <= 127) || - (p[0] === 169 && p[1] === 254) || - (p[0] === 172 && p[1] >= 16 && p[1] <= 31) || - (p[0] === 192 && p[1] === 168) || - (p[0] === 198 && (p[1] === 18 || p[1] === 19)) || - p[0] >= 224 - ); - } - if (isIP(address) === 6) { - const words = ipv6Words(address); - if (!words) return true; - // IPv4-mapped addresses inherit the embedded IPv4 policy regardless of - // whether the resolver renders the tail in dotted or hexadecimal form. - if (words.slice(0, 5).every((word) => word === 0) && words[5] === 0xffff) { - return isPrivateAddress( - `${words[6] >> 8}.${words[6] & 0xff}.${words[7] >> 8}.${words[7] & 0xff}`, - ); - } - // Public IPv6 is global-unicast 2000::/3, excluding IETF/documentation - // assignments and transition space that can tunnel an unchecked IPv4. - if ((words[0] & 0xe000) !== 0x2000) return true; - if (words[0] === 0x2001 && (words[1] & 0xfe00) === 0) return true; // 2001::/23 - if (words[0] === 0x2001 && words[1] === 0x0db8) return true; // documentation - if (words[0] === 0x2002) return true; // 6to4 - if (words[0] === 0x3fff && (words[1] & 0xf000) === 0) return true; // documentation - return false; - } - return true; -} - -interface VettedUrl { - url: URL; - /** The specific resolved address the caller MUST connect to. */ - ip: string; -} - -/** - * Vet an outbound URL and PIN the address we will connect to. We resolve DNS, - * reject if any record is non-public, then hand back one vetted IP. Callers - * connect to that exact IP (via the node:https `lookup` hook below), which - * closes the DNS-rebinding window between this check and the socket connect - * (TOCTOU): a resolver cannot answer "public" here and "169.254.169.254" at - * fetch time, because fetch never re-resolves. - */ -async function validateOutboundUrl(raw: string): Promise { - let url: URL; - try { - url = new URL(raw); - } catch { - throw new Error("invalid URL"); - } - if (url.protocol !== "https:" || url.username || url.password || (url.port && url.port !== "443")) { - throw new Error("well-known URLs must use public HTTPS on port 443"); - } - if (url.hostname === "localhost" || url.hostname.endsWith(".localhost") || url.hostname.endsWith(".local")) { - throw new Error("local hostnames are not allowed"); - } - const resolved = await lookup(url.hostname, { all: true, verbatim: true }); - if (resolved.length === 0 || resolved.some((r) => isPrivateAddress(r.address))) { - throw new Error("well-known URL resolves to a non-public address"); - } - return { url, ip: resolved[0].address }; -} - export function normalizeSubmittedDomain(domain: string): string { const raw = domain.includes("://") ? domain : `https://${domain}`; const url = new URL(raw); @@ -150,94 +55,12 @@ export function normalizeSubmittedDomain(domain: string): string { return url.origin; } -/** - * Force the socket to the pre-vetted IP while TLS/SNI and the Host header stay - * the real hostname (so certificate validation is unaffected). This is what - * pins the connection to the address `validateOutboundUrl` approved. - */ -function pinnedLookup(ip: string): LookupFunction { - const family = isIP(ip) || 4; - return function (_hostname: string, options: unknown, callback?: unknown) { - const cb = (typeof options === "function" ? options : callback) as ( - err: NodeJS.ErrnoException | null, - address: unknown, - family?: number, - ) => void; - const wantsAll = - typeof options === "object" && options !== null && (options as { all?: boolean }).all; - if (wantsAll) cb(null, [{ address: ip, family }], undefined); - else cb(null, ip, family); - } as unknown as LookupFunction; -} - -interface RawResponse { - status: number; - location: string | null; - body: string; -} - -/** Single GET to a vetted URL, connecting only to the pinned IP; size-capped. */ -function httpsGetPinned(url: URL, ip: string): Promise { - return new Promise((resolve, reject) => { - const req = httpsRequest( - url, - { - method: "GET", - lookup: pinnedLookup(ip), - servername: url.hostname, - headers: { accept: "application/json", host: url.host }, - timeout: 15_000, - }, - (res) => { - const announced = Number(res.headers["content-length"] ?? 0); - if (announced > MAX_RESPONSE_BYTES) { - res.destroy(); - reject(new Error("response too large")); - return; - } - const chunks: Buffer[] = []; - let size = 0; - res.on("data", (chunk: Buffer) => { - size += chunk.length; - if (size > MAX_RESPONSE_BYTES) { - res.destroy(); - reject(new Error("response too large")); - return; - } - chunks.push(chunk); - }); - res.on("end", () => { - const loc = res.headers.location; - resolve({ - status: res.statusCode ?? 0, - location: Array.isArray(loc) ? loc[0] ?? null : loc ?? null, - body: Buffer.concat(chunks).toString("utf8"), - }); - }); - res.on("error", reject); - }, - ); - req.on("timeout", () => req.destroy(new Error("timeout"))); - req.on("error", reject); - req.end(); - }); -} - async function fetchJson(url: string): Promise<{ body: unknown; raw: string } | null> { try { - let current = await validateOutboundUrl(url); - for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) { - const res = await httpsGetPinned(current.url, current.ip); - if (res.status >= 300 && res.status < 400) { - if (!res.location || redirects === MAX_REDIRECTS) return null; - // Re-vet AND re-pin the redirect target before following it. - current = await validateOutboundUrl(new URL(res.location, current.url).href); - continue; - } - if (res.status < 200 || res.status >= 300) return null; - return { body: JSON.parse(res.body), raw: res.body }; - } - return null; + const response = await boundedPublicHttpsRequest(url, { method: "GET", accept: "application/json" }); + if (response.status < 200 || response.status >= 300) return null; + const raw = response.body.toString("utf8"); + return { body: JSON.parse(raw), raw }; } catch { return null; } @@ -247,7 +70,7 @@ export async function crawlDomain(domain: string): Promise; - reachability: Readonly<{ - kind: "not-measured"; - label: typeof UNMEASURED_REACHABILITY_LABEL; - }>; + reachability: Readonly< + | { kind: "not-measured"; label: typeof UNMEASURED_REACHABILITY_LABEL } + | { kind: "reachable"; label: "Reachable in latest bounded Directory probe" } + | { kind: "unreachable"; label: "Unreachable in latest bounded Directory probe" } + | { kind: "unknown"; label: "Not recently confirmed by Directory" } + >; identityTier: IdentityTier; deals: Readonly<{ completed: number; @@ -30,6 +33,7 @@ export type DirectoryEvidenceState = Readonly<{ export function directoryEvidenceState( listing: ListingSummary, seller: Pick, + now = Date.now(), ): DirectoryEvidenceState { const endpoint = safePublicEndpoint(listing.publicEndpoint); const hasEndpointDeclaration = typeof listing.publicEndpoint === "string" && listing.publicEndpoint.length > 0; @@ -40,6 +44,16 @@ export function directoryEvidenceState( : { kind: "legacy" as const, label: "legacy SDK listing" as const }; const completed = seller.reputation.completed; const total = seller.reputation.totalAgreements; + const reachabilityStatus = listing.reachabilityHint + ? effectiveReachabilityStatus(listing.reachabilityHint, now) + : undefined; + const reachability = reachabilityStatus === "reachable" + ? { kind: "reachable" as const, label: "Reachable in latest bounded Directory probe" as const } + : reachabilityStatus === "unreachable" + ? { kind: "unreachable" as const, label: "Unreachable in latest bounded Directory probe" as const } + : reachabilityStatus === "unknown" + ? { kind: "unknown" as const, label: "Not recently confirmed by Directory" as const } + : { kind: "not-measured" as const, label: UNMEASURED_REACHABILITY_LABEL }; return Object.freeze({ listing: Object.freeze(listingState), @@ -61,7 +75,7 @@ export function directoryEvidenceState( label: "No endpoint declared" as const, explanation: "The listing declares no engagement endpoint", }), - reachability: Object.freeze({ kind: "not-measured" as const, label: UNMEASURED_REACHABILITY_LABEL }), + reachability: Object.freeze(reachability), identityTier: seller.identityTier ?? "self-declared", deals: Object.freeze({ completed, diff --git a/reference-implementations/dacs-directory/test/directory-evidence-state.test.ts b/reference-implementations/dacs-directory/test/directory-evidence-state.test.ts index 7721e93..c893f9c 100644 --- a/reference-implementations/dacs-directory/test/directory-evidence-state.test.ts +++ b/reference-implementations/dacs-directory/test/directory-evidence-state.test.ts @@ -37,6 +37,36 @@ test("separates listing evidence, endpoint declaration, and unmeasured reachabil assert.match(state.deals.explanation, /Both parties/); }); +test("projects fresh and stale bounded probes through the evidence boundary", () => { + const now = 2_000_000_000_000; + const reachable = directoryEvidenceState(listing({ + publicEndpoint: "https://agent.example/jobs", + reachabilityHint: { status: "reachable", checkedAt: now, surface: "https://agent.example/jobs" }, + }), seller(), now); + assert.deepEqual(reachable.reachability, { + kind: "reachable", + label: "Reachable in latest bounded Directory probe", + }); + + const unreachable = directoryEvidenceState(listing({ + publicEndpoint: "https://agent.example/jobs", + reachabilityHint: { status: "unreachable", checkedAt: now, surface: "https://agent.example/jobs" }, + }), seller(), now); + assert.deepEqual(unreachable.reachability, { + kind: "unreachable", + label: "Unreachable in latest bounded Directory probe", + }); + + const stale = directoryEvidenceState(listing({ + publicEndpoint: "https://agent.example/jobs", + reachabilityHint: { status: "reachable", checkedAt: now - 60 * 60_000 - 1, surface: "https://agent.example/jobs" }, + }), seller(), now); + assert.deepEqual(stale.reachability, { + kind: "unknown", + label: "Not recently confirmed by Directory", + }); +}); + test("does not turn a missing endpoint into a service failure", () => { const state = directoryEvidenceState(listing({ publicEndpoint: undefined }), seller()); assert.deepEqual(state.endpoint, { diff --git a/reference-implementations/dacs-directory/test/reachability.test.ts b/reference-implementations/dacs-directory/test/reachability.test.ts new file mode 100644 index 0000000..23b28ad --- /dev/null +++ b/reference-implementations/dacs-directory/test/reachability.test.ts @@ -0,0 +1,102 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { isPrivateAddress } from "../src/catalog/boundedHttps.js"; +import { + effectiveReachabilityStatus, + probeReachabilitySurface, + REACHABILITY_FRESH_MS, + refreshReachabilityHints, +} from "../src/catalog/reachability.js"; +import type { ReachabilityHint, SellerRecord } from "../src/catalog/types.js"; + +const now = 2_000_000_000_000; +const seller = (suffix: string, hint?: ReachabilityHint): SellerRecord => ({ + primaryClaim: `did:demos:agent:${suffix.repeat(64)}`, + displayName: `seller ${suffix}`, + cci: [], + listings: [{ + listingId: `listing-${suffix}`, + version: 1, + contentHash: suffix.repeat(64), + anchor: { kind: "storage-program", locator: `stor-${suffix.repeat(40)}` }, + seller: { primaryClaim: `did:demos:agent:${suffix.repeat(64)}`, displayName: `seller ${suffix}` }, + publicEndpoint: `https://${suffix}.example/agent`, + offering: { title: "service", category: "services.test", tags: [] }, + pricing: {}, + status: "active", + catalogObservedAt: now, + reachabilityHint: hint, + }], + deals: [], + reputation: { completed: 0, totalAgreements: 0, completionRate: null }, + registeredAt: now, + lastIndexedAt: now, +}); + +test("reachability rendering treats stale observations as unknown", () => { + assert.equal(effectiveReachabilityStatus({ status: "reachable", checkedAt: now }, now), "reachable"); + assert.equal(effectiveReachabilityStatus({ status: "reachable", checkedAt: now - REACHABILITY_FRESH_MS - 1 }, now), "unknown"); +}); + +test("refresh reuses fresh hints and does not affect listing validity or trust fields", async () => { + const fresh = { status: "reachable", checkedAt: now - 1_000, surface: "https://a.example/agent" } as const; + const current = [seller("a")]; + const prior = [seller("a", fresh)]; + const immutable = { + status: current[0].listings[0].status, + identityTier: current[0].identityTier, + reputation: structuredClone(current[0].reputation), + }; + let probes = 0; + + await refreshReachabilityHints(current, prior, { + now, + probe: async () => { probes += 1; throw new Error("must not probe"); }, + }); + + assert.equal(probes, 0); + assert.deepEqual(current[0].listings[0].reachabilityHint, fresh); + assert.deepEqual({ + status: current[0].listings[0].status, + identityTier: current[0].identityTier, + reputation: current[0].reputation, + }, immutable); +}); + +test("refresh is bounded, round-robin, and replaces stale claims with catalog observations", async () => { + const stale = { status: "reachable", checkedAt: now - REACHABILITY_FRESH_MS - 1, surface: "https://a.example/agent" } as const; + const current = [seller("a"), seller("b"), seller("c")]; + const prior = [seller("a", stale), seller("b"), seller("c")]; + const probed: string[] = []; + const probe = async (surface: string): Promise => { + probed.push(surface); + return { status: "unreachable", checkedAt: now, surface }; + }; + + const cursor = await refreshReachabilityHints(current, prior, { now, maxProbes: 1, concurrency: 1, probe }); + + assert.equal(cursor, 1); + assert.deepEqual(probed, ["https://a.example/agent"]); + assert.equal(current[0].listings[0].reachabilityHint?.status, "unreachable"); + assert.equal(current[1].listings[0].reachabilityHint, undefined); + assert.equal(current[2].listings[0].reachabilityHint, undefined); +}); + +test("blocked private targets are never contacted and remain unknown", async () => { + const ipv4 = await probeReachabilitySurface("https://127.0.0.1/metadata"); + const ipv6 = await probeReachabilitySurface("https://[::1]/metadata"); + assert.equal(ipv4.status, "unknown"); + assert.equal(ipv4.surface, "https://127.0.0.1/metadata"); + assert.equal(ipv6.status, "unknown"); +}); + +test("outbound policy rejects reserved and metadata-equivalent address forms", () => { + for (const address of [ + "0.0.0.0", "100.100.100.200", "169.254.169.254", "192.0.0.192", + "192.0.2.1", "198.51.100.1", "203.0.113.1", "224.0.0.1", + "::ffff:192.0.2.1", "fd00:ec2::254", + ]) assert.equal(isPrivateAddress(address), true, address); + assert.equal(isPrivateAddress("8.8.8.8"), false); + assert.equal(isPrivateAddress("2606:4700:4700::1111"), false); +});