From f9c3ca640d4306120180e04cb47298cca70ce6e5 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:31:24 +0000 Subject: [PATCH 1/3] Charge AI training crawlers for access (@profullstack/x402-gateway) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- .env.example | 4 ++++ apps/web/package.json | 1 + apps/web/src/app.js | 28 ++++++++++++++++++++++++++-- bun.lock | 3 +++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index ab6535a..765e1f3 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,7 @@ PORT=8080 # Optional privacy-preserving analytics endpoint. Unset = no third-party script at all. # ANALYTICS_SRC= # ANALYTICS_SITE_ID= + +# Crawl gateway (@profullstack/x402-gateway): training crawlers pay $1/day over x402. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/web/package.json b/apps/web/package.json index 2a806fe..6fc4ea3 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,6 +8,7 @@ "dependencies": { "@d3vices/config": "workspace:*", "@d3vices/tests": "workspace:*", + "@profullstack/x402-gateway": "^0.1.0", "hono": "^4.9.0" } } diff --git a/apps/web/src/app.js b/apps/web/src/app.js index 5929bf9..ce5ba3b 100644 --- a/apps/web/src/app.js +++ b/apps/web/src/app.js @@ -1,9 +1,11 @@ import { config } from '@d3vices/config'; +import { createGateway } from '@profullstack/x402-gateway'; +import { x402Gateway } from '@profullstack/x402-gateway/hono'; import { TEST_BY_SLUG, TESTS } from '@d3vices/tests/registry'; import { Hono } from 'hono'; import { serveStatic } from 'hono/bun'; import { compress } from 'hono/compress'; -import { llmsFullTxt, llmsTxt, robotsTxt, securityTxt, skillMd } from './agents.js'; +import { llmsFullTxt, llmsTxt, securityTxt, skillMd } from './agents.js'; import { THEME_SCRIPT_HASH } from './inline-scripts.js'; import { About } from './pages/About.jsx'; import { Changelog } from './pages/Changelog.jsx'; @@ -18,6 +20,21 @@ import { render } from './render.js'; const app = new Hono(); +/** + * Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, + * Applebot-Extended) pay by the day over x402 (@profullstack/x402-gateway). + * People, search engines and retrieval crawlers pass through untouched. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO they still get 402, with an empty offer. + */ +export const crawlGateway = createGateway({ + siteUrl: config.siteUrl, + siteName: 'd3vices', + coinpay: { apiKey: process.env.COINPAY_X402_KEY }, + payTo: process.env.CRAWL_PAY_TO, + contact: 'mailto:anthony@profullstack.com', +}); +app.use('*', x402Gateway(crawlGateway)); + /** * A trailing slash is a different URL to the router, so `/camera/` fell through * to the 404 page while `/camera` served the test. Every such URL is a dead link @@ -276,7 +293,14 @@ app.get('/manifest.webmanifest', (c) => { ); }); -app.get('/robots.txt', (c) => c.text(robotsTxt())); +app.get('/robots.txt', (c) => + c.text( + crawlGateway.robotsTxt({ + disallow: ['/api/'], + comments: ['Everything here is public and MIT licensed. Training crawls are paid: see /crawl.'], + }), + ), +); /** * What an answer engine or an agent reads instead of crawling 24 pages. All of diff --git a/bun.lock b/bun.lock index af4aec9..0954688 100644 --- a/bun.lock +++ b/bun.lock @@ -22,6 +22,7 @@ "dependencies": { "@d3vices/config": "workspace:*", "@d3vices/tests": "workspace:*", + "@profullstack/x402-gateway": "^0.1.0", "hono": "^4.9.0", }, }, @@ -91,6 +92,8 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.1.0", "", {}, "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ=="], + "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], "@szmarczak/http-timer": ["@szmarczak/http-timer@4.0.6", "", { "dependencies": { "defer-to-connect": "^2.0.0" } }, "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w=="], From ffdc59d46e241b788a4dcc6f50e1ef64399b45d4 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:39:43 +0000 Subject: [PATCH 2/3] Sort imports Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/web/src/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app.js b/apps/web/src/app.js index ce5ba3b..8670bd8 100644 --- a/apps/web/src/app.js +++ b/apps/web/src/app.js @@ -1,7 +1,7 @@ import { config } from '@d3vices/config'; +import { TEST_BY_SLUG, TESTS } from '@d3vices/tests/registry'; import { createGateway } from '@profullstack/x402-gateway'; import { x402Gateway } from '@profullstack/x402-gateway/hono'; -import { TEST_BY_SLUG, TESTS } from '@d3vices/tests/registry'; import { Hono } from 'hono'; import { serveStatic } from 'hono/bun'; import { compress } from 'hono/compress'; From ed5bd3c531f0ee35378db75c0c1c6015b1bd95cc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:45:58 +0000 Subject: [PATCH 3/3] robots test: refused crawler groups carry Disallow: / rather than /api/ Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- test/routes.test.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/routes.test.js b/test/routes.test.js index 50c6c11..86b7cc7 100644 --- a/test/routes.test.js +++ b/test/routes.test.js @@ -230,8 +230,13 @@ describe('what agents and answer engines read', () => { expect(txt).toContain('Sitemap:'); // Every group carries the same rules, so a parser that reads only the first // matching group still gets the whole policy. + // Refused training crawlers get `Disallow: /` (plus the sales page); every + // other group keeps `Disallow: /api/`. Together they cover every group. const groups = txt.split('User-agent:').length - 1; - expect(txt.split('Disallow: /api/').length - 1).toBe(groups); + const keepOut = txt.split('Disallow: /api/').length - 1; + const refused = txt.split('Disallow: /\n').length - 1; + expect(keepOut + refused).toBe(groups); + expect(txt).toContain('Allow: /crawl'); }); });