Charge AI training crawlers for access (x402 gateway)

.env.example

@@ -11,3 +11,7 @@ PORT=8080
 # Optional privacy-preserving analytics endpoint. Unset = no third-party script at all.
 # ANALYTICS_SRC=
 # ANALYTICS_SITE_ID=
+
+# Crawl gateway (@profullstack/x402-gateway): training crawlers pay $1/day over x402.
+COINPAY_X402_KEY=
+CRAWL_PAY_TO=

apps/web/package.json

@@ -8,6 +8,7 @@
   "dependencies": {
     "@d3vices/config": "workspace:*",
     "@d3vices/tests": "workspace:*",
+    "@profullstack/x402-gateway": "^0.1.0",
     "hono": "^4.9.0"
   }
 }

apps/web/src/app.js

@@ -1,9 +1,11 @@
 import { config } from '@d3vices/config';
 import { TEST_BY_SLUG, TESTS } from '@d3vices/tests/registry';
+import { createGateway } from '@profullstack/x402-gateway';
+import { x402Gateway } from '@profullstack/x402-gateway/hono';
 import { Hono } from 'hono';
 import { serveStatic } from 'hono/bun';
 import { compress } from 'hono/compress';
-import { llmsFullTxt, llmsTxt, robotsTxt, securityTxt, skillMd } from './agents.js';
+import { llmsFullTxt, llmsTxt, securityTxt, skillMd } from './agents.js';
 import { THEME_SCRIPT_HASH } from './inline-scripts.js';
 import { About } from './pages/About.jsx';
 import { Changelog } from './pages/Changelog.jsx';
@@ -18,6 +20,21 @@ import { render } from './render.js';
 
 const app = new Hono();
 
+/**
+ * Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider,
+ * Applebot-Extended) pay by the day over x402 (@profullstack/x402-gateway).
+ * People, search engines and retrieval crawlers pass through untouched. Without
+ * COINPAY_X402_KEY and CRAWL_PAY_TO they still get 402, with an empty offer.
+ */
+export const crawlGateway = createGateway({
+  siteUrl: config.siteUrl,
+  siteName: 'd3vices',
+  coinpay: { apiKey: process.env.COINPAY_X402_KEY },
+  payTo: process.env.CRAWL_PAY_TO,
+  contact: 'mailto:anthony@profullstack.com',
+});
+app.use('*', x402Gateway(crawlGateway));
+
 /**
  * A trailing slash is a different URL to the router, so `/camera/` fell through
  * to the 404 page while `/camera` served the test. Every such URL is a dead link
@@ -276,7 +293,14 @@ app.get('/manifest.webmanifest', (c) => {
   );
 });
 
-app.get('/robots.txt', (c) => c.text(robotsTxt()));
+app.get('/robots.txt', (c) =>
+  c.text(
+    crawlGateway.robotsTxt({
+      disallow: ['/api/'],
+      comments: ['Everything here is public and MIT licensed. Training crawls are paid: see /crawl.'],
+    }),
+  ),
+);
 
 /**
  * What an answer engine or an agent reads instead of crawling 24 pages. All of

test/routes.test.js

@@ -230,8 +230,13 @@ describe('what agents and answer engines read', () => {
     expect(txt).toContain('Sitemap:');
     // Every group carries the same rules, so a parser that reads only the first
     // matching group still gets the whole policy.
+    // Refused training crawlers get `Disallow: /` (plus the sales page); every
+    // other group keeps `Disallow: /api/`. Together they cover every group.
     const groups = txt.split('User-agent:').length - 1;
-    expect(txt.split('Disallow: /api/').length - 1).toBe(groups);
+    const keepOut = txt.split('Disallow: /api/').length - 1;
+    const refused = txt.split('Disallow: /\n').length - 1;
+    expect(keepOut + refused).toBe(groups);
+    expect(txt).toContain('Allow: /crawl');
   });
 });