From 5f84ea9d87b88b34eb49aa5badadeeb1f9e07c74 Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Wed, 22 Jul 2026 11:45:59 +0800 Subject: [PATCH 1/3] test(e2e): default the harness to admin-off; held-back tests opt in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip AppOverrides.admin to default false, so the whole suite runs with no admin listener bound — the shape once the Admin API is removed. Only the tests whose subject IS the Admin API surface keep it, opting in with admin: true (a no-op behaviorally — this was the previous default): apikey-budget/-lifecycle, auth-baseline, file-resource-source, health-minimal, prometheus-metrics, openai-sdk-compat, seed-vs-admin-characterization, status-models. Everything else — the majority, already seeding via SeedClient/etcd — now runs admin-off with no per-file change. Scoped by sweeping every case for real admin-listener use (app.adminUrl in a request / AdminClient calls other than listModelStatuses, which reads the metrics listener); the result was exactly those 9 files. Last step of the e2e de-admin-API work (after #791 switch, #792 seed migration): full suite green admin-off proves the Admin API isn't needed to run it, so removal deletes only the admin listener + these 9 tests. --- tests/e2e/src/cases/apikey-budget-e2e.test.ts | 4 +++- .../src/cases/apikey-lifecycle-e2e.test.ts | 4 +++- tests/e2e/src/cases/auth-baseline-e2e.test.ts | 4 +++- .../cases/file-resource-source-e2e.test.ts | 8 +++++++- .../e2e/src/cases/health-minimal-e2e.test.ts | 4 +++- tests/e2e/src/cases/openai-sdk-compat.test.ts | 4 +++- .../src/cases/prometheus-metrics-e2e.test.ts | 8 +++++--- ...seed-vs-admin-characterization-e2e.test.ts | 4 +++- tests/e2e/src/cases/status-models-e2e.test.ts | 5 ++++- tests/e2e/src/harness/app.ts | 20 +++++++++++-------- 10 files changed, 46 insertions(+), 19 deletions(-) diff --git a/tests/e2e/src/cases/apikey-budget-e2e.test.ts b/tests/e2e/src/cases/apikey-budget-e2e.test.ts index 50f1efdd..3905474a 100644 --- a/tests/e2e/src/cases/apikey-budget-e2e.test.ts +++ b/tests/e2e/src/cases/apikey-budget-e2e.test.ts @@ -22,7 +22,9 @@ describe("apikey max_budget_usd e2e: standalone admin rejects removed field", () etcdReachable = await new EtcdClient().ping(); if (!etcdReachable) return; - app = await spawnApp(); + // Held-back: this test drives the Admin API surface itself, so it + // keeps the admin listener bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); admin = new AdminClient(app.adminUrl, app.adminKey); }); diff --git a/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts b/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts index 6e981d24..ecb4d915 100644 --- a/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts +++ b/tests/e2e/src/cases/apikey-lifecycle-e2e.test.ts @@ -54,7 +54,9 @@ describe("api key lifecycle e2e: expired/disabled keys fail closed, rotate swaps if (!etcdReachable) return; upstream = await startOpenAiUpstream(); - app = await spawnApp(); + // Held-back: this test drives the Admin API surface itself, so it + // keeps the admin listener bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); admin = new AdminClient(app.adminUrl, app.adminKey); seed = new SeedClient(etcd, app.etcdPrefix); diff --git a/tests/e2e/src/cases/auth-baseline-e2e.test.ts b/tests/e2e/src/cases/auth-baseline-e2e.test.ts index c5cf7d63..428f7e2c 100644 --- a/tests/e2e/src/cases/auth-baseline-e2e.test.ts +++ b/tests/e2e/src/cases/auth-baseline-e2e.test.ts @@ -50,7 +50,9 @@ describe("auth baseline e2e: missing/malformed/unknown bearer all fail closed", if (!etcdReachable) return; upstream = await startOpenAiUpstream(); - app = await spawnApp(); + // Held-back: this test drives the Admin API surface itself, so it + // keeps the admin listener bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); seed = new SeedClient(etcd, app.etcdPrefix); // A single Model + ProviderKey + ApiKey configured. The valid diff --git a/tests/e2e/src/cases/file-resource-source-e2e.test.ts b/tests/e2e/src/cases/file-resource-source-e2e.test.ts index 3d71ecab..5097acbc 100644 --- a/tests/e2e/src/cases/file-resource-source-e2e.test.ts +++ b/tests/e2e/src/cases/file-resource-source-e2e.test.ts @@ -74,7 +74,10 @@ describe("file resource source: smoke + admin write-guard", () => { beforeAll(async () => { upstream = await startOpenAiUpstream(); + // Held-back: exercises the file-mode admin write-rejection (409), so + // the admin listener stays bound (the suite default is now admin-off). app = await spawnApp({ + admin: true, resourcesFile: fileModeResources(upstream.baseUrl), extraEnv: { [CALLER_KEY_ENV]: CALLER_PLAINTEXT }, }); @@ -255,6 +258,7 @@ describe("file resource source: differential vs etcd mode", () => { // The SAME logical resource set on both gateways: one provider key, // an allowed model, a forbidden model, one caller key. fileApp = await spawnApp({ + admin: true, resourcesFile: ` _format_version: "1" provider_keys: @@ -278,7 +282,7 @@ api_keys: `, }); - etcdApp = await spawnApp(); + etcdApp = await spawnApp({ admin: true }); const seed = new SeedClient(etcd, etcdApp.etcdPrefix); const pk = await seed.createProviderKey({ display_name: "diff-pk", @@ -411,6 +415,7 @@ api_keys: beforeAll(async () => { upstream = await startOpenAiUpstream(); app = await spawnApp({ + admin: true, resourcesFile: reloadFile(upstream.baseUrl, ["reload-a"]), }); }); @@ -487,6 +492,7 @@ describe("file resource source: fail-fast boot", () => { let caught: unknown; try { await spawnApp({ + admin: true, resourcesFile: ` _format_version: "1" models: diff --git a/tests/e2e/src/cases/health-minimal-e2e.test.ts b/tests/e2e/src/cases/health-minimal-e2e.test.ts index 30ab3049..61d7d74d 100644 --- a/tests/e2e/src/cases/health-minimal-e2e.test.ts +++ b/tests/e2e/src/cases/health-minimal-e2e.test.ts @@ -9,7 +9,9 @@ describe("livez e2e: public liveness route is /livez and /health is gone", () => beforeAll(async () => { etcdReachable = await new EtcdClient().ping(); if (!etcdReachable) return; - app = await spawnApp(); + // Held-back: this test drives the admin listener's health endpoint, + // so it keeps admin bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); }); afterAll(async () => { diff --git a/tests/e2e/src/cases/openai-sdk-compat.test.ts b/tests/e2e/src/cases/openai-sdk-compat.test.ts index cac79d4b..ea9ae2e2 100644 --- a/tests/e2e/src/cases/openai-sdk-compat.test.ts +++ b/tests/e2e/src/cases/openai-sdk-compat.test.ts @@ -51,7 +51,9 @@ describe("openai SDK compat: drive gateway through real client", () => { ], }); - app = await spawnApp(); + // Held-back: this test drives the Admin API surface itself, so it + // keeps the admin listener bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); // Deliberately seeds via the Admin API: deprecation-window coverage. admin = new AdminClient(app.adminUrl, app.adminKey); diff --git a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts index c633e86a..1ceeeecd 100644 --- a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts +++ b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts @@ -30,7 +30,9 @@ describe("prometheus metrics e2e", () => { upstream = await startOpenAiUpstream({ nonStreamBody: responseBody(), }); - app = await spawnApp(); + // Held-back: asserts the admin listener's health endpoint, so it keeps + // admin bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); seed = new SeedClient(etcd, app.etcdPrefix); await configureOpenAi(seed, upstream, "prometheus-gpt"); @@ -92,7 +94,7 @@ describe("prometheus metrics e2e", () => { const customUpstream = await startOpenAiUpstream({ nonStreamBody: responseBody(), }); - const customApp = await spawnApp({ prometheusPath: "/custom-metrics" }); + const customApp = await spawnApp({ admin: true, prometheusPath: "/custom-metrics" }); try { const customSeed = new SeedClient(new EtcdClient(), customApp.etcdPrefix); await configureOpenAi(customSeed, customUpstream, "prometheus-custom-gpt"); @@ -187,7 +189,7 @@ describe("prometheus metrics e2e", () => { return; } - const disabledApp = await spawnApp({ prometheus: false }); + const disabledApp = await spawnApp({ admin: true, prometheus: false }); try { // No listener at all: the fetch must fail at the connection level, // not return an HTTP error. diff --git a/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts b/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts index 6ca35c8c..5446559c 100644 --- a/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts +++ b/tests/e2e/src/cases/seed-vs-admin-characterization-e2e.test.ts @@ -65,7 +65,9 @@ describe("seed-vs-admin characterization: direct etcd writes ≡ Admin API write if (!etcdReachable) return; upstream = await startOpenAiUpstream(); - app = await spawnApp(); + // Held-back: this test's subject is seed-vs-admin equivalence, so it + // keeps the admin listener bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); admin = new AdminClient(app.adminUrl, app.adminKey); seed = new SeedClient(etcd, app.etcdPrefix); diff --git a/tests/e2e/src/cases/status-models-e2e.test.ts b/tests/e2e/src/cases/status-models-e2e.test.ts index 91a5b677..ddbcd611 100644 --- a/tests/e2e/src/cases/status-models-e2e.test.ts +++ b/tests/e2e/src/cases/status-models-e2e.test.ts @@ -86,7 +86,9 @@ describe("status/models: etcd mode — equivalence with the admin endpoint", () }, }); - app = await spawnApp(); + // Held-back: compares /status/models to /admin/v1/models/status, so it + // keeps admin bound (the suite default is now admin-off). + app = await spawnApp({ admin: true }); const seed = new SeedClient(etcd, app.etcdPrefix); const failPk = await seed.createProviderKey({ @@ -201,6 +203,7 @@ describe("status/models: standalone file source", () => { beforeAll(async () => { app = await spawnApp({ + admin: true, resourcesFile: ` _format_version: "1" provider_keys: diff --git a/tests/e2e/src/harness/app.ts b/tests/e2e/src/harness/app.ts index 249003b8..56984973 100644 --- a/tests/e2e/src/harness/app.ts +++ b/tests/e2e/src/harness/app.ts @@ -11,13 +11,17 @@ import { harnessRequest } from "./http.js"; export interface AppOverrides { /** - * Whether to bind the admin listener. Defaults to `true`. When `false`, - * the gateway runs with `admin.enabled = false` — no admin listener is - * bound even in etcd/file mode, mirroring the post-removal world. Seed - * resources through `SeedClient`/`EtcdClient` (never the Admin API), and - * readiness is gated on the proxy `/livez` plus the metrics listener - * instead of the admin health endpoint. `adminUrl`/`adminKey` are still - * returned but point at an unbound port. + * Whether to bind the admin listener. **Defaults to `false`** — the + * gateway runs with `admin.enabled = false`, no admin listener bound, + * mirroring the post-removal world; readiness gates on the proxy + * `/livez` plus the metrics listener, and `adminUrl`/`adminKey` are + * still returned but point at an unbound port. Resources are seeded + * through `SeedClient`/`EtcdClient`, never the Admin API. + * + * Only tests whose subject IS the Admin API surface (the held-back set — + * admin auth, write-rejection, deprecation headers, status-equivalence, + * key rotation) opt back in with `admin: true`; they stay admin-on until + * the Admin API is removed, then get deleted. */ admin?: boolean; /** Inserted into `admin.admin_keys`. Defaults to a fresh random key. */ @@ -149,7 +153,7 @@ async function spawnAppOnce(overrides: AppOverrides = {}): Promise { } const prometheusEnabled = overrides.prometheus ?? true; - const adminEnabled = overrides.admin ?? true; + const adminEnabled = overrides.admin ?? false; const [proxyPort, adminPort, metricsPort] = await pickFreePorts(3); const adminKey = overrides.adminKey ?? `admin-${randomUUID()}`; const etcdPrefix = `/aisix-e2e-${randomUUID()}`; From 639a30ad9002c99b731dd459af02174118c41568 Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Wed, 22 Jul 2026 11:53:08 +0800 Subject: [PATCH 2/3] test(e2e): tighten prometheus-metrics admin scoping (audit LOW) Cold-audit LOW: keep admin:true only on the beforeAll app, whose `fetch(${app.adminUrl}/metrics)` 404 assertion needs the admin listener bound; the custom-path and prometheus-false spawns only probe the metrics port, so drop their redundant admin:true and let them run admin-off. Also correct the beforeAll comment to state the real reason admin is bound. --- tests/e2e/src/cases/prometheus-metrics-e2e.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts index 1ceeeecd..0a0d5d8e 100644 --- a/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts +++ b/tests/e2e/src/cases/prometheus-metrics-e2e.test.ts @@ -30,8 +30,10 @@ describe("prometheus metrics e2e", () => { upstream = await startOpenAiUpstream({ nonStreamBody: responseBody(), }); - // Held-back: asserts the admin listener's health endpoint, so it keeps - // admin bound (the suite default is now admin-off). + // Held-back: the "admin listener does not serve /metrics" test fetches + // `${app.adminUrl}/metrics` and expects 404, which needs the admin + // listener bound (unbound → connection refused, never a 404). The suite + // default is now admin-off, so this test opts back in. app = await spawnApp({ admin: true }); seed = new SeedClient(etcd, app.etcdPrefix); @@ -94,7 +96,7 @@ describe("prometheus metrics e2e", () => { const customUpstream = await startOpenAiUpstream({ nonStreamBody: responseBody(), }); - const customApp = await spawnApp({ admin: true, prometheusPath: "/custom-metrics" }); + const customApp = await spawnApp({ prometheusPath: "/custom-metrics" }); try { const customSeed = new SeedClient(new EtcdClient(), customApp.etcdPrefix); await configureOpenAi(customSeed, customUpstream, "prometheus-custom-gpt"); @@ -189,7 +191,7 @@ describe("prometheus metrics e2e", () => { return; } - const disabledApp = await spawnApp({ admin: true, prometheus: false }); + const disabledApp = await spawnApp({ prometheus: false }); try { // No listener at all: the fetch must fail at the connection level, // not return an HTTP error. From a087c7b8735517fd0ec13599b38fb1e1d0431132 Mon Sep 17 00:00:00 2001 From: Ming Wen Date: Wed, 22 Jul 2026 12:02:08 +0800 Subject: [PATCH 3/3] test(e2e): guard against extra.admin diverging from the admin readiness gate CodeRabbit: `overrides.extra` is spread over the generated config at the top level, so an `extra.admin` would replace the generated admin block and could bind the listener while readiness still keys off `adminEnabled` and skips the admin health gate. Reject `extra.admin` so the `admin` boolean stays the single source of truth for the admin listener. --- tests/e2e/src/harness/app.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/e2e/src/harness/app.ts b/tests/e2e/src/harness/app.ts index 56984973..7103da42 100644 --- a/tests/e2e/src/harness/app.ts +++ b/tests/e2e/src/harness/app.ts @@ -154,6 +154,16 @@ async function spawnAppOnce(overrides: AppOverrides = {}): Promise { const prometheusEnabled = overrides.prometheus ?? true; const adminEnabled = overrides.admin ?? false; + // `extra` is spread over the generated config at the top level, so an + // `extra.admin` would replace the generated admin block and could bind + // the listener while readiness still keys off `adminEnabled` and skips + // the admin health gate. Keep the `admin` boolean the single source of + // truth for the admin listener. + if (overrides.extra && "admin" in overrides.extra) { + throw new Error( + "spawnApp: control the admin listener with the `admin` boolean override, not `extra.admin`", + ); + } const [proxyPort, adminPort, metricsPort] = await pickFreePorts(3); const adminKey = overrides.adminKey ?? `admin-${randomUUID()}`; const etcdPrefix = `/aisix-e2e-${randomUUID()}`;