diff --git a/.github/workflows/main-build.yml b/.github/workflows/main-build.yml index 90f40f242..8bcbf91fe 100644 --- a/.github/workflows/main-build.yml +++ b/.github/workflows/main-build.yml @@ -24,14 +24,8 @@ jobs: with: go-version-file: go.mod - # Needed so release_build_cgo can produce arm64 artifacts on amd64 runner - - name: Install cross toolchains for CGO - run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - name: Build binaries for multiple architectures - run: make release_build_cross + run: make release_build_nocgo - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -51,21 +45,6 @@ jobs: type=sha,format=short,suffix=-rc type=ref,event=branch,pattern=latest - - name: Docker Metadata action (cgo) - id: meta_cgo - uses: docker/metadata-action@v5 - env: - DOCKER_METADATA_PR_HEAD_SHA: "true" - with: - images: | - ghcr.io/pokt-network/path - tags: | - type=semver,pattern={{version}},suffix=-cgo - type=semver,pattern={{major}}.{{minor}},suffix=-cgo - type=ref,event=tag,suffix=-rc-cgo - type=sha,format=short,suffix=-rc-cgo - type=ref,event=branch,pattern=latest,suffix=-cgo - - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -73,7 +52,7 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - # Non-CGO image (multi-arch, Alpine runtime) + # Runtime image (multi-arch, Alpine) - name: Build and push Docker image uses: docker/build-push-action@v5 with: @@ -85,18 +64,3 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - - # CGO image (multi-arch, glibc runtime) - - name: Build and push Docker image (cgo) - uses: docker/build-push-action@v5 - with: - push: true - tags: ${{ steps.meta_cgo.outputs.tags }} - build-args: | - IMAGE_TAG=${{ steps.meta.outputs.version }} - BINARY_SUFFIX=_cgo - platforms: linux/amd64,linux/arm64 - file: Dockerfile.release.glibc - cache-from: type=gha - cache-to: type=gha,mode=max - context: . diff --git a/.github/workflows/run-lint-and-test.yml b/.github/workflows/run-lint-and-test.yml index 3f6306198..22736814d 100644 --- a/.github/workflows/run-lint-and-test.yml +++ b/.github/workflows/run-lint-and-test.yml @@ -43,8 +43,5 @@ jobs: run: | git config --global url."https://${{ github.token }}:x-oauth-basic@github.com/".insteadOf "https://github.com/" - - name: Run unit tests without CGO + - name: Run unit tests run: CGO_ENABLED=0 go test ./... -short - - - name: Run unit tests with CGO - run: CGO_ENABLED=1 go test -tags "ethereum_secp256k1" ./... -short diff --git a/CLAUDE.md b/CLAUDE.md index efd1f7763..0e9e6bb82 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -393,6 +393,48 @@ shortening a drain is worse than saying no), the shared key carries a TTL past i drain, and expired entries are filtered on read and reaped. There is no way to bench an operator indefinitely through this endpoint. Re-issue to extend. +**Request Sample** (`GET /admin/request-sample[/{serviceId}]`) + +Answers one question about a service's traffic: **many different requests, or the same few +over and over?** Every quality signal PATH has — latency, success, hedge wins, the +reputation score — rewards whoever answers fastest, and an endpoint fronted by a cache +answers a *repeated* request in sub-millisecond time without touching a node. Against +repetitive traffic it wins every race; against unique traffic it is an ordinary node. Whether +a fast operator is fast or merely cached therefore has to be read from the **traffic**, and +nothing recorded the traffic's shape: `method` is a label, `params` never were, and a +thousand `getAccountInfo` for a thousand accounts and a thousand for one account were one +number. + +One request in N (`PATH_REQUEST_SAMPLE_RATE`, default 100, `0` disables) is fingerprinted: +JSON-RPC → one fingerprint per item on `method` + compacted `params` (id and whitespace +excluded, so rotating ids cannot make repetition look diverse); anything else → HTTP method ++ path + body. Counted per service in fixed windows (`PATH_REQUEST_SAMPLE_WINDOW`, default +`10m`); the last completed window is kept. Table bounded at +`PATH_REQUEST_SAMPLE_MAX_FINGERPRINTS` (default 5000) — past it new fingerprints are +*counted* in `table_overflow` but not stored, so `uniqueness` stays honest and a big overflow +is itself the answer (the traffic is diverse). + +```bash +curl -s localhost:13069/admin/request-sample # one row per service +curl -s "localhost:13069/admin/request-sample/solana?window=previous&top=20" +``` + +Read: `uniqueness` = distinct / sampled (1.0 all different, →0 the same few repeated); +`top1_share`; `methods[]` with **per-method** uniqueness — block-height calls are legitimately +repetitive, account/transaction lookups are not, so judge the method not the service; `top[]` +with a 200-byte snippet of each most-repeated payload. `requests_seen` is every request, +`sampled` is the 1-in-N — never read the sample as the total. + +Gauges `path_request_sample_uniqueness{service_id}` / `path_request_sample_top1_share` carry +the last completed window — per service_id only, nothing about methods or payloads, so +cardinality is the service list. **Per-pod, in-memory**: query the pod carrying the traffic, +or several, before a fleet conclusion. + +What it cannot tell: requests, not clients — there is no client identity behind the edge +([[no_portal_no_client_identity]]) — so repetition cannot be attributed to a sender; and a low +ratio is a property of the traffic, not evidence against an operator: it says the conditions +under which a cache wins are present, not that anyone runs one. + **Domain Blacklist (`blocked_domains`) — the nuclear ban** Permanently bans an operator domain from serving specific RPC types on **ALL services**, @@ -445,6 +487,21 @@ call site was revert-checked (filter removed → tests fail). - When a domain is stuck in circuit breaker state due to a transient issue that has resolved - Rolling restarts alone don't work because `refreshFromRedis` repopulates in-memory state from Redis +**Circuit breaker failure-rate gate — what feeds the denominator.** The gate breaks a hostname +on failures / (failures + successes) over a 30s window. Failures arrive from every path (each +failed attempt re-enters the retry loop → `MarkBroken`); successes only arrive where the +returning path calls `RecordSuccess`. The hedge-race success branch did not, and with a hedge +delay configured *every* first attempt returns through it (including `primary_only`), so the +gate saw ~5% of a high-volume operator's successes and 26% fleet-wide — a low-volume host read +30–66% failure where the relay counters read ~21%, and no threshold tuning (hysteresis +included) can hold against a rate inflated past it by construction. When the gate and +`path_relays_total` disagree about a host's rate, suspect a missing `RecordSuccess` site +before suspecting the threshold. `path_circuit_breaker_outcome_total{domain=}` +shows both sides of the fraction the gate actually computes; compare its success side against +`path_relays_total{status_code="200"}` per operator — they should agree within the +health-check/retry slice. Test through the real retry loop +(`gateway/circuit_breaker_hedge_denominator_test.go`), not through the breaker's own API. + ## WebSocket Frames Are Reward-Eligible Relays **Every endpoint→client WebSocket frame is signed by the relay miner and mined as a @@ -580,6 +637,73 @@ Recorded on **every** band pick, so `outcome="reshaped"` over the total is the r **What to watch after enabling:** `path_supplier_exhausted_total` for the **thin** operators the excess lands on, not the capped one — a solo-registration backend gains share while still holding one supplier's per-session allowance. Same failure mode as the backend-URL dedup, and self-correcting. Retry success rate — `path_relays_total{request_type="retry"}` split by `status_code` — must not fall; roughly 60% of retries already fail, so that pool is marginal to begin with. +## Adding a Prometheus Label — What Actually Bounds Cardinality + +Two rounds of a production cardinality incident (2026-08-12) converged on one rule: +**only a label's VALUE SET bounds it.** Neither a sanitizer nor a guard does, and each fails +in a way that looks like success. + +- A **sanitizer** bounds a value's *shape*, never the *set*. `SanitizeMethodLabel` was already + wired when 5,000 route-shaped probe paths sailed through it. +- A **cardinality guard** bounds the **live registry**, never the number of distinct series + Prometheus retains. `path_supplier_signal_total` sat at ~26% of its 25K cap and was still one + of the two largest series sources in the whole job — 6,523 tuples live in a 10-minute window + against **60,674 distinct over one pod's 7.7h life**. Eviction is not the cause and removing + it would not help: re-admitting an evicted tuple recreates the *same label set*, hence the + same series with a gap, never a new one. Eviction only decides whether the cost also lands on + pod heap. + +Tiers, in the order to reach for them: + +| label source | example | verdict | +|---|---|---| +| our config | `service_id`, `rpc_type`, `reason`, `role`, status class | safe — fixed at deploy | +| operator set | `domain` (eTLD+1) | safe — 15 values fleet-wide, grows only when an operator joins | +| **the chain** | `supplier` | **never safe at any cap** — ~5,200 addresses, grows with the network, rotates every session | +| the client | `method`, REST path | guard-only, and only because the cap converts unbounded minting into a bounded cost plus a WARN | + +**A label on a histogram costs ~12× what it costs on the counter beside it** (one series per +bucket plus `_sum`/`_count`). `path_relay_latency_seconds_bucket` was 31.6% of all gateway +series because it carried `status_code` × `reputation_signal` — a 20× pair that no dashboard +ever queried *from the histogram*. Put the outcome taxonomy on the counter; keep the histogram +on topology labels only. + +**`supplier` is gone from every aggregate metric.** Per-supplier questions are served by +`GET /ready/?detailed=true` — a point lookup, not 74K retained timeseries. Three +metrics keep it deliberately (`supplier_exhausted_total`, `supplier_nil_pubkey_total`, +`supplier_pubkey_cache_events_total`): there the address is the actionable payload, not a way +of naming an operator. `Test_SupplierLabelIsGone` enforces the rest. + +Churn diagnostic — run it whenever a metric looks cheap but the TSDB disagrees: + +```promql +count(count_over_time({pod=""}[10m])) # live +count(count_over_time({pod=""}[8h])) # distinct over 8h +``` + +Above ~1.5× means the label set rotates and the metric costs multiples of its instant count. +`path_relay_latency_seconds_bucket` at 1.0× is the control. + +**Testing traps specific to metrics** (same family as the routing ones below): + +- `Gather()` reports the labels of **child series**, so a vec with no children reports no + labels at all. A registry-walk test passes on a revert that re-adds the label. **Populate + through the production `Record*` helper first**, then walk — and assert the population + happened, or the test decays into asserting nothing. +- Detect a **removed** metric by registration collision (`Register` a same-named probe and + expect no `AlreadyRegisteredError`), not by walking `Gather()` — a re-added vec that nothing + populates is invisible to a walk. +- A `supplier`→`domain` re-key **compiles silently** when the call site keeps passing the + address: both are strings and the label *name* is right. Assert on the label **value** from + the production caller — a bech32 address sanitizes to the `supplier_addr` sentinel, which is + the tell. `qos/evm/qos_filter_rejection_label_test.go` and + `gateway/hedge_outcome_label_test.go` do this; both were revert-checked. + +**Never `labeldrop` these on the Prometheus side.** Collapsing thousands of series onto one +label set produces `duplicate sample for timestamp`, which fails the **whole scrape** — +`up=0`, every gateway metric lost, not a partial blinding. The collision-free stopgap is +`action: drop` on `__name__` for a specific metric. + ## Testing Changes That Affect Routing Three separate bugs shipped in the admin-drain feature, all with passing tests, all the same diff --git a/cmd/main.go b/cmd/main.go index c7b68f2d6..adc5ec5ea 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -282,6 +282,10 @@ func main() { // Uses the same Redis client as leader election. Nil-safe (local-only mode if no Redis). domainCircuitBreaker := gateway.NewDomainCircuitBreaker(redisClient, logger) + // Request-shape sampler behind GET /admin/request-sample/{serviceId}. nil when disabled + // via PATH_REQUEST_SAMPLE_RATE=0; the gateway hook and the admin endpoint both accept nil. + requestSampler := gateway.NewRequestSamplerFromEnv(logger) + healthCheckExecutor, leaderElector := setupHealthCheckExecutor( backgroundCtx, logger, @@ -324,6 +328,7 @@ func main() { WebsocketMessageBufferSize: config.GetRouterConfig().WebsocketMessageBufferSize, ObservationQueue: observationQueue, DomainCircuitBreaker: domainCircuitBreaker, + RequestSampler: requestSampler, WebsocketConnectionLimiter: gateway.NewWebsocketConnectionLimiter(config.GetRouterConfig().MaxConcurrentWebsocketConnections), } @@ -383,6 +388,7 @@ func main() { websocketAdmin, reputationAdmin, unifiedServicesConfig, + requestSampleAdminOrNil(requestSampler), ) // -------------------- Start PATH API Router -------------------- @@ -591,3 +597,13 @@ func getConfigPath(defaultConfigPath string) (string, error) { return configPath, nil } + +// requestSampleAdminOrNil keeps a disabled sampler (typed nil) from reaching the router as +// a non-nil interface, so the admin endpoint reports 503 "not enabled" instead of an empty +// report. +func requestSampleAdminOrNil(s *gateway.RequestSampler) router.RequestSampleAdmin { + if s == nil { + return nil + } + return s +} diff --git a/gateway/circuit_breaker_hedge_denominator_test.go b/gateway/circuit_breaker_hedge_denominator_test.go new file mode 100644 index 000000000..d0eddbdc8 --- /dev/null +++ b/gateway/circuit_breaker_hedge_denominator_test.go @@ -0,0 +1,146 @@ +package gateway + +import ( + "context" + "net/http" + "sync" + "testing" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" + + protocolobservations "github.com/pokt-network/path/observation/protocol" + "github.com/pokt-network/path/protocol" +) + +// The circuit breaker's failure-rate gate divides failures by (failures + successes). Every +// attempt that fails reaches MarkBroken through the retry loop, but a success only counts if +// the path it returned on calls RecordSuccess — and the hedge-race path did not. On a service +// with a hedge delay configured EVERY first attempt goes through the racer, including the +// overwhelming majority where the hedge never fires ("primary_only"), so the gate's +// denominator was fed almost exclusively by retries and batch items. +// +// Measured 2026-08-21, one environment, one hour: a high-volume operator produced 680,893 +// successful first-attempt relays and the gate counted 36,287 of them; fleet-wide the gate +// saw 26% of successes. A low-volume operator whose traffic is mostly hedges is judged on a +// fraction dominated by its failures, breaks at a rate it does not have, and the hysteresis +// margin added to stop marginal hosts flapping never gets a chance to hold — the rate it +// sees is inflated past the margin by construction. +// +// The normal path is exercised alongside as the control: the same request, the same +// response, one success either way. +func TestHandleSingleRelayRequest_HedgeRaceSuccessFeedsCircuitBreakerDenominator(t *testing.T) { + const ( + serviceID = "test-service" + endpoint = protocol.EndpointAddr("pokt1a-https://a.example.com") + domain = "a.example.com" + ) + + for _, tc := range []struct { + name string + hedgeDelay *time.Duration + }{ + {name: "hedge race path", hedgeDelay: hedgeDelayPtr()}, + {name: "normal path (control)", hedgeDelay: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + cb := NewDomainCircuitBreaker(nil, testCircuitBreakerLogger()) + protocolCtx := &okProtocolCtx{endpoint: endpoint} + rc := &requestContext{ + logger: polyzero.NewLogger(), + context: context.Background(), + serviceID: serviceID, + qosCtx: &recordingQoSContext{}, + circuitBreaker: cb, + protocol: &okProtocol{ + mockProtocolForRetry: mockProtocolForRetry{ + retryConfig: &ServiceRetryConfig{ + Enabled: boolPtr(true), + MaxRetries: intPtr(0), + HedgeDelay: tc.hedgeDelay, + }, + }, + endpoints: protocol.EndpointAddrList{ + endpoint, + "pokt1b-https://b.example.net", + }, + protocolCtx: protocolCtx, + }, + protocolContexts: []ProtocolRequestContext{protocolCtx}, + originalHTTPRequest: httptestRequest(), + } + + require.NoError(t, rc.handleSingleRelayRequest()) + require.GreaterOrEqual(t, protocolCtx.calls(), 1, "the relay must actually have been sent") + + cb.statsMu.Lock() + w := cb.windowLocked(serviceID, domain, time.Now()) + successes, failures := w.successes, w.failures + cb.statsMu.Unlock() + + require.Equal(t, 0, failures, "a successful relay must not register as a failure") + require.Equal(t, 1, successes, + "a successful relay must be counted exactly once in the gate's denominator, on whichever path returned it") + }) + } +} + +// okProtocolCtx answers every relay with a valid JSON-RPC result from a fixed endpoint. +type okProtocolCtx struct { + endpoint protocol.EndpointAddr + + mu sync.Mutex + callCount int +} + +func (m *okProtocolCtx) calls() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.callCount +} + +func (m *okProtocolCtx) HandleServiceRequest([]protocol.Payload) ([]protocol.Response, error) { + m.mu.Lock() + m.callCount++ + m.mu.Unlock() + return []protocol.Response{{ + Bytes: []byte(`{"jsonrpc":"2.0","id":1,"result":"0x10"}`), + HTTPStatusCode: http.StatusOK, + EndpointAddr: m.endpoint, + }}, nil +} + +func (m *okProtocolCtx) SetParentContext(context.Context) {} +func (m *okProtocolCtx) MarkAsHedge() {} +func (m *okProtocolCtx) MarkAsRetry() {} +func (m *okProtocolCtx) MarkAsHealthCheck() {} +func (m *okProtocolCtx) GetObservations() protocolobservations.Observations { + return protocolobservations.Observations{} +} + +// okProtocol supplies endpoints and the always-succeeding protocol context to the retry loop. +type okProtocol struct { + mockProtocolForRetry + endpoints protocol.EndpointAddrList + protocolCtx *okProtocolCtx +} + +func (m *okProtocol) GetUnifiedServicesConfig() *UnifiedServicesConfig { + return &UnifiedServicesConfig{ + Services: []ServiceConfig{{ID: "test-service", RetryConfig: m.retryConfig}}, + } +} + +func (m *okProtocol) AvailableHTTPEndpoints( + _ context.Context, _ protocol.ServiceID, _ sharedtypes.RPCType, _ *http.Request, +) (protocol.EndpointAddrList, protocolobservations.Observations, error) { + return m.endpoints, protocolobservations.Observations{}, nil +} + +func (m *okProtocol) BuildHTTPRequestContextForEndpoint( + _ context.Context, _ protocol.ServiceID, _ protocol.EndpointAddr, _ sharedtypes.RPCType, _ *http.Request, _ bool, +) (ProtocolRequestContext, protocolobservations.Observations, error) { + return m.protocolCtx, protocolobservations.Observations{}, nil +} diff --git a/gateway/domain_circuit_breaker.go b/gateway/domain_circuit_breaker.go index 01cdefc9f..f3a32c1e2 100644 --- a/gateway/domain_circuit_breaker.go +++ b/gateway/domain_circuit_breaker.go @@ -45,6 +45,27 @@ const ( // recovers. Escalation is meant to punish an domain that breaks AGAIN after being let // back in; without memory across expiry every episode looks like a first offence. defaultEscalationMemory = 60 * time.Minute + // defaultRebreakMargin widens the threshold for a domain that broke recently, giving the + // gate hysteresis instead of a single line it oscillates across. + // + // WHY. The threshold is used to break and nothing but TTL expiry restores, so a domain + // whose true failure rate sits just above failureRateThreshold breaks every single time + // it is let back in, forever, while a domain far above it is treated identically. + // + // Measured in production: six relay-miner hosts behind one operator domain, success rates + // against a threshold of exactly 80%: + // + // marginal: 79.7% · 79.3% · 78.3% · 72.5% genuinely broken: 58.6% · 49.7% + // + // The four marginal hosts sat within 1.7 points of the line and flapped, spending 69-92% + // of a six-hour window removed from the pool — while one of them, still marked broken, + // answered 40 consecutive probes with zero errors at a latency matching the host carrying + // nearly all of that service. The two genuinely broken hosts must stay broken. + // + // 0.15 puts the re-break line at 35% failure: comfortably above where the marginal hosts + // live and comfortably below where the broken ones do. A first break is unaffected — this + // only governs whether a domain that already served its TTL is removed again. + defaultRebreakMargin = 0.15 ) // classifyCircuitBreakReason maps the free-text reason string passed to @@ -99,6 +120,7 @@ type DomainCircuitBreaker struct { failureWindow time.Duration minFailures int failureRateThreshold float64 + rebreakMargin float64 escalationMemory time.Duration statsMu sync.Mutex stats map[string]map[string]*domainOutcomeWindow // serviceID -> domain @@ -153,6 +175,7 @@ func NewDomainCircuitBreaker(redisClient *redis.Client, logger polylog.Logger) * failureWindow: defaultFailureWindow, minFailures: defaultMinFailures, failureRateThreshold: defaultFailureRateThreshold, + rebreakMargin: defaultRebreakMargin, escalationMemory: defaultEscalationMemory, stats: make(map[string]map[string]*domainOutcomeWindow), } @@ -166,9 +189,14 @@ func (cb *DomainCircuitBreaker) RecordSuccess(serviceID, domain string) { return } cb.statsMu.Lock() - defer cb.statsMu.Unlock() w := cb.windowLocked(serviceID, domain, time.Now()) w.successes++ + cb.statsMu.Unlock() + + // Expose the gate's denominator. Recorded outside the lock: the metric has its own + // synchronisation and holding statsMu across it would put a Prometheus write on the path + // every successful relay takes. + metrics.RecordCircuitBreakerOutcome(serviceID, domain, metrics.CircuitBreakerOutcomeSuccess) } // windowLocked returns the outcome window for a domain, rolling it over if the current one @@ -208,12 +236,30 @@ func (cb *DomainCircuitBreaker) shouldBreak(serviceID, domain string, now time.T w := cb.windowLocked(serviceID, domain, now) w.failures++ + // Numerator counterpart to RecordSuccess. Deliberately here rather than in MarkBroken: + // this is the point a failure actually enters the gate's window, and MarkBroken returns + // earlier for a domain already broken — counting there would credit failures the rate + // calculation never saw. + defer metrics.RecordCircuitBreakerOutcome(serviceID, domain, metrics.CircuitBreakerOutcomeFailure) total := w.failures + w.successes if w.failures < cb.minFailures || total == 0 { return false, 0 } - if float64(w.failures)/float64(total) < cb.failureRateThreshold { + + // A domain that broke recently must be CLEARLY worse to be removed again, not merely + // over the same line it was over last time. Without this a domain sitting just above the + // threshold re-breaks on every readmission indefinitely, and escalation then holds it out + // for progressively longer — the flap this margin exists to stop. See defaultRebreakMargin. + // + // Deliberately shares its predicate with escalation below: "broke recently" must mean the + // same thing to both, or a domain could be escalated for an episode the margin let pass. + recentlyBroken := !w.lastEpisodeAt.IsZero() && now.Sub(w.lastEpisodeAt) <= cb.escalationMemory + threshold := cb.failureRateThreshold + if recentlyBroken { + threshold += cb.rebreakMargin + } + if float64(w.failures)/float64(total) < threshold { return false, 0 } @@ -221,7 +267,7 @@ func (cb *DomainCircuitBreaker) shouldBreak(serviceID, domain string, now time.T // was let back in and failed again. Concurrent duplicate marks within one episode are // filtered upstream in MarkBroken and never reach here. hitCount := 1 - if !w.lastEpisodeAt.IsZero() && now.Sub(w.lastEpisodeAt) <= cb.escalationMemory { + if recentlyBroken { hitCount = w.lastHitCount + 1 } w.lastEpisodeAt = now diff --git a/gateway/domain_circuit_breaker_hysteresis_test.go b/gateway/domain_circuit_breaker_hysteresis_test.go new file mode 100644 index 000000000..562d6b3b6 --- /dev/null +++ b/gateway/domain_circuit_breaker_hysteresis_test.go @@ -0,0 +1,217 @@ +package gateway + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + + "github.com/pokt-network/path/metrics" +) + +// driveAtRate feeds the gate `total` outcomes with `failPct` of them failing, interleaved so +// the running rate is representative rather than front-loaded with failures. Returns whether +// the domain is broken at the end, read through the PRODUCTION caller. +// +// Asserting through GetBrokenDomains rather than shouldBreak is deliberate: shouldBreak is +// where the change lives, so a test that calls it directly would pass on a version whose +// verdict never reaches selection. +func driveAtRate(t *testing.T, cb *DomainCircuitBreaker, ctx context.Context, serviceID, domain string, total, failPct int) bool { + t.Helper() + acc := 0 + for i := 0; i < total; i++ { + acc += failPct + if acc >= 100 { + acc -= 100 + cb.MarkBroken(ctx, serviceID, domain, "retry: simulated") + // Stop at the break. Continuing would keep feeding the window while the domain + // is broken, and MarkBroken short-circuits as "duplicate" in that state without + // recording the failure — so successes would accumulate unopposed and poison the + // NEXT episode's rate. Production cannot reach that state: a broken domain is + // filtered out of selection and receives nothing. + if cb.GetBrokenDomains(ctx, serviceID)[domain] { + return true + } + } else { + cb.RecordSuccess(serviceID, domain) + } + } + return cb.GetBrokenDomains(ctx, serviceID)[domain] +} + +// A domain whose failure rate sits just above the break threshold must be removed ONCE, and +// must NOT be removed again every time its TTL lets it back in. +// +// This is the measured production case (solana, 2026-08-21): an operator's relay-miner hosts +// sat at 78.3-79.7% success against an 80% line and flapped — broken 71-86% of the time, +// while one of them served 40 consecutive probes with zero errors at latency level with the +// operator carrying 99% of the service. +func TestCircuitBreaker_MarginalDomainDoesNotFlap(t *testing.T) { + cb := NewDomainCircuitBreaker(nil, testCircuitBreakerLogger()) + cb.defaultTTL = 20 * time.Millisecond + cb.cacheTTL = time.Millisecond + cb.failureWindow = time.Hour // one window for the whole test; isolate hysteresis from rollover + ctx := context.Background() + + const domain = "marginal.example.com" + + // 21% failure — just past the 20% threshold, the shape of the marginal hosts. + if !driveAtRate(t, cb, ctx, "solana", domain, 200, 21) { + t.Fatal("a domain over the threshold must break the first time") + } + + // Let the break expire, exactly as it does in production. + time.Sleep(cb.defaultTTL + 10*time.Millisecond) + if cb.GetBrokenDomains(ctx, "solana")[domain] { + t.Fatal("break did not expire; test cannot measure re-break") + } + + // Same behaviour again. It is still marginal, not newly worse. + if driveAtRate(t, cb, ctx, "solana", domain, 200, 21) { + t.Fatal("marginal domain re-broke after readmission: this is the production flap — " + + "it never gets the traffic to prove itself and is held out for escalating TTLs") + } +} + +// The margin must not disarm the breaker. A domain that is genuinely far past the line has to +// break again after readmission, and has to escalate — otherwise this trades one bug for a +// worse one. +// +// Production shape: two hosts at 49.7% and 58.6% success, against the same 80% line as the +// marginal hosts above. The whole point of the margin is that these two populations separate. +func TestCircuitBreaker_BadlyBrokenDomainStillRebreaks(t *testing.T) { + cb := NewDomainCircuitBreaker(nil, testCircuitBreakerLogger()) + cb.defaultTTL = 20 * time.Millisecond + cb.cacheTTL = time.Millisecond + cb.failureWindow = time.Hour + ctx := context.Background() + + const domain = "genuinely-broken.example.com" + + if !driveAtRate(t, cb, ctx, "solana", domain, 200, 50) { + t.Fatal("50% failure must break") + } + + time.Sleep(cb.defaultTTL + 10*time.Millisecond) + if cb.GetBrokenDomains(ctx, "solana")[domain] { + t.Fatal("break did not expire; test cannot measure re-break") + } + + if !driveAtRate(t, cb, ctx, "solana", domain, 200, 50) { + t.Fatal("a domain at 50% failure must STILL break after readmission — the hysteresis " + + "margin is meant to spare marginal domains, not broken ones") + } + + cb.mu.RLock() + state := cb.cache["solana"].domains[domain] + cb.mu.RUnlock() + if state.hitCount != 2 { + t.Fatalf("re-break must escalate: hitCount=%d, want 2", state.hitCount) + } +} + +// A totally dead host — no successes at all — must break on the first episode AND on every +// readmission. This is a real production shape (one host had 0 successes in 82 lifetime +// attempts) and is the case an attempt-count floor would have wrongly spared; that idea was +// tried and dropped because of this test. +func TestCircuitBreaker_DeadHostBreaksWithNoSuccesses(t *testing.T) { + cb := NewDomainCircuitBreaker(nil, testCircuitBreakerLogger()) + cb.defaultTTL = 20 * time.Millisecond + cb.cacheTTL = time.Millisecond + cb.failureWindow = time.Hour + ctx := context.Background() + + const domain = "dead.example.com" + + breakDomain(cb, ctx, "solana", domain, "retry: dead") + if !cb.GetBrokenDomains(ctx, "solana")[domain] { + t.Fatal("100% failure must break with no successes recorded") + } + + time.Sleep(cb.defaultTTL + 10*time.Millisecond) + cb.GetBrokenDomains(ctx, "solana") + + breakDomain(cb, ctx, "solana", domain, "retry: dead") + if !cb.GetBrokenDomains(ctx, "solana")[domain] { + t.Fatal("a host with no successes must re-break; 100% is far past threshold+margin") + } +} + +// The margin applies only within escalationMemory. Once a domain has behaved long enough for +// its history to lapse, it is a first offender again and the ordinary threshold applies — +// otherwise the margin would be a permanent grant of leniency to anything that ever broke. +func TestCircuitBreaker_MarginLapsesWithEscalationMemory(t *testing.T) { + cb := NewDomainCircuitBreaker(nil, testCircuitBreakerLogger()) + cb.defaultTTL = 10 * time.Millisecond + cb.cacheTTL = time.Millisecond + cb.failureWindow = time.Hour + cb.escalationMemory = 30 * time.Millisecond + ctx := context.Background() + + const domain = "lapsed.example.com" + + if !driveAtRate(t, cb, ctx, "solana", domain, 200, 21) { + t.Fatal("first break expected") + } + + // Outlive both the break and the escalation memory. + time.Sleep(cb.escalationMemory + 20*time.Millisecond) + cb.GetBrokenDomains(ctx, "solana") + + if !driveAtRate(t, cb, ctx, "solana", domain, 200, 21) { + t.Fatal("after escalation memory lapses the domain is a first offender again and the " + + "plain threshold must apply") + } +} + +// The gate's inputs must be observable at the granularity the gate DECIDES at. +// +// path_relays_total keys on eTLD+1, so an operator running several relay miners under one +// domain reports one blended rate and a per-host verdict cannot be checked against it. This +// asserts the new counter keys on the full hostname instead, and that both sides of the +// fraction are recorded — a numerator with no denominator is how the pre-rate-gate breaker +// made every single failure look like a 100% failure rate. +func TestCircuitBreaker_OutcomeMetricIsKeyedOnHostname(t *testing.T) { + cb := NewDomainCircuitBreaker(nil, testCircuitBreakerLogger()) + cb.failureWindow = time.Hour + ctx := context.Background() + + // Two hosts under ONE registrable domain, the shape that motivated the metric. + const good = "host-a.relayminer.example.com" + const bad = "host-b.relayminer.example.com" + + for i := 0; i < 12; i++ { + cb.RecordSuccess("solana", good) + } + for i := 0; i < defaultMinFailures; i++ { + cb.MarkBroken(ctx, "solana", bad, "retry: boom") + } + + read := func(domain, outcome string) float64 { + c, err := metrics.CircuitBreakerOutcomeTotal.GetMetricWithLabelValues("solana", domain, outcome) + if err != nil { + t.Fatalf("metric lookup failed for %s/%s: %v", domain, outcome, err) + } + var m dto.Metric + if err := c.(prometheus.Metric).Write(&m); err != nil { + t.Fatalf("metric write failed: %v", err) + } + return m.GetCounter().GetValue() + } + + if got := read(good, metrics.CircuitBreakerOutcomeSuccess); got != 12 { + t.Fatalf("successes for the healthy host: got %v, want 12 — the gate's DENOMINATOR "+ + "must be visible, not just its failures", got) + } + if got := read(bad, metrics.CircuitBreakerOutcomeFailure); got != float64(defaultMinFailures) { + t.Fatalf("failures for the failing host: got %v, want %d", got, defaultMinFailures) + } + // The distinction the metric exists for: one host's failures must not be attributed to + // the sibling sharing its registrable domain. + if got := read(good, metrics.CircuitBreakerOutcomeFailure); got != 0 { + t.Fatalf("healthy host was charged %v failures from its domain sibling — the counter "+ + "has collapsed to eTLD+1 and answers nothing path_relays_total does not", got) + } +} diff --git a/gateway/gateway.go b/gateway/gateway.go index 134f0938b..8a32fe1aa 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -74,6 +74,11 @@ type Gateway struct { // Optional - if nil, no cross-pod domain circuit breaking occurs. DomainCircuitBreaker *DomainCircuitBreaker + // RequestSampler fingerprints one request in N per service to answer whether a + // service's traffic is diverse or the same few requests repeated. nil disables it. + // Backs GET /admin/request-sample/{serviceId}. + RequestSampler *RequestSampler + // WebsocketConnectionLimiter bounds the number of concurrent live websocket // connections held open by this gateway. Optional - if nil, no limit is applied. WebsocketConnectionLimiter *WebsocketConnectionLimiter @@ -160,6 +165,10 @@ func (g Gateway) handleHTTPServiceRequest( return } + // Sample the request shape (method + params) — before any relay, so the sample + // describes what clients send rather than what succeeded. + g.RequestSampler.Observe(gatewayRequestCtx.serviceID, httpReq.Method, httpReq.URL.Path, gatewayRequestCtx.httpRequestBody) + // TODO_CHECK_IF_DONE(@adshmh): Pass the context with deadline to QoS once it can handle deadlines. // Build the QoS context for the target service ID using the HTTP request's payload. err = gatewayRequestCtx.BuildQoSContextFromHTTP(httpReq) diff --git a/gateway/health_check_executor.go b/gateway/health_check_executor.go index 07fd2339a..974551ae3 100644 --- a/gateway/health_check_executor.go +++ b/gateway/health_check_executor.go @@ -1212,7 +1212,7 @@ func (e *HealthCheckExecutor) ExecuteCheckViaProtocol( Msg("Health check relay request failed") // Record relay metric for failed request - metrics.RecordRelay(domain, rpcTypeStr, string(serviceID), "error", metrics.SignalMajorError, metrics.RelayTypeHealthCheck, latency.Seconds()) + metrics.RecordRelay(domain, rpcTypeStr, string(serviceID), metrics.StatusCategoryError, metrics.SignalMajorError, metrics.RelayTypeHealthCheck, latency.Seconds()) // Still publish observations for failed requests e.publishHealthCheckObservations(serviceID, endpointAddr, startTime, protocolCtx, &protocolObs) @@ -1446,7 +1446,8 @@ func (e *HealthCheckExecutor) publishHealthCheckObservations( // Apply protocol observations (for sanctioning, etc.) if observations != nil { - if err := e.protocol.ApplyHTTPObservations(observations); err != nil { + // true: these observations are a probe by construction — this executor issues them. + if err := e.protocol.ApplyHTTPObservations(observations, true); err != nil { e.logger.Debug().Err(err).Msg("Failed to apply protocol observations for health check") } } @@ -1518,9 +1519,26 @@ func (e *HealthCheckExecutor) processObservationSync( Msg("Health check observation processed synchronously for block height extraction") } -// archivalTTL is how long an archival status from health checks remains valid. -// Health checks run periodically, so this should be longer than the health check interval. -const archivalTTL = 30 * time.Minute +// ArchivalStatusTTL is how long an archival status remains valid, for BOTH sources that +// can grant it: an archival health check here, and a user-traffic confirmation in +// qos/evm. It is exported and shared so the two cannot drift. +// +// They had drifted, by 16x, in the direction that does the most damage. The health-check +// path pins an exact expected historical value in the rules file, so a node that ignores +// the block parameter and answers from current state fails it — and that verified mark +// expired in 30 minutes. The user-traffic path cannot pin a value, because the query is +// whatever a client happened to send, so it grants archival status on any successful call +// to an archival method — and that UNVERIFIED mark lasted 8 hours. +// +// Weaker evidence must not outlive stronger evidence. The comment on the 8h constant even +// claimed it "matches health check archival TTL", which is probably why nobody noticed. +// +// Should be longer than the health check interval, so a passing endpoint is re-confirmed +// before its mark lapses. +const ArchivalStatusTTL = 30 * time.Minute + +// archivalTTL is the internal alias kept for readability at this package's call sites. +const archivalTTL = ArchivalStatusTTL // markEndpointArchival marks an endpoint as archival-capable via the reputation service. // This is called ONLY after an archival health check passes ALL validations including error_detection. @@ -1774,7 +1792,8 @@ func (e *HealthCheckExecutor) ExecuteWebSocketCheckViaProtocol( Msg("Skipping websocket check reputation signal for over-serviced (stake-exhausted) supplier") default: - if err := e.protocol.ApplyWebSocketObservations(protocolObs); err != nil { + // true: a websocket health-check probe, same as the HTTP path above. + if err := e.protocol.ApplyWebSocketObservations(protocolObs, true); err != nil { e.logger.Warn(). Err(err). Str("service_id", string(serviceID)). diff --git a/gateway/healthcheck_observation_stamp_test.go b/gateway/healthcheck_observation_stamp_test.go new file mode 100644 index 000000000..bb8d292cb --- /dev/null +++ b/gateway/healthcheck_observation_stamp_test.go @@ -0,0 +1,89 @@ +package gateway + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/observation" + protocolobservations "github.com/pokt-network/path/observation/protocol" + "github.com/pokt-network/path/protocol" +) + +// observationVerdictProtocol records the isHealthCheck verdict each Apply*Observations +// caller supplies. The verdict decides whether a probe result reaches the +// volume-independent rate detectors, and it is set by the CALLER — so it has to be asserted +// at the caller, not inside the protocol implementation the caller passes it to. +type observationVerdictProtocol struct { + *mockProtocolForRetry + + httpCalls atomic.Int32 + httpFlag atomic.Bool +} + +func (m *observationVerdictProtocol) ApplyHTTPObservations(_ *protocolobservations.Observations, isHealthCheck bool) error { + m.httpCalls.Add(1) + m.httpFlag.Store(isHealthCheck) + return nil +} + +// Test_HealthCheckExecutor_MarksHTTPObservationsAsProbe guards the HTTP half of the +// probe-contamination fix at its call site. +// +// One health check produces TWO reputation signals: the relay itself (stamped in +// protocol/shannon/context.go) and this observation publish. Only this call site knows the +// observations are synthetic — the Shannon layer receiving them cannot tell — so a caller +// that passes the wrong verdict silently restores the whole bug while every +// protocol-package test stays green. +func Test_HealthCheckExecutor_MarksHTTPObservationsAsProbe(t *testing.T) { + c := require.New(t) + + proto := &observationVerdictProtocol{mockProtocolForRetry: &mockProtocolForRetry{}} + executor := NewHealthCheckExecutor(HealthCheckExecutorConfig{ + Config: &ActiveHealthChecksConfig{Enabled: true}, + Logger: polyzero.NewLogger(), + Protocol: proto, + MaxWorkers: 1, + }) + t.Cleanup(executor.Stop) + + executor.publishHealthCheckObservations( + protocol.ServiceID("solana"), + protocol.EndpointAddr("pokt1supplier-https://probe.example.com/rpc"), + time.Now(), + nil, + &protocolobservations.Observations{}, + ) + + c.Equal(int32(1), proto.httpCalls.Load(), + "the health-check executor must apply its protocol observations exactly once") + c.True(proto.httpFlag.Load(), + "health-check observations must be applied as a probe; applied as user traffic they feed "+ + "the rate detectors and the endpoint benches itself off its own probes") +} + +// Test_RequestContext_MarksHTTPObservationsAsUserTraffic is the control at the OTHER call +// site. Without it, a change that hard-codes true everywhere would pass the test above +// while making the rate detectors permanently inert for real user traffic. +func Test_RequestContext_MarksHTTPObservationsAsUserTraffic(t *testing.T) { + c := require.New(t) + + proto := &observationVerdictProtocol{mockProtocolForRetry: &mockProtocolForRetry{}} + rc := &requestContext{ + logger: polyzero.NewLogger(), + context: context.Background(), + protocol: proto, + protocolObservations: &protocolobservations.Observations{}, + gatewayObservations: &observation.GatewayObservations{}, + } + + rc.broadcastObservationsInternal() + + c.Equal(int32(1), proto.httpCalls.Load()) + c.False(proto.httpFlag.Load(), + "a user request's observations must reach the rate detectors") +} diff --git a/gateway/hedge.go b/gateway/hedge.go index bf1914556..b202e6a62 100644 --- a/gateway/hedge.go +++ b/gateway/hedge.go @@ -572,10 +572,22 @@ func (hr *hedgeRacer) recordWinner(result hedgeResult) { Bool("success", result.err == nil). Msg("🏆 Race winner determined") - metrics.RecordHedgeSupplierOutcome(result.supplierAddr, metrics.HedgeRoleWinner, result.duration.Seconds()) + metrics.RecordHedgeSupplierOutcome(hedgeOutcomeDomain(result), metrics.HedgeRoleWinner, result.duration.Seconds()) }) } +// hedgeOutcomeDomain resolves the operator domain for a hedge outcome. +// +// path_hedge_supplier_outcome_total is keyed on domain (eTLD+1), not on the +// supplier address: hedge asks an operator-level question — "did racing a +// different operator help" — and as a supplier-keyed counter it was 49,231 series +// fleet-wide with 1.8× churn, scaling with the chain's supplier set rather than +// with our traffic. Derived from endpointAddr rather than supplierAddr because the +// address alone carries no domain. +func hedgeOutcomeDomain(result hedgeResult) string { + return metrics.DomainFromEndpointAddr(string(result.endpointAddr)) +} + // recordLoser records the losing request for reputation tracking. func (hr *hedgeRacer) recordLoser(result hedgeResult) { // Track loser's supplier for X-Suppliers-Tried header using thread-safe method @@ -601,7 +613,7 @@ func (hr *hedgeRacer) recordLoser(result hedgeResult) { Err(result.err). Msg("Race loser recorded") - metrics.RecordHedgeSupplierOutcome(result.supplierAddr, metrics.HedgeRoleLoser, result.duration.Seconds()) + metrics.RecordHedgeSupplierOutcome(hedgeOutcomeDomain(result), metrics.HedgeRoleLoser, result.duration.Seconds()) } // collectLoserSync gives the loser a short (100ms) synchronous window to be recorded diff --git a/gateway/hedge_outcome_label_test.go b/gateway/hedge_outcome_label_test.go new file mode 100644 index 000000000..9d7f8c1c7 --- /dev/null +++ b/gateway/hedge_outcome_label_test.go @@ -0,0 +1,95 @@ +package gateway + +import ( + "testing" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/metrics" + "github.com/pokt-network/path/protocol" +) + +// Test_HedgeOutcome_KeysOnDomain asserts the CALL SITE, through recordWinner and +// recordLoser, not through the metric helper. +// +// path_hedge_supplier_outcome_total was re-keyed from `supplier` to `domain` on +// 2026-08-12 (49,231 series fleet-wide, 1.8× churn, scaling with the chain's +// ~5,200-supplier set rather than with our traffic). hedgeResult carries BOTH a +// supplierAddr and an endpointAddr, both strings, so passing the wrong one +// compiles and the label keeps its name — the metric-side test cannot see it. +// +// The distinguishing observable is the label VALUE: SanitizeDomainLabel collapses +// a bech32 address to DomainSupplierAddr, so a call site handing over +// supplierAddr shows up as the sentinel rather than as the operator. +func Test_HedgeOutcome_KeysOnDomain(t *testing.T) { + metrics.HedgeSupplierOutcomeTotal.Reset() + + const ( + supplierA = "pokt1ylsjqcl0yunve78etutw660a327avc26fxrlfr" + supplierB = "pokt1othersupplieraddresshere0000000000000" + ) + // Same operator, different suppliers and subdomains: must collapse to one + // domain across both the winner and the loser branch. + winner := hedgeResult{ + endpointAddr: protocol.EndpointAddr(supplierA + "-https://relayminer.eu.hedge-operator.example"), + supplierAddr: supplierA, + duration: 120 * time.Millisecond, + } + loser := hedgeResult{ + endpointAddr: protocol.EndpointAddr(supplierB + "-https://other.us.hedge-operator.example"), + supplierAddr: supplierB, + duration: 400 * time.Millisecond, + isHedge: true, + } + + hr := &hedgeRacer{logger: polyzero.NewLogger(), rc: &requestContext{}} + hr.recordWinner(winner) + hr.recordLoser(loser) + + domains := hedgeEmittedLabelValues(t, metrics.HedgeSupplierOutcomeTotal, "domain") + + require.NotContains(t, domains, supplierA, + "the winner call site is passing the supplier address where a domain is expected") + require.NotContains(t, domains, supplierB, + "the loser call site is passing the supplier address where a domain is expected") + require.NotContains(t, domains, metrics.DomainSupplierAddr, + "a call site passed a bech32 address; the sanitizer caught it, but the label is now a "+ + "sentinel instead of the operator it is supposed to name") + require.Equal(t, map[string]struct{}{"hedge-operator.example": {}}, domains, + "winner and loser on the same operator must collapse onto one domain") + + // Both roles must still be distinct series — the fix narrows the metric, it + // must not flatten win-rate into a single number. + require.Equal(t, 1.0, testutilToFloat(t, metrics.HedgeSupplierOutcomeTotal, "hedge-operator.example", metrics.HedgeRoleWinner)) + require.Equal(t, 1.0, testutilToFloat(t, metrics.HedgeSupplierOutcomeTotal, "hedge-operator.example", metrics.HedgeRoleLoser)) +} + +func hedgeEmittedLabelValues(t *testing.T, c prometheus.Collector, labelName string) map[string]struct{} { + t.Helper() + ch := make(chan prometheus.Metric, 1<<12) + c.Collect(ch) + close(ch) + + out := map[string]struct{}{} + for m := range ch { + var pb dto.Metric + require.NoError(t, m.Write(&pb)) + for _, lp := range pb.GetLabel() { + if lp.GetName() == labelName { + out[lp.GetValue()] = struct{}{} + } + } + } + return out +} + +func testutilToFloat(t *testing.T, vec *prometheus.CounterVec, labelValues ...string) float64 { + t.Helper() + var pb dto.Metric + require.NoError(t, vec.WithLabelValues(labelValues...).Write(&pb)) + return pb.GetCounter().GetValue() +} diff --git a/gateway/http_request_context.go b/gateway/http_request_context.go index f9de27c90..342452b42 100644 --- a/gateway/http_request_context.go +++ b/gateway/http_request_context.go @@ -835,7 +835,8 @@ func (rc *requestContext) broadcastObservationsInternal() { // update protocol-level observations: no errors encountered setting up the protocol context. rc.updateProtocolObservations(nil) if rc.protocolObservations != nil { - err := rc.protocol.ApplyHTTPObservations(rc.protocolObservations) + // false: user traffic. Health-check relays never reach this request context. + err := rc.protocol.ApplyHTTPObservations(rc.protocolObservations, false) if err != nil { rc.logger.Warn().Err(err).Msg("error applying protocol observations.") } diff --git a/gateway/http_request_context_handle_request.go b/gateway/http_request_context_handle_request.go index dfa71403a..2391f03a1 100644 --- a/gateway/http_request_context_handle_request.go +++ b/gateway/http_request_context_handle_request.go @@ -657,6 +657,16 @@ func (rc *requestContext) handleSingleRelayRequest() error { } checkResult := checkResponseSuccess(hedgeErr, statusCode, responseBytesForHeuristic, heuristicRPCType, jsonrpcMethod, hedgeRequestID, logger) if checkResult.Success { + // Denominator for the circuit breaker's failure-rate gate — see RecordSuccess. + // On a service with a hedge delay EVERY first attempt returns through here, + // including the vast majority where the hedge never fires, so without this + // the gate is fed almost no successes and judges every domain on a fraction + // dominated by its failures. + if rc.circuitBreaker != nil && endpointAddr != "" { + if domain := extractDomainFromEndpoint(endpointAddr); domain != "" { + rc.circuitBreaker.RecordSuccess(string(rc.serviceID), domain) + } + } // Success! Process the response for _, endpointResponse := range hedgeResponses { rc.qosCtx.UpdateWithResponse(endpointResponse.EndpointAddr, endpointResponse.Bytes, endpointResponse.HTTPStatusCode, endpointResponse.RequestID) @@ -1342,6 +1352,12 @@ func (rc *requestContext) processSinglePayloadWithRetry( batchRequestID := extractRequestIDFromPayload(payload) checkResult := checkResponseSuccess(nil, resp.HTTPStatusCode, resp.Bytes, heuristicRPCType, jsonrpcMethod, batchRequestID, logger) if checkResult.Success { + // Denominator for the circuit breaker's failure-rate gate — see RecordSuccess. + if rc.circuitBreaker != nil && resp.EndpointAddr != "" { + if domain := extractDomainFromEndpoint(resp.EndpointAddr); domain != "" { + rc.circuitBreaker.RecordSuccess(string(rc.serviceID), domain) + } + } // Extract request ID from payload resp.RequestID = batchRequestID logger.Debug(). diff --git a/gateway/protocol.go b/gateway/protocol.go index daf0509bd..4d0bd2d72 100644 --- a/gateway/protocol.go +++ b/gateway/protocol.go @@ -96,14 +96,24 @@ type Protocol interface { // - protocol: Shannon // - observation: "endpoint maxed-out or over-serviced (i.e. onchain rate limiting)" // - result: skip the endpoint for a set time period until a new session begins. - ApplyHTTPObservations(*protocolobservations.Observations) error + // + // isHealthCheck reports whether these observations came from an active health-check + // probe rather than user traffic. It is a required parameter rather than a second + // method so that every caller has to state which it is at compile time: this path + // records reputation signals, and the volume-independent rate detectors must not act + // on probe results. The health-check executor is one of only two callers and its + // observations are 100% probes, so a silent default here was silently wrong. + ApplyHTTPObservations(observations *protocolobservations.Observations, isHealthCheck bool) error // ApplyWebSocketObservations applies the supplied observations to the protocol instance's internal state. // Hypothetical example (for illustrative purposes only): // - protocol: Shannon // - observation: "endpoint maxed-out or over-serviced (i.e. onchain rate limiting)" // - result: skip the endpoint for a set time period until a new session begins. - ApplyWebSocketObservations(*protocolobservations.Observations) error + // + // isHealthCheck carries the same meaning as on ApplyHTTPObservations: probe-originated + // observations must not reach the volume-independent rate detectors. + ApplyWebSocketObservations(observations *protocolobservations.Observations, isHealthCheck bool) error // TODO_FUTURE(@adshmh): support specifying the app(s) used for sending/signing synthetic relay requests by the hydrator. // TODO_TECHDEBT: Enable the hydrator for gateway modes beyond Centralized only. diff --git a/gateway/request_sampler.go b/gateway/request_sampler.go new file mode 100644 index 000000000..0a4c13e9c --- /dev/null +++ b/gateway/request_sampler.go @@ -0,0 +1,533 @@ +package gateway + +import ( + "bytes" + "encoding/json" + "hash/fnv" + "os" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog" + + "github.com/pokt-network/path/metrics" + "github.com/pokt-network/path/protocol" +) + +// RequestSampler answers one question about a service's traffic: is it many different +// requests, or the same few requests over and over? +// +// Why it exists. Every quality signal PATH has — latency, success rate, hedge wins, the +// reputation score — rewards an endpoint that answers fast. An endpoint fronted by a cache +// answers a repeated request in sub-millisecond time without touching a node, so against +// repetitive traffic it wins every race and accumulates every reward, while against unique +// traffic it is an ordinary node. Whether a fast operator is fast or merely cached therefore +// cannot be read from the operator; it has to be read from the traffic. Nothing in PATH +// recorded what the traffic looked like: the method label exists, the params do not, and a +// thousand getAccountInfo calls for a thousand accounts and a thousand for the same account +// are one number. +// +// What it records. One request in every `rate` is fingerprinted: for JSON-RPC, each item's +// method plus its params with the JSON compacted (so formatting and the request id do not +// split one logical request into many fingerprints); for anything else, the HTTP method, +// path and compacted body. Fingerprints are counted per service in a fixed-length window; +// the previous completed window is kept so a reader always has one full window to look at. +// The table is bounded: past maxFingerprints distinct entries, new fingerprints are counted +// in an overflow bucket rather than stored — the report says so, and a large overflow is +// itself the answer (the traffic is diverse). +// +// What it costs. One hash per sampled request and a bounded table per service. No label on +// any metric carries a fingerprint or a method; the two gauges are per service_id only. +// +// What it cannot tell. It sees requests, not clients — PATH has no client identity behind +// the edge — so "the same request over and over" cannot be attributed to one sender. And a +// low uniqueness ratio is a property of the traffic, not evidence against any operator: it +// says the conditions under which a cache wins are present, not that anyone is running one. +type RequestSampler struct { + logger polylog.Logger + rate uint64 + window time.Duration + maxFingerprints int + snippetBytes int + now func() time.Time + + counter atomic.Uint64 + + mu sync.Mutex + services map[protocol.ServiceID]*serviceSample +} + +type serviceSample struct { + current *sampleWindow + previous *sampleWindow +} + +type sampleWindow struct { + start time.Time + end time.Time // zero while current + requestsSeen uint64 // every request, sampled or not + sampled uint64 // fingerprinted items (a batch contributes one per item) + overflow uint64 // items whose fingerprint was new but the table was full + fingerprints map[uint64]*fingerprintEntry + methods map[string]*methodSample +} + +type fingerprintEntry struct { + method string + snippet string + count uint64 + firstSeen time.Time + lastSeen time.Time +} + +type methodSample struct { + sampled uint64 + distinct uint64 +} + +const ( + defaultRequestSampleRate = 100 + defaultRequestSampleWindow = 10 * time.Minute + defaultRequestSampleMaxFPs = 5000 + defaultRequestSampleSnippetBytes = 200 + // requestSampleMaxWindowBytes caps how much of a body is hashed and snippeted. Bodies + // past it are fingerprinted on their prefix — a huge batch still gets one fingerprint + // per item up to the limit. + requestSampleMaxBodyBytes = 1 << 20 +) + +// NewRequestSamplerFromEnv builds a sampler from PATH_REQUEST_SAMPLE_RATE (1-in-N, default +// 100, 0 disables), PATH_REQUEST_SAMPLE_WINDOW (Go duration, default 10m) and +// PATH_REQUEST_SAMPLE_MAX_FINGERPRINTS (default 5000). Returns nil when disabled so the +// gateway and the admin endpoint can both treat "no sampler" uniformly. +func NewRequestSamplerFromEnv(logger polylog.Logger) *RequestSampler { + rate := uint64(defaultRequestSampleRate) + if v := os.Getenv("PATH_REQUEST_SAMPLE_RATE"); v != "" { + n, err := strconv.ParseUint(v, 10, 64) + if err != nil { + logger.Warn().Str("value", v).Msg("PATH_REQUEST_SAMPLE_RATE is not an unsigned integer; using default") + } else { + rate = n + } + } + if rate == 0 { + logger.Info().Msg("request sampling disabled (PATH_REQUEST_SAMPLE_RATE=0)") + return nil + } + window := defaultRequestSampleWindow + if v := os.Getenv("PATH_REQUEST_SAMPLE_WINDOW"); v != "" { + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + logger.Warn().Str("value", v).Msg("PATH_REQUEST_SAMPLE_WINDOW is not a positive duration; using default") + } else { + window = d + } + } + maxFPs := defaultRequestSampleMaxFPs + if v := os.Getenv("PATH_REQUEST_SAMPLE_MAX_FINGERPRINTS"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + logger.Warn().Str("value", v).Msg("PATH_REQUEST_SAMPLE_MAX_FINGERPRINTS is not a positive integer; using default") + } else { + maxFPs = n + } + } + return NewRequestSampler(logger, rate, window, maxFPs) +} + +// NewRequestSampler builds a sampler that fingerprints one request in every `rate`. +func NewRequestSampler(logger polylog.Logger, rate uint64, window time.Duration, maxFingerprints int) *RequestSampler { + if rate == 0 { + rate = 1 + } + return &RequestSampler{ + logger: logger.With("component", "request_sampler"), + rate: rate, + window: window, + maxFingerprints: maxFingerprints, + snippetBytes: defaultRequestSampleSnippetBytes, + now: time.Now, + services: make(map[protocol.ServiceID]*serviceSample), + } +} + +// Observe is the gateway's hook: called once per HTTP service request after the service ID +// is known. Cheap when the request is not the one-in-N sampled: one atomic increment and +// one counter bump under the lock. +func (s *RequestSampler) Observe(serviceID protocol.ServiceID, httpMethod, path string, body []byte) { + if s == nil { + return + } + n := s.counter.Add(1) + now := s.now() + + s.mu.Lock() + defer s.mu.Unlock() + + w := s.currentWindowLocked(serviceID, now) + w.requestsSeen++ + if n%s.rate != 0 { + return + } + if len(body) > requestSampleMaxBodyBytes { + body = body[:requestSampleMaxBodyBytes] + } + for _, item := range fingerprintRequest(httpMethod, path, body) { + s.recordLocked(w, item, now) + } +} + +func (s *RequestSampler) recordLocked(w *sampleWindow, item requestFingerprint, now time.Time) { + w.sampled++ + ms := w.methods[item.method] + if ms == nil { + ms = &methodSample{} + w.methods[item.method] = ms + } + ms.sampled++ + + if e, ok := w.fingerprints[item.hash]; ok { + e.count++ + e.lastSeen = now + return + } + ms.distinct++ + if len(w.fingerprints) >= s.maxFingerprints { + w.overflow++ + return + } + snippet := item.canonical + if len(snippet) > s.snippetBytes { + snippet = snippet[:s.snippetBytes] + "…" + } + w.fingerprints[item.hash] = &fingerprintEntry{ + method: item.method, + snippet: snippet, + count: 1, + firstSeen: now, + lastSeen: now, + } +} + +// currentWindowLocked returns the service's live window, rotating it if it has run past +// its length. Rotation is what publishes the gauges: they describe the last COMPLETED +// window, so a reader never sees a ratio computed over three samples. +func (s *RequestSampler) currentWindowLocked(serviceID protocol.ServiceID, now time.Time) *sampleWindow { + svc := s.services[serviceID] + if svc == nil { + svc = &serviceSample{current: newSampleWindow(now)} + s.services[serviceID] = svc + return svc.current + } + if now.Sub(svc.current.start) >= s.window { + svc.current.end = now + svc.previous = svc.current + svc.current = newSampleWindow(now) + s.publishLocked(serviceID, svc.previous) + } + return svc.current +} + +func newSampleWindow(now time.Time) *sampleWindow { + return &sampleWindow{ + start: now, + fingerprints: make(map[uint64]*fingerprintEntry), + methods: make(map[string]*methodSample), + } +} + +func (s *RequestSampler) publishLocked(serviceID protocol.ServiceID, w *sampleWindow) { + if w.sampled == 0 { + return + } + distinct := uint64(len(w.fingerprints)) + w.overflow + var top1 uint64 + for _, e := range w.fingerprints { + if e.count > top1 { + top1 = e.count + } + } + metrics.SetRequestSampleUniqueness(string(serviceID), + float64(distinct)/float64(w.sampled), + float64(top1)/float64(w.sampled)) +} + +// requestFingerprint is one logical request: a JSON-RPC item, or a whole non-JSON-RPC body. +type requestFingerprint struct { + method string + canonical string + hash uint64 +} + +// fingerprintRequest reduces a body to its fingerprints. JSON-RPC (single or batch): one per +// item, keyed on method + compacted params, id deliberately excluded. Anything else: one +// fingerprint for the HTTP method, path and compacted body. Malformed JSON falls back to the +// raw bytes so a garbage request still counts as a request. +func fingerprintRequest(httpMethod, path string, body []byte) []requestFingerprint { + trimmed := bytes.TrimSpace(body) + if len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') { + if fps := fingerprintJSONRPC(trimmed); len(fps) > 0 { + return fps + } + } + canonical := httpMethod + " " + path + if len(trimmed) > 0 { + var compact bytes.Buffer + if err := json.Compact(&compact, trimmed); err == nil { + canonical += " " + compact.String() + } else { + canonical += " " + string(trimmed) + } + } + return []requestFingerprint{{method: httpMethod + " " + path, canonical: canonical, hash: fnvHash(canonical)}} +} + +type jsonrpcItemForFingerprint struct { + Method string `json:"method"` + Params json.RawMessage `json:"params"` +} + +func fingerprintJSONRPC(body []byte) []requestFingerprint { + var items []jsonrpcItemForFingerprint + if body[0] == '[' { + if err := json.Unmarshal(body, &items); err != nil { + return nil + } + } else { + var single jsonrpcItemForFingerprint + if err := json.Unmarshal(body, &single); err != nil { + return nil + } + items = []jsonrpcItemForFingerprint{single} + } + out := make([]requestFingerprint, 0, len(items)) + for _, it := range items { + if it.Method == "" { + continue + } + canonical := it.Method + if len(it.Params) > 0 { + var compact bytes.Buffer + if err := json.Compact(&compact, it.Params); err == nil { + canonical += " " + compact.String() + } else { + canonical += " " + string(it.Params) + } + } + out = append(out, requestFingerprint{method: it.Method, canonical: canonical, hash: fnvHash(canonical)}) + } + return out +} + +func fnvHash(s string) uint64 { + h := fnv.New64a() + _, _ = h.Write([]byte(s)) + return h.Sum64() +} + +// --------------------------------------------------------------------------- +// Reporting — backs GET /admin/request-sample/{serviceId} +// --------------------------------------------------------------------------- + +// RequestSampleReport is the JSON body of the admin endpoint for one service and window. +type RequestSampleReport struct { + ServiceID string `json:"service_id"` + Window string `json:"window"` // "current" | "previous" + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end,omitempty"` + WindowLength string `json:"window_length"` + SampleRate string `json:"sample_rate"` // "1-in-N" + RequestsSeen uint64 `json:"requests_seen"` + Sampled uint64 `json:"sampled"` + Distinct uint64 `json:"distinct"` + TableOverflow uint64 `json:"table_overflow"` + MaxFingerprint int `json:"max_fingerprints"` + // Uniqueness = distinct / sampled. 1.0 = every sampled request different; near 0 = the + // same few requests repeated. + Uniqueness float64 `json:"uniqueness"` + // Top1Share / TopNShare = fraction of sampled requests that were the single most + // repeated fingerprint / the N listed below. A high Top1Share on a method where + // parameters should vary (account lookups, transactions) is the shape to look for. + Top1Share float64 `json:"top1_share"` + TopNShare float64 `json:"topn_share"` + Methods []RequestSampleMethodEntry `json:"methods"` + Top []RequestSampleEntry `json:"top"` +} + +// RequestSampleMethodEntry gives per-method sampled vs distinct counts. Uniqueness per +// method is the more telling number: block-height calls are legitimately repetitive, +// account lookups are not. +type RequestSampleMethodEntry struct { + Method string `json:"method"` + Sampled uint64 `json:"sampled"` + Distinct uint64 `json:"distinct"` + Uniqueness float64 `json:"uniqueness"` + Share float64 `json:"share"` +} + +// RequestSampleEntry is one fingerprint. +type RequestSampleEntry struct { + Method string `json:"method"` + Count uint64 `json:"count"` + Share float64 `json:"share"` + FirstSeen time.Time `json:"first_seen"` + LastSeen time.Time `json:"last_seen"` + Snippet string `json:"snippet"` +} + +// RequestSampleSummary is one row of GET /admin/request-sample (all services). +type RequestSampleSummary struct { + ServiceID string `json:"service_id"` + Window string `json:"window"` + RequestsSeen uint64 `json:"requests_seen"` + Sampled uint64 `json:"sampled"` + Distinct uint64 `json:"distinct"` + Uniqueness float64 `json:"uniqueness"` + Top1Share float64 `json:"top1_share"` + TopMethod string `json:"top_method"` +} + +// Report renders one service's window. previous=true reads the last completed window; +// otherwise the live one. found=false when the service has not been observed at all. +func (s *RequestSampler) Report(serviceID string, previous bool, top int) (report RequestSampleReport, found bool) { + if s == nil { + return RequestSampleReport{}, false + } + if top <= 0 { + top = 20 + } + s.mu.Lock() + defer s.mu.Unlock() + + svc := s.services[protocol.ServiceID(serviceID)] + if svc == nil { + return RequestSampleReport{}, false + } + // Rotate if due, so "previous" is never staler than one window. + s.currentWindowLocked(protocol.ServiceID(serviceID), s.now()) + w, label := svc.current, "current" + if previous { + if svc.previous == nil { + return RequestSampleReport{}, false + } + w, label = svc.previous, "previous" + } + return s.renderLocked(serviceID, label, w, top), true +} + +func (s *RequestSampler) renderLocked(serviceID, label string, w *sampleWindow, top int) RequestSampleReport { + r := RequestSampleReport{ + ServiceID: serviceID, + Window: label, + WindowStart: w.start, + WindowEnd: w.end, + WindowLength: s.window.String(), + SampleRate: "1-in-" + strconv.FormatUint(s.rate, 10), + RequestsSeen: w.requestsSeen, + Sampled: w.sampled, + Distinct: uint64(len(w.fingerprints)) + w.overflow, + TableOverflow: w.overflow, + MaxFingerprint: s.maxFingerprints, + } + if w.sampled == 0 { + return r + } + r.Uniqueness = float64(r.Distinct) / float64(w.sampled) + + entries := make([]RequestSampleEntry, 0, len(w.fingerprints)) + for _, e := range w.fingerprints { + entries = append(entries, RequestSampleEntry{ + Method: e.method, + Count: e.count, + Share: float64(e.count) / float64(w.sampled), + FirstSeen: e.firstSeen, + LastSeen: e.lastSeen, + Snippet: e.snippet, + }) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Count != entries[j].Count { + return entries[i].Count > entries[j].Count + } + return entries[i].Snippet < entries[j].Snippet + }) + if len(entries) > 0 { + r.Top1Share = entries[0].Share + } + if len(entries) > top { + entries = entries[:top] + } + for _, e := range entries { + r.TopNShare += e.Share + } + r.Top = entries + + r.Methods = make([]RequestSampleMethodEntry, 0, len(w.methods)) + for m, ms := range w.methods { + r.Methods = append(r.Methods, RequestSampleMethodEntry{ + Method: m, + Sampled: ms.sampled, + Distinct: ms.distinct, + Uniqueness: float64(ms.distinct) / float64(ms.sampled), + Share: float64(ms.sampled) / float64(w.sampled), + }) + } + sort.Slice(r.Methods, func(i, j int) bool { + if r.Methods[i].Sampled != r.Methods[j].Sampled { + return r.Methods[i].Sampled > r.Methods[j].Sampled + } + return r.Methods[i].Method < r.Methods[j].Method + }) + return r +} + +// Summary renders one row per observed service, sorted by requests seen. Uses the +// previous (completed) window when there is one, otherwise the live one. +func (s *RequestSampler) Summary() []RequestSampleSummary { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + out := make([]RequestSampleSummary, 0, len(s.services)) + for id := range s.services { + s.currentWindowLocked(id, now) + svc := s.services[id] + w, label := svc.current, "current" + if svc.previous != nil { + w, label = svc.previous, "previous" + } + row := RequestSampleSummary{ServiceID: string(id), Window: label, RequestsSeen: w.requestsSeen, Sampled: w.sampled} + row.Distinct = uint64(len(w.fingerprints)) + w.overflow + if w.sampled > 0 { + row.Uniqueness = float64(row.Distinct) / float64(w.sampled) + var top1 uint64 + for _, e := range w.fingerprints { + if e.count > top1 { + top1 = e.count + } + } + row.Top1Share = float64(top1) / float64(w.sampled) + var topM string + var topN uint64 + for m, ms := range w.methods { + if ms.sampled > topN || (ms.sampled == topN && m < topM) { + topM, topN = m, ms.sampled + } + } + row.TopMethod = topM + } + out = append(out, row) + } + sort.Slice(out, func(i, j int) bool { + if out[i].RequestsSeen != out[j].RequestsSeen { + return out[i].RequestsSeen > out[j].RequestsSeen + } + return out[i].ServiceID < out[j].ServiceID + }) + return out +} diff --git a/gateway/request_sampler_test.go b/gateway/request_sampler_test.go new file mode 100644 index 000000000..089194644 --- /dev/null +++ b/gateway/request_sampler_test.go @@ -0,0 +1,177 @@ +package gateway + +import ( + "testing" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/metrics" + "github.com/pokt-network/path/protocol" +) + +func newTestSampler(rate uint64, window time.Duration, maxFPs int) (*RequestSampler, *time.Time) { + s := NewRequestSampler(polyzero.NewLogger(), rate, window, maxFPs) + now := time.Date(2026, 8, 21, 20, 0, 0, 0, time.UTC) + s.now = func() time.Time { return now } + return s, &now +} + +// The fingerprint is the logical request: method + params. The JSON-RPC id and formatting +// must not split one request into many, or repeated traffic with rotating ids reads as +// diverse — which is exactly the traffic shape the sampler exists to expose. +func TestRequestSampler_FingerprintIgnoresIDAndFormatting(t *testing.T) { + s, _ := newTestSampler(1, time.Hour, 100) + svc := protocol.ServiceID("solana") + bodies := []string{ + `{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["Vote111111111111111111111111111111111111111"]}`, + `{"jsonrpc":"2.0","id":2,"method":"getAccountInfo","params":["Vote111111111111111111111111111111111111111"]}`, + `{"jsonrpc": "2.0", "id": "abc", "method": "getAccountInfo", "params": [ "Vote111111111111111111111111111111111111111" ]}`, + `{"id":9,"method":"getAccountInfo","params":["Vote111111111111111111111111111111111111111"],"jsonrpc":"2.0"}`, + } + for _, b := range bodies { + s.Observe(svc, "POST", "/v1", []byte(b)) + } + // One genuinely different request. + s.Observe(svc, "POST", "/v1", []byte(`{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["11111111111111111111111111111111"]}`)) + + r, ok := s.Report("solana", false, 10) + require.True(t, ok) + require.Equal(t, uint64(5), r.RequestsSeen) + require.Equal(t, uint64(5), r.Sampled) + require.Equal(t, uint64(2), r.Distinct, "four id/formatting variants of one request plus one other request = 2 fingerprints") + require.InDelta(t, 0.4, r.Uniqueness, 1e-9) + require.InDelta(t, 0.8, r.Top1Share, 1e-9) + require.Equal(t, "getAccountInfo", r.Top[0].Method) + require.Equal(t, uint64(4), r.Top[0].Count) + require.Len(t, r.Methods, 1) + require.Equal(t, uint64(2), r.Methods[0].Distinct) +} + +// A batch contributes one fingerprint per item, so a client that packs the same call into +// batches is measured on its calls, not on its envelopes. +func TestRequestSampler_BatchItemsAreSeparateFingerprints(t *testing.T) { + s, _ := newTestSampler(1, time.Hour, 100) + svc := protocol.ServiceID("eth") + s.Observe(svc, "POST", "/v1", []byte(`[{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]},{"jsonrpc":"2.0","id":2,"method":"eth_blockNumber","params":[]},{"jsonrpc":"2.0","id":3,"method":"eth_getBalance","params":["0xabc","latest"]}]`)) + + r, ok := s.Report("eth", false, 10) + require.True(t, ok) + require.Equal(t, uint64(1), r.RequestsSeen) + require.Equal(t, uint64(3), r.Sampled) + require.Equal(t, uint64(2), r.Distinct) + require.Equal(t, "eth_blockNumber", r.Top[0].Method) + require.Equal(t, uint64(2), r.Top[0].Count) +} + +// Only one request in `rate` is fingerprinted, but every request is counted as seen — the +// report must make the sampling visible rather than present a 1-in-100 sample as the total. +func TestRequestSampler_SamplesOneInN(t *testing.T) { + s, _ := newTestSampler(10, time.Hour, 100) + svc := protocol.ServiceID("poly") + for i := 0; i < 100; i++ { + s.Observe(svc, "POST", "/v1", []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}`)) + } + r, ok := s.Report("poly", false, 10) + require.True(t, ok) + require.Equal(t, uint64(100), r.RequestsSeen) + require.Equal(t, uint64(10), r.Sampled) + require.Equal(t, "1-in-10", r.SampleRate) +} + +// The table is bounded. Past the cap, new fingerprints are counted (so uniqueness stays +// honest) but not stored, and the report says how many were dropped. +func TestRequestSampler_TableIsBoundedAndOverflowIsCountedNotStored(t *testing.T) { + s, _ := newTestSampler(1, time.Hour, 3) + svc := protocol.ServiceID("solana") + for _, acct := range []string{"a", "b", "c", "d", "e"} { + s.Observe(svc, "POST", "/v1", []byte(`{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["`+acct+`"]}`)) + } + r, ok := s.Report("solana", false, 10) + require.True(t, ok) + require.Equal(t, uint64(5), r.Sampled) + require.Equal(t, uint64(5), r.Distinct, "overflowed fingerprints still count as distinct") + require.Equal(t, uint64(2), r.TableOverflow) + require.Len(t, r.Top, 3, "only the stored fingerprints are listed") + require.InDelta(t, 1.0, r.Uniqueness, 1e-9) +} + +// Windows rotate on the clock; the completed window is kept as "previous" and is what the +// gauges describe, so a reader always has a full window and never a ratio over three samples. +func TestRequestSampler_WindowRotationKeepsPreviousAndPublishes(t *testing.T) { + s, now := newTestSampler(1, 10*time.Minute, 100) + svc := protocol.ServiceID("solana") + same := []byte(`{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[]}`) + for i := 0; i < 4; i++ { + s.Observe(svc, "POST", "/v1", same) + } + _, ok := s.Report("solana", true, 10) + require.False(t, ok, "no completed window yet") + + *now = now.Add(11 * time.Minute) + s.Observe(svc, "POST", "/v1", []byte(`{"jsonrpc":"2.0","id":1,"method":"getSlot","params":[{"commitment":"finalized"}]}`)) + + prev, ok := s.Report("solana", true, 10) + require.True(t, ok) + require.Equal(t, "previous", prev.Window) + require.Equal(t, uint64(4), prev.Sampled) + require.Equal(t, uint64(1), prev.Distinct) + require.False(t, prev.WindowEnd.IsZero()) + + cur, ok := s.Report("solana", false, 10) + require.True(t, ok) + require.Equal(t, "current", cur.Window) + require.Equal(t, uint64(1), cur.Sampled) + + // The gauges describe the completed window: 1 distinct / 4 sampled. + require.InDelta(t, 0.25, gaugeValue(t, "path_request_sample_uniqueness", "solana"), 1e-9) + require.InDelta(t, 1.0, gaugeValue(t, "path_request_sample_top1_share", "solana"), 1e-9) + + sum := s.Summary() + require.Len(t, sum, 1) + require.Equal(t, "previous", sum[0].Window) + require.Equal(t, "getSlot", sum[0].TopMethod) +} + +// A nil sampler (sampling disabled) is a no-op everywhere the gateway and router touch it. +func TestRequestSampler_NilIsNoOp(t *testing.T) { + var s *RequestSampler + s.Observe("solana", "POST", "/v1", []byte(`{}`)) + _, ok := s.Report("solana", false, 10) + require.False(t, ok) + require.Nil(t, s.Summary()) +} + +// Non-JSON-RPC traffic (REST, CometBFT GET paths) is fingerprinted on method + path + body. +func TestRequestSampler_RESTFingerprint(t *testing.T) { + s, _ := newTestSampler(1, time.Hour, 100) + svc := protocol.ServiceID("xrplevm") + s.Observe(svc, "GET", "/v1/cosmos/base/tendermint/v1beta1/blocks/latest", nil) + s.Observe(svc, "GET", "/v1/cosmos/base/tendermint/v1beta1/blocks/latest", nil) + s.Observe(svc, "GET", "/v1/status", nil) + r, ok := s.Report("xrplevm", false, 10) + require.True(t, ok) + require.Equal(t, uint64(3), r.Sampled) + require.Equal(t, uint64(2), r.Distinct) + require.Equal(t, "GET /v1/cosmos/base/tendermint/v1beta1/blocks/latest", r.Top[0].Method) +} + +// gaugeValue reads a per-service gauge from the production metric vec. +func gaugeValue(t *testing.T, name, serviceID string) float64 { + t.Helper() + var g prometheus.Gauge + switch name { + case "path_request_sample_uniqueness": + g = metrics.RequestSampleUniqueness.WithLabelValues(serviceID) + case "path_request_sample_top1_share": + g = metrics.RequestSampleTop1Share.WithLabelValues(serviceID) + default: + t.Fatalf("unknown gauge %s", name) + } + var m dto.Metric + require.NoError(t, g.Write(&m)) + return m.GetGauge().GetValue() +} diff --git a/gateway/retry_test.go b/gateway/retry_test.go index 62b10ab86..b0b64d1ac 100644 --- a/gateway/retry_test.go +++ b/gateway/retry_test.go @@ -377,11 +377,11 @@ func (m *mockProtocolForRetry) SupportedGatewayModes() []protocol.GatewayMode { return nil } -func (m *mockProtocolForRetry) ApplyHTTPObservations(observations *protocolobservations.Observations) error { +func (m *mockProtocolForRetry) ApplyHTTPObservations(observations *protocolobservations.Observations, _ bool) error { return nil } -func (m *mockProtocolForRetry) ApplyWebSocketObservations(observations *protocolobservations.Observations) error { +func (m *mockProtocolForRetry) ApplyWebSocketObservations(observations *protocolobservations.Observations, _ bool) error { return nil } diff --git a/gateway/websocket_check_reputation_test.go b/gateway/websocket_check_reputation_test.go index ad52ede87..86b8cc9aa 100644 --- a/gateway/websocket_check_reputation_test.go +++ b/gateway/websocket_check_reputation_test.go @@ -54,7 +54,7 @@ func (m *websocketCheckProtocol) CheckWebsocketConnection( return nil, m.obs } -func (m *websocketCheckProtocol) ApplyWebSocketObservations(obs *protocolobservations.Observations) error { +func (m *websocketCheckProtocol) ApplyWebSocketObservations(obs *protocolobservations.Observations, isHealthCheck bool) error { if obs == nil || obs.GetShannon() == nil { return nil } @@ -67,6 +67,9 @@ func (m *websocketCheckProtocol) ApplyWebSocketObservations(obs *protocolobserva if connObs.GetErrorType() == protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_UNSPECIFIED { signal = reputation.NewSuccessSignal(0) } + // Mirrors the real implementation, which stamps the caller's verdict onto the + // signal so the volume-independent rate detectors can exclude probe results. + signal.IsHealthCheck = isHealthCheck if err := m.rep.RecordSignal(context.Background(), reputation.EndpointKey{}, signal); err != nil { return err } @@ -278,6 +281,46 @@ func Test_WebsocketCheck_RecordsExactlyOneReputationSignal(t *testing.T) { }) } +// Test_WebsocketCheck_StampsSignalsAsHealthCheck covers the websocket half of the +// probe-contamination fix. +// +// Both reputation writers on this path must stamp the signal, and they are reached by +// different routes: a FAILING check goes out through ApplyWebSocketObservations, a PASSING +// one is recorded directly by the executor because the protocol emits no observation on +// success. Fixing only one leaves half of every websocket service's probe volume feeding +// the volume-independent rate detectors, which is what benches an endpoint that no user +// ever complained about. +func Test_WebsocketCheck_StampsSignalsAsHealthCheck(t *testing.T) { + t.Run("failing check - stamped on the observation path", func(t *testing.T) { + c := require.New(t) + + executor, _, rep, svcConfig := newWebsocketCheckExecutor(t, failedWebsocketObservation("i/o timeout")) + + executor.runEndpointChecks(context.Background(), wsCheckService, wsCheckEndpoint, svcConfig, false, true, true) + executor.wsPool.StopAndWait() + + signals := rep.RecordedSignals() + c.Len(signals, 1) + c.True(signals[0].IsHealthCheck, + "a websocket probe failure must not be able to trip the rate detectors on its own") + }) + + t.Run("passing check - stamped by the executor", func(t *testing.T) { + c := require.New(t) + + executor, _, rep, svcConfig := newWebsocketCheckExecutor(t, nil) + + executor.runEndpointChecks(context.Background(), wsCheckService, wsCheckEndpoint, svcConfig, false, true, true) + executor.wsPool.StopAndWait() + + signals := rep.RecordedSignals() + c.Len(signals, 1) + c.True(signals[0].IsHealthCheck, + "a probe success must be excluded from the rate detectors too: the EWMAs are ratios, "+ + "so leaving successes in the denominator while excluding failures biases them") + }) +} + // websocketObservationError is the whole basis of the failure verdict, so pin its contract: // only a real failure produces an error, and the error stays diagnosable. func Test_websocketObservationError(t *testing.T) { diff --git a/gateway/websocket_request_context.go b/gateway/websocket_request_context.go index 371ff8d57..637ba6595 100644 --- a/gateway/websocket_request_context.go +++ b/gateway/websocket_request_context.go @@ -494,7 +494,8 @@ func (wrc *websocketRequestContext) BroadcastMessageObservations( } if protocolObservations := messageObservations.GetProtocol(); protocolObservations != nil { - err := wrc.protocol.ApplyWebSocketObservations(protocolObservations) + // false: a user's websocket connection. + err := wrc.protocol.ApplyWebSocketObservations(protocolObservations, false) if err != nil { wrc.logger.Warn().Err(err).Msg("error applying protocol observations for websocket.") } diff --git a/go.mod b/go.mod index 6084f500f..2cd4ead0f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pokt-network/path -go 1.26.4 +go 1.26.6 // DEVELOPER_TIP: Uncomment to use a local copies // replace github.com/pokt-network/poktroll => /Users/olshansky/workspace/pocket/poktroll @@ -26,9 +26,9 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/ory/dockertest/v3 v3.12.0 - github.com/pokt-network/poktroll v0.1.34 + github.com/pokt-network/poktroll v0.1.35 github.com/pokt-network/ring-go v0.2.0 - github.com/pokt-network/shannon-sdk v0.0.0-20260702172744-c2af5007ed72 + github.com/pokt-network/shannon-sdk v0.0.0-20260812141256-a508808fbbe0 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 github.com/redis/go-redis/v9 v9.19.0 @@ -255,6 +255,7 @@ require ( github.com/rs/cors v1.11.1 // indirect github.com/rs/zerolog v1.34.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect github.com/sasha-s/go-deadlock v0.3.9 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/sirupsen/logrus v1.9.4 // indirect @@ -301,13 +302,13 @@ require ( go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.17.0 // indirect - golang.org/x/crypto v0.53.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.271.0 // indirect google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect diff --git a/go.sum b/go.sum index 862a20bd7..11bbbb2b4 100644 --- a/go.sum +++ b/go.sum @@ -327,6 +327,8 @@ github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654 h1:XOPLOMn/zT4jIgxfx github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/cli v29.2.1+incompatible h1:n3Jt0QVCN65eiVBoUTZQM9mcQICCJt3akW4pKAbKdJg= github.com/docker/cli v29.2.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= @@ -824,12 +826,12 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pokt-network/go-dleq v0.0.0-20250925202155-488f42ad642a h1:zgSFcOX8m9dmriiqPiRbGJJpR3sJCFxLskxQ7/ofS0o= github.com/pokt-network/go-dleq v0.0.0-20250925202155-488f42ad642a/go.mod h1:KVsT8HO2EXHwMY1wnyYhR/lSZFNFIit8dJ+aDb1dSkA= -github.com/pokt-network/poktroll v0.1.34 h1:QlcUOyLgpNuEXIyukECPDWZQROUXUZD+nCTpQWmq5xw= -github.com/pokt-network/poktroll v0.1.34/go.mod h1:K7v1yjw6ZQFGBOd31LiTjPjIUBSzKTlOIckFaKC8PUg= +github.com/pokt-network/poktroll v0.1.35 h1:JdJYY9y5Uotko5IFBdptOegntXY+ez79zzJBlhMqvh8= +github.com/pokt-network/poktroll v0.1.35/go.mod h1:cH4r4HaR1treTgNaFR1gJVRrODkFID9RM78NT7W3G/Y= github.com/pokt-network/ring-go v0.2.0 h1:jXF/SOS8DgRfCCJBKXket5dBkTTqMDxplX+EnE4+aNU= github.com/pokt-network/ring-go v0.2.0/go.mod h1:B6wWq+Pj19jNyV93Hyr1TD26gDsrIYV9tYGiaJR89RE= -github.com/pokt-network/shannon-sdk v0.0.0-20260702172744-c2af5007ed72 h1:wynFx7tJRHC8XSHaWCkcvkQs26x6yulgWmaFyGX4c2M= -github.com/pokt-network/shannon-sdk v0.0.0-20260702172744-c2af5007ed72/go.mod h1:vZ40ZUGGOTkKW+J2WgMrT8TISoyUXFyfiaqlYcntXpQ= +github.com/pokt-network/shannon-sdk v0.0.0-20260812141256-a508808fbbe0 h1:++QTUCRbK0l8yc8ZVWaZZ5Un40yoZBLrATRwvUforMY= +github.com/pokt-network/shannon-sdk v0.0.0-20260812141256-a508808fbbe0/go.mod h1:YoFQsznZEk/ofl+sQt68jjGpUrkkAkFVSpINBNAyUVo= github.com/pokt-network/smt v0.14.1 h1:q8pZCo01RY+kzGurRcHArSGGEV3UhBywUZnbVn9nDM8= github.com/pokt-network/smt v0.14.1/go.mod h1:TehzlxITd3EqLzo428VY0QID7Ajdn7QJP4ZzdPiYbZE= github.com/pokt-network/smt/kvstore/pebble v0.0.0-20240822175047-21ea8639c188 h1:QK1WmFKQ/OzNVob/br55Brh+EFbWhcdq41WGC8UMihM= @@ -900,6 +902,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sasha-s/go-deadlock v0.3.9 h1:fiaT9rB7g5sr5ddNZvlwheclN9IP86eFW9WgqlEQV+w= github.com/sasha-s/go-deadlock v0.3.9/go.mod h1:KuZj51ZFmx42q/mPaYbRk0P1xcwe697zsJKE03vD4/Y= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1085,8 +1089,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1106,8 +1110,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1151,8 +1155,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1211,20 +1215,20 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -1250,8 +1254,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/makefiles/debug.mk b/makefiles/debug.mk index 757e1e175..f5ae9a301 100644 --- a/makefiles/debug.mk +++ b/makefiles/debug.mk @@ -20,6 +20,6 @@ check_graphviz: debug_goroutines: check_docker @docker run --rm \ --network=host \ - golang:1.25-alpine \ + golang:1.26-alpine \ apk add --no-cache graphviz && \ go tool pprof -http="0.0.0.0:8081" http://localhost:6060/debug/pprof/goroutine diff --git a/makefiles/release.mk b/makefiles/release.mk index db1c42815..fb79a340c 100644 --- a/makefiles/release.mk +++ b/makefiles/release.mk @@ -144,23 +144,38 @@ release_build_cross: release_build_nocgo release_build_cgo ## Build both CGO-dis @echo "All binaries built successfully!" .PHONY: release_build_nocgo +# Platforms build CONCURRENTLY. `go build` already parallelises internally, so the win comes +# from overlapping each target's largely single-threaded link phase rather than from the +# compile phase — expect a partial speedup, not a halving, and less of one on a CPU-starved +# runner. +# +# Failures must be propagated by hand: `set -e` does not fire for a background job, so a +# failed cross-compile would otherwise leave this target reporting success with a missing or +# stale binary, which the image build would then happily package. release_build_nocgo: ## Build CGO-disabled (static-friendly) binaries for multiple platforms - @echo "Building (CGO=0) binaries for multiple platforms..." + @echo "Building (CGO=0) binaries for multiple platforms (concurrently)..." @mkdir -p $(RELEASE_DIR) @set -e; \ + pids=""; \ for platform in $(RELEASE_PLATFORMS); do \ GOOS=$${platform%%/*}; \ GOARCH=$${platform##*/}; \ out_nocgo="$(RELEASE_DIR)/path-$$GOOS-$$GOARCH"; \ echo "→ CGO=0: $$GOOS/$$GOARCH"; \ TAGS="$(NOCGO_EFFECTIVE_TAGS)"; \ - if [ -n "$$TAGS" ]; then \ - CGO_ENABLED=0 GOOS=$$GOOS GOARCH=$$GOARCH go build -tags "$$TAGS" -ldflags '$(LDFLAGS)' -o "$$out_nocgo" ./cmd; \ - else \ - CGO_ENABLED=0 GOOS=$$GOOS GOARCH=$$GOARCH go build -ldflags '$(LDFLAGS)' -o "$$out_nocgo" ./cmd; \ - fi; \ - echo " ✓ Built $$out_nocgo"; \ - done + ( \ + if [ -n "$$TAGS" ]; then \ + CGO_ENABLED=0 GOOS=$$GOOS GOARCH=$$GOARCH go build -tags "$$TAGS" -ldflags '$(LDFLAGS)' -o "$$out_nocgo" ./cmd; \ + else \ + CGO_ENABLED=0 GOOS=$$GOOS GOARCH=$$GOARCH go build -ldflags '$(LDFLAGS)' -o "$$out_nocgo" ./cmd; \ + fi; \ + echo " ✓ Built $$out_nocgo"; \ + ) & \ + pids="$$pids $$!"; \ + done; \ + rc=0; \ + for pid in $$pids; do wait $$pid || rc=1; done; \ + if [ $$rc -ne 0 ]; then echo " ✗ one or more platform builds FAILED"; exit 1; fi .PHONY: release_build_cgo release_build_cgo: ## Build CGO-enabled (glibc) binaries for multiple platforms diff --git a/metrics/cardinality_guard.go b/metrics/cardinality_guard.go index a1266cbb4..c8872d717 100644 --- a/metrics/cardinality_guard.go +++ b/metrics/cardinality_guard.go @@ -38,6 +38,52 @@ import ( // to cover realistic active per-supplier workloads (~1000 suppliers × ~5 // active services × small fan-out) without exposing the heap to a runaway // label leak. +// +// ⭐ WHAT THIS CANNOT BOUND, and why no cap value fixes it (measured 2026-08-12) +// +// A guard bounds the number of label tuples LIVE IN THIS PROCESS'S REGISTRY. It +// does NOT bound the number of distinct series the scraping Prometheus has to +// store, and those two numbers diverge without limit when a label's value set +// rotates over time. +// +// path_supplier_signal_total was guarded, honored its guard at ~26% of the cap +// (6,523 live tuples on one pod), and was still one of the two largest sources of +// series in the entire gateway job: 60,674 DISTINCT series over that pod's 7.7h +// life. path_supplier_reputation_score was worse — 4,510 live, 74,639 distinct, +// 16.5×, ~232K series/pod/day, each retained for the full 6-day window. The +// control is path_relay_latency_seconds_bucket at exactly 1.0×: its labels are +// persistent, so its registry count and its TSDB cost are the same number. +// +// Eviction is NOT the cause and removing it would NOT help: evicting a tuple and +// later re-admitting it recreates the SAME label set, hence the same Prometheus +// series with a gap in it, never a new one. The distinct-series count is +// identical with or without eviction — eviction only decides whether the cost +// lands on this pod's heap as well. +// +// So a metric can sit at 20% of its cap forever and still be the most expensive +// thing in the TSDB. The only thing that bounds the series stream is the size of +// each label's VALUE SET. Concretely, when adding a label: +// +// - Bounded by our config (service_id, rpc_type, reason, status class, role): +// safe. The set is fixed at deploy time. +// - Bounded by the operator set (domain / eTLD+1): safe. 13 values measured +// fleet-wide, and it grows only when a new operator joins. +// - Bounded by the CHAIN (supplier address): NOT safe at any cap value. ~5,200 +// suppliers and growing with the network, rotating in and out of sessions +// every ~20 blocks. Aggregate to domain, or serve the per-supplier question +// from /ready/?detailed=true, which is a point lookup rather than a +// retained timeseries. +// - Client-controlled (method, path): only a guard makes this survivable, and +// then only because the cap converts unbounded minting into a bounded cost +// plus a WARN. See observationPipelineGuard. +// +// Diagnostic, per pod (see the PNF follow-up report for the full recipe): +// +// count(count_over_time({pod=""}[10m])) # live +// count(count_over_time({pod=""}[8h])) # distinct over 8h +// +// A ratio above ~1.5 means the label set rotates and the metric costs multiples +// of what its instantaneous count suggests. const DefaultSeriesLimit = 25_000 // seriesLimitEnvVar overrides DefaultSeriesLimit at process start. @@ -161,14 +207,13 @@ func InitCardinalityGuards(l polylog.Logger) { // guards must not end up on a process-wide list. func packageGuards() []*cardinalityGuard { return []*cardinalityGuard{ - supplierSignalGuard, - supplierReputationGuard, hedgeSupplierGuard, qosFilterRejectionGuard, healthCheckStatusGuard, probationEventsGuard, observationPipelineGuard, circuitBreakerEventsGuard, + circuitBreakerOutcomeGuard, rpcTypeFallbackGuard, } } @@ -413,15 +458,13 @@ func hashLabelValues(labelValues []string) uint64 { // keyed on its metric's FULL label tuple, in declaration order, so eviction can // delete precisely the series it reclaims (see DefaultSeriesLimit). var ( - supplierSignalGuard = newCardinalityGuard("supplier_signal_total", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { - SupplierSignalTotal.DeleteLabelValues(lv...) - }) - - supplierReputationGuard = newCardinalityGuard("supplier_reputation_score", defaultSeriesLimit). - withEviction(defaultGuardIdleWindow, func(lv []string) { - SupplierReputationScore.DeleteLabelValues(lv...) - }) + // supplierSignalGuard and supplierReputationGuard were removed along with + // path_supplier_signal_total and path_supplier_reputation_score (2026-08-12). + // Both metrics honored their cap and were still among the largest series + // sources in the gateway job — see the finding recorded on + // DefaultSeriesLimit. Their remaining two peers below are now keyed on + // `domain` rather than `supplier`, so they are backstops rather than working + // caps. hedgeSupplierGuard = newCardinalityGuard("hedge_supplier_latency_seconds", defaultSeriesLimit). withEviction(defaultGuardIdleWindow, func(lv []string) { @@ -483,6 +526,13 @@ var ( DomainCircuitBreakerEventsTotal.DeleteLabelValues(lv...) }) + // circuitBreakerOutcomeGuard — same service_id x domain pair as its sibling above, minus + // reason_category x event. Guarded on the same principle, not on an observed incident. + circuitBreakerOutcomeGuard = newCardinalityGuard("circuit_breaker_outcome_total", defaultSeriesLimit). + withEviction(defaultGuardIdleWindow, func(lv []string) { + CircuitBreakerOutcomeTotal.DeleteLabelValues(lv...) + }) + // rpcTypeFallbackGuard — backstop only. The real fix was dropping the // `supplier` label (3,289 values doing essentially all of the metric's // 201,068-series multiplication against 9 domains × 12 service_ids). diff --git a/metrics/cardinality_regression_test.go b/metrics/cardinality_regression_test.go index a7bfeba22..7d5b6516d 100644 --- a/metrics/cardinality_regression_test.go +++ b/metrics/cardinality_regression_test.go @@ -1,10 +1,13 @@ package metrics import ( + "errors" "fmt" + "strings" "testing" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -154,3 +157,199 @@ func Test_RPCTypeFallback_SupplierLabelDropped(t *testing.T) { require.Empty(t, collectLabelValues(t, RPCTypeFallbackTotal, LabelSupplier), "supplier is still being emitted as a label") } + +// ============================================================================= +// Follow-up round: F5 (histogram label multiplication) and F6 (label-set churn), +// reported 2026-08-12 22:40Z after 7.6h on the F1/F2/F3 fix. +// +// The first round bounded label VALUE SETS. This round bounds two things a +// sanitizer and a guard both miss: +// F5 — a label on a HISTOGRAM costs ~12 series per tuple, so a 20× label pair +// multiplies 12× harder there than on the counter beside it. +// F6 — a guard caps the LIVE registry; it cannot cap the number of distinct +// series Prometheus retains when a label's value set rotates over time. +// ============================================================================= + +// Test_RelayLatency_HistogramLabelsAreTopologyBounded is the F5 regression. +// +// path_relay_latency_seconds_bucket was 341,840 series fleet-wide, 31.6% of the +// entire gateway job and the largest single source of ongoing growth. Its labels +// were domain(13) × rpc_type(3) × service_id(61) × request_type(4) × +// status_code(5) × reputation_signal(4). The last two contribute nothing that is +// queried from the histogram and multiply it 20× — at ~12 series per tuple. +// +// This asserts through RecordRelay, not on the metric declaration: the histogram +// must not gain a series when only status_code or reputation_signal varies, while +// the counter beside it must. +func Test_RelayLatency_HistogramLabelsAreTopologyBounded(t *testing.T) { + RelaysTotal.Reset() + RelayLatency.Reset() + + const ( + domain = "relay-latency-labels.example" + rpcType = "json_rpc" + serviceID = "eth" + ) + + // Same topology tuple, every combination of the two dropped labels. + for _, statusCode := range []string{"2xx", "4xx", "5xx"} { + for _, signal := range []string{SignalOK, "minor_error", "major_error", "critical_error"} { + RecordRelay(domain, rpcType, serviceID, statusCode, signal, RelayTypeNormal, 0.1) + } + } + + require.Equal(t, 1, testutil.CollectAndCount(RelayLatency), + "histogram must hold ONE tuple: status_code and reputation_signal must not reach it") + require.Equal(t, 12, testutil.CollectAndCount(RelaysTotal), + "the counter must keep the full outcome taxonomy (3 status_code × 4 reputation_signal)") + + // The labels the histogram DOES keep must still separate series, or the fix + // would have flattened the metric into uselessness rather than narrowing it. + RecordRelay(domain, "websocket", serviceID, "2xx", SignalOK, RelayTypeNormal, 0.1) + RecordRelay(domain, rpcType, serviceID, "2xx", SignalOK, RelayTypeHedge, 0.1) + RecordRelay("other-operator.example", rpcType, serviceID, "2xx", SignalOK, RelayTypeNormal, 0.1) + RecordRelay(domain, rpcType, "poly", "2xx", SignalOK, RelayTypeNormal, 0.1) + require.Equal(t, 5, testutil.CollectAndCount(RelayLatency), + "domain, rpc_type, service_id and request_type must each still separate series") + + // Arity assertion: passing the counter's 6 labels to the histogram would panic. + require.Empty(t, collectLabelValues(t, RelayLatency, LabelStatusCode)) + require.Empty(t, collectLabelValues(t, RelayLatency, LabelReputationSignal)) +} + +// Test_SupplierLabelIsGone is the F6 regression, and the only test here that is a +// property of the whole package rather than of one metric. +// +// Six metrics carried a raw `supplier` label: 303,309 series in a 10-minute +// window. The supplier set is ~5,200 on chain and grows with the NETWORK, not +// with our traffic, and it rotates every session — so these metrics minted +// multiples of their live count in distinct series every day (measured on one +// pod over 7.7h: supplier_reputation_score 16.5×, qos_filter_rejection 19.4×, +// supplier_signal 9.3×, against a 1.0× control). +// +// A cardinality guard cannot fix that. Two of the six were guarded, honored their +// caps, and were still among the largest series sources in the job. +// +// ⭐ Each metric is populated THROUGH ITS PRODUCTION Record* HELPER first, then +// the registry is walked. That order is load-bearing and was got wrong once here: +// Gather() reports the labels of CHILD SERIES, so a vec with no children reports +// no labels at all. A registry walk on its own passes whatever the label set is — +// the revert check (restore `supplier` on both re-keyed metrics, expect a +// failure) came back green, which is the same class of mistake as asserting on a +// helper's return value instead of the caller's. +// +// The walk is kept as a second net over everything else the suite has populated, +// so a NEW metric that both carries a supplier label and gets exercised anywhere +// in this package fails too. +// +// The exemptions are the metrics where the supplier IS the subject — a specific +// account you have to name to act on it — rather than a way of naming an +// operator. PNF's ask draws exactly this line: aggregate to domain where the +// metric is an AGGREGATE SIGNAL. All three are also tiny in practice, which is +// the corroborating evidence rather than the reason. +func Test_SupplierLabelIsGone(t *testing.T) { + allowed := map[string]struct{}{ + // 313 series fleet-wide. The allowance it reports is per (supplier, + // session) — aggregating to domain would destroy the quantity. + MetricPrefix + "supplier_exhausted_total": {}, + // 0 series in production. Fires when a supplier ACCOUNT has never signed a + // transaction, so the address is the actionable payload; our dashboard + // queries it `by(supplier)` for precisely that reason. + MetricPrefix + "supplier_nil_pubkey_total": {}, + // 0 series in production. Same shape: names the account whose cached pubkey + // was invalidated or recovered. + MetricPrefix + "supplier_pubkey_cache_events_total": {}, + } + + // Populate every metric that used to carry `supplier`, via the production + // helper, so the registry walk below actually has children to inspect. A + // supplier address is passed wherever the helper still accepts one: if it ever + // reaches a label again, the walk sees it. + const leakedSupplier = "pokt1supplierlabelregression" + RecordSupplierBlacklist("supplier-label-gone.example", leakedSupplier, "eth", BlacklistReasonSignatureError) + RecordQoSFilterRejection("supplier-label-gone.example", "eth", QoSFilterReasonBlockHeightLag) + RecordHedgeSupplierOutcome("supplier-label-gone.example", HedgeRoleWinner, 0.1) + RecordHealthCheck("supplier-label-gone.example", leakedSupplier, "json_rpc", "eth", "block_height", SignalOK) + RecordRPCTypeFallback("supplier-label-gone.example", leakedSupplier, "eth", "COMET_BFT", "JSON_RPC") + RecordRelay("supplier-label-gone.example", "json_rpc", "eth", "2xx", SignalOK, RelayTypeNormal, 0.1) + + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + var offenders []string + populated := map[string]struct{}{} + for _, fam := range families { + name := fam.GetName() + if !strings.HasPrefix(name, MetricPrefix) { + continue + } + if len(fam.GetMetric()) > 0 { + populated[name] = struct{}{} + } + if _, ok := allowed[name]; ok { + continue + } + for _, m := range fam.GetMetric() { + for _, lp := range m.GetLabel() { + if lp.GetName() == LabelSupplier { + offenders = append(offenders, name) + } + } + } + } + + require.Empty(t, offenders, + "these metrics carry a raw `supplier` label; aggregate to `domain` or serve the "+ + "per-supplier question from /ready/?detailed=true") + + // Guard the guard: if a helper above stops emitting, the walk silently stops + // covering that metric and this test decays into asserting nothing. + for _, name := range []string{ + MetricPrefix + "supplier_blacklist_total", + MetricPrefix + "qos_filter_rejection_total", + MetricPrefix + "hedge_supplier_outcome_total", + MetricPrefix + "health_check_status_total", + MetricPrefix + "rpc_type_fallback_total", + MetricPrefix + "relays_total", + } { + require.Containsf(t, populated, name, + "%s was not populated, so the label walk did not actually inspect it", name) + } +} + +// Test_RemovedSupplierMetricsStayRemoved pins the two deletions. +// +// Both were guarded AND honored their guard AND were still enormous, which is the +// counter-intuitive part worth a test: someone reading only the guard code would +// reasonably conclude they were safe to re-add. +// +// Detected by REGISTRATION COLLISION, not by walking Gather(). Gather() reports +// only families that have at least one child series, so a re-added vec that no +// test happens to populate would be invisible to a registry walk — the metric +// would be back, minting series in production, with this test green. Registering +// a same-named collector answers "is this name taken" regardless of children. +func Test_RemovedSupplierMetricsStayRemoved(t *testing.T) { + removed := []string{ + MetricPrefix + "supplier_reputation_score", // 4,510 live vs 74,639 distinct/7.7h/pod + MetricPrefix + "supplier_signal_total", // 6,523 live vs 60,674 distinct/7.7h/pod + } + + for _, name := range removed { + probe := prometheus.NewCounter(prometheus.CounterOpts{ + Name: name, + Help: "registration probe", + }) + err := prometheus.DefaultRegisterer.Register(probe) + + var already prometheus.AlreadyRegisteredError + require.Falsef(t, errors.As(err, &already), + "%s is registered again; it was removed for unbounded label-set churn (a guard "+ + "cannot bound it — see DefaultSeriesLimit). Per-operator reading is "+ + "path_reputation_mean_score; per-supplier is /ready/?detailed=true", name) + require.NoErrorf(t, err, "unexpected error probing %s", name) + + // Leave the registry as found, or the probe itself becomes a phantom metric + // for every later test in this package. + prometheus.DefaultRegisterer.Unregister(probe) + } +} diff --git a/metrics/domain_sanitizer.go b/metrics/domain_sanitizer.go index b35601560..da58c2e98 100644 --- a/metrics/domain_sanitizer.go +++ b/metrics/domain_sanitizer.go @@ -3,6 +3,8 @@ package metrics import ( "strings" "unicode/utf8" + + shannonmetrics "github.com/pokt-network/path/metrics/protocol/shannon" ) const ( @@ -21,6 +23,37 @@ const ( DomainLabelMaxLen = 64 ) +// DomainFromEndpointAddr derives an operator `domain` from a PATH endpoint +// address of the form "-" (protocol.EndpointAddr). Returns "" when +// no domain can be derived, so callers keep their skip-on-empty behavior instead +// of emitting a DomainUnknown series — SanitizeDomainLabel("") returns +// DomainUnknown, so an unconditional sanitize would turn "no endpoint context" +// into a real timeseries. +// +// Exists so metrics call sites that hold an EndpointAddr can key on the operator +// rather than the supplier address without each one re-deriving it. Cheap enough +// for hot paths: an IndexByte plus a slice for the split, then a sync.Map hit in +// shannonmetrics.ExtractDomainOrHost, which memoizes the url.Parse + +// publicsuffix lookup over the bounded set of supplier URLs. Called at ~9,500/s +// fleet-wide from the QoS filter path. +// +// The returned value is NOT yet sanitized — the Record* helper sanitizes after +// its empty check, in that order, for the reason above. +func DomainFromEndpointAddr(endpointAddr string) string { + // EndpointAddr is "-"; everything after the first dash is the + // URL. Matches protocol.EndpointAddr.GetURL without importing it (the metrics + // package must not depend on protocol). + i := strings.IndexByte(endpointAddr, '-') + if i < 0 { + return "" + } + domain, err := shannonmetrics.ExtractDomainOrHost(endpointAddr[i+1:]) + if err != nil { + return "" + } + return domain +} + // SanitizeDomainLabel bounds the cardinality of the `domain` Prometheus label. // MUST be called on every value flowing into a `domain` label. // diff --git a/metrics/leaderboard.go b/metrics/leaderboard.go index 67ff7e87e..20cb56006 100644 --- a/metrics/leaderboard.go +++ b/metrics/leaderboard.go @@ -32,16 +32,6 @@ type MeanScoreEntry struct { MeanScore float64 // Average score across all endpoints for this combination } -// SupplierScoreEntry represents the per-(supplier, service_id, rpc_type) reputation score. -// One value per triple — a supplier serving multiple RPC types (e.g. json_rpc + websocket) -// gets one entry per type, since reputation is tracked and acted on per rpc_type. -type SupplierScoreEntry struct { - Supplier string - ServiceID string - RPCType string - Score float64 -} - // CooldownCountEntry represents per-domain count of endpoints currently in strike // cooldown for a given service / rpc_type. type CooldownCountEntry struct { @@ -55,11 +45,14 @@ type CooldownCountEntry struct { type LeaderboardDataProvider interface { // GetEndpointLeaderboardData returns all endpoint entries grouped by the required dimensions GetEndpointLeaderboardData(ctx context.Context) ([]EndpointLeaderboardEntry, error) - // GetMeanScoreData returns mean reputation scores per domain/service/rpc_type + // GetMeanScoreData returns mean reputation scores per domain/service/rpc_type. + // + // Per-OPERATOR, deliberately. A GetSupplierScoreData sibling existed until + // 2026-08-12 and was removed with path_supplier_reputation_score: keyed on the + // supplier address, it minted ~232K distinct Prometheus series per pod per day + // while nothing queried it. Per-supplier scores are served on demand by + // GET /ready/?detailed=true instead. GetMeanScoreData(ctx context.Context) ([]MeanScoreEntry, error) - // GetSupplierScoreData returns per-(supplier, service_id) reputation scores. - // Optional: implementations may return nil if per-supplier scoring is not supported. - GetSupplierScoreData(ctx context.Context) ([]SupplierScoreEntry, error) // GetCooldownCountData returns per-(domain, service_id, rpc_type) counts of // endpoints currently in strike cooldown. Optional: implementations may return // nil if cooldown tracking is not supported. @@ -188,23 +181,14 @@ func (lp *LeaderboardPublisher) publishLeaderboard(ctx context.Context) { } } - // Publish per-(supplier, service_id) reputation scores. Reset between - // snapshots — suppliers may rotate out of sessions and stale series - // would persist forever otherwise. - supplierScores, err := lp.provider.GetSupplierScoreData(ctx) - if err != nil { - lp.logger.Warn().Err(err).Msg("Failed to get supplier score data") - return - } - - SupplierReputationScore.Reset() - - if len(supplierScores) > 0 { - for _, entry := range supplierScores { - SetSupplierReputationScore(entry.Supplier, entry.ServiceID, entry.RPCType, entry.Score) - } - lp.logger.Debug().Int("entries", len(supplierScores)).Msg("Published supplier scores") - } + // A per-(supplier, service_id, rpc_type) score gauge was published here until + // 2026-08-12. Its Reset() — needed so a supplier rotating out of a session did + // not stick at its last score via Prometheus' 5-minute staleness window — is + // exactly what made it the worst churn source in the gateway: the live set was + // only ever the current sessions' suppliers while the cumulative set grew + // toward the whole chain (4,510 live vs 74,639 distinct over 7.7h on one pod). + // Removed; use path_reputation_mean_score above for the per-operator reading + // and /ready/?detailed=true for a per-supplier one. // Publish per-domain endpoint cooldown counts. Reset between snapshots so a // domain that drops to zero cooldown'd endpoints actually shows zero (instead diff --git a/metrics/metrics.go b/metrics/metrics.go index 1837b37dd..9f3f5984a 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -36,6 +36,7 @@ const ( LabelSupplier = "supplier" LabelSignalType = "signal_type" LabelSeverity = "severity" + LabelOutcome = "outcome" // --- Latency signal values @@ -466,6 +467,58 @@ func RecordCircuitBreakerEvent(serviceID, domain, reasonCategory, event string) DomainCircuitBreakerEventsTotal.WithLabelValues(serviceID, domain, reasonCategory, event).Inc() } +// CircuitBreakerOutcomeTotal exposes the failure-rate gate's OWN inputs, per host. +// +// WHY THIS EXISTS. The gate keys on the full hostname, while path_relays_total keys on +// eTLD+1. An operator running several relay miners under one domain therefore reports one +// blended success rate, so a domain whose hosts range from 50% to 80% is indistinguishable +// from one where every host sits at 65%. Those call for opposite responses — replace the bad +// hosts, versus the operator has a systemic problem — and telling them apart cost four wrong +// hypotheses in one production investigation purely because the number did not exist at the +// granularity the decision is made at. +// +// CARDINALITY. Deliberately two labels, not three. Its sibling +// circuit_breaker_events_total carries the same service_id x domain pair PLUS +// reason_category x event and reached 233,269 series — the cross product, not a leaking +// label. Dropping to a 2-value outcome keeps this at roughly a twelfth of that, and the +// guard below bounds the live registry the same way. Note the guard bounds the REGISTRY, not +// the series Prometheus retains: if the churn diagnostic ever shows distinct-over-8h running +// well above the instant count, the label set is rotating and this metric costs multiples of +// what it appears to. +// +// Counts what the GATE sees, which is deliberately not every relay: while a domain is broken +// MarkBroken short-circuits before the gate, so failures are not counted and the host goes +// ABSENT rather than reading as healthy. Read it as "the evidence the breaker acted on", not +// as a traffic meter — a host at a suspiciously good rate here may simply have been broken +// for most of the window. +var CircuitBreakerOutcomeTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricPrefix + "circuit_breaker_outcome_total", + Help: "Relay outcomes as counted by the domain circuit-breaker's failure-rate gate, keyed on the full HOSTNAME the gate uses (path_relays_total keys on eTLD+1). outcome ∈ {success, failure}. Numerator and denominator of the rate that decides a break. Only counts what the gate sees: a broken domain is absent, not healthy.", + }, + []string{LabelServiceID, LabelDomain, LabelOutcome}, +) + +// Outcome values for CircuitBreakerOutcomeTotal. +const ( + CircuitBreakerOutcomeSuccess = "success" + CircuitBreakerOutcomeFailure = "failure" +) + +// RecordCircuitBreakerOutcome records one outcome against the gate's window. Skipped silently +// when domain is empty, matching RecordCircuitBreakerEvent — and sanitized after that check, +// since SanitizeDomainLabel maps "" to DomainUnknown and would defeat the skip. +func RecordCircuitBreakerOutcome(serviceID, domain, outcome string) { + if domain == "" { + return + } + domain = SanitizeDomainLabel(domain) + if !circuitBreakerOutcomeGuard.allow(serviceID, domain, outcome) { + return + } + CircuitBreakerOutcomeTotal.WithLabelValues(serviceID, domain, outcome).Inc() +} + // ============================================================================= // Endpoints In Cooldown (Gauge, published every 10s via leaderboard publisher) // Labels: domain, rpc_type, service_id @@ -534,17 +587,24 @@ var BlockedDomainsConfigured = promauto.NewGaugeVec( // ============================================================================= // Supplier Blacklist Events (Counter) -// Labels: domain, supplier, service_id, reason +// Labels: domain, service_id, reason // Value: count // Purpose: Track suppliers blacklisted for validation/signature errors +// +// No `supplier` label: the supplier set is ~5,200 on chain and grows with the +// network, not with our traffic, so a raw supplier label makes this metric scale +// with chain growth (9,410 series fleet-wide, measured 2026-08-12, for a metric +// whose only dashboard consumer aggregates by service_id and reason). A +// blacklisting is acted on per operator anyway, and the specific address is in +// the WARN log at the call site. // ============================================================================= var SupplierBlacklistTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Name: MetricPrefix + "supplier_blacklist_total", - Help: "Suppliers blacklisted by domain, supplier address, service_id, and reason.", + Help: "Suppliers blacklisted by domain, service_id, and reason. No supplier label: it scaled with the on-chain supplier set rather than with traffic; the blacklisted address is in the log line.", }, - []string{LabelDomain, LabelSupplier, LabelServiceID, "reason"}, + []string{LabelDomain, LabelServiceID, "reason"}, ) // Blacklist reason constants @@ -595,7 +655,9 @@ const ( // ============================================================================= // RPC Type Fallback (Counter) -// Labels: domain, supplier, service_id, requested_rpc_type, fallback_rpc_type +// Labels: domain, service_id, requested_rpc_type, fallback_rpc_type +// (`supplier` was dropped in the 2026-08-12 F3 fix: 3,289 values against the +// metric's own 9 domains × 12 service_ids, ~all of its 201,068 series.) // Purpose: Track when suppliers don't support the requested RPC type and fallback is used // ============================================================================= @@ -653,11 +715,21 @@ func RecordSupplierExhausted(supplier, serviceID string) { // ============================================================================= // QoS Filter Rejections (Counter) -// Labels: supplier, service_id, reason -// Purpose: Per-supplier visibility into why QoS dropped an endpoint pre-relay. -// These rejections are silent today — operators can't tell whether their +// Labels: domain, service_id, reason +// Purpose: Per-operator visibility into why QoS dropped an endpoint pre-relay. +// These rejections are otherwise silent — operators can't tell whether their // endpoint was filtered for being block-behind, missing chain_id, lacking // archival capability, etc. +// +// Keyed on domain (eTLD+1), not supplier. This metric fires ~9,500/s fleet-wide, +// and with a raw supplier label it had the WORST churn of any gateway metric: +// 1,271 series live in a 10-minute window against 24,708 distinct series minted +// over 7.7h on a single pod (19.4×), because rejections are sporadic per +// supplier while the supplier set rotates every session. A cardinality guard +// cannot bound that — it caps the live registry, and re-admitting an evicted +// tuple produces the same series, so the number of distinct series Prometheus +// must store is identical with or without eviction. Only fewer label VALUES +// bound it, and the routing decision this metric informs is made per operator. // ============================================================================= const ( @@ -668,46 +740,78 @@ const ( QoSFilterReasonInvalidResponse = "invalid_response" QoSFilterReasonEmptyResponse = "empty_response" QoSFilterReasonCapabilityLimited = "capability_limitation" + // QoSFilterReasonHealthUnknown: the endpoint has no health-probe observation at all. + // Distinct from Unhealthy — "never asked" and "answered badly" call for opposite + // responses, and a shared bucket makes them indistinguishable without inferring from the + // absence of a sibling series. + QoSFilterReasonHealthUnknown = "health_unknown" + // QoSFilterReasonUnhealthy: the endpoint answered a health probe with a not-OK result. + QoSFilterReasonUnhealthy = "unhealthy" + // QoSFilterReasonEpochLag: the endpoint is more than the allowed number of epochs behind + // the perceived chain epoch. Solana-specific; separate from block_height_lag because the + // two have different time constants (~2.5 days versus ~400ms) and conflating them would + // make an epoch-rollover blip look like ordinary sync lag. + QoSFilterReasonEpochLag = "epoch_lag" ) var QoSFilterRejectionTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Name: MetricPrefix + "qos_filter_rejection_total", - Help: "QoS filter rejections by supplier, service_id, and reason. Reasons: block_height_lag, block_height_unknown, chain_id_mismatch, archival_required, invalid_response, empty_response, capability_limitation.", + Help: "QoS filter rejections by domain (eTLD+1), service_id, and reason. Reasons: block_height_lag, block_height_unknown, chain_id_mismatch, archival_required, invalid_response, empty_response, capability_limitation, health_unknown, unhealthy, epoch_lag.", }, - []string{LabelSupplier, LabelServiceID, "reason"}, + []string{LabelDomain, LabelServiceID, "reason"}, ) -// RecordQoSFilterRejection counts a per-supplier QoS filter rejection. -// Skipped when supplier is empty (e.g., missing endpoint context) or when -// the cardinality guard has tripped for this metric. -func RecordQoSFilterRejection(supplier, serviceID, reason string) { - if supplier == "" { +// RecordQoSFilterRejection counts a per-operator QoS filter rejection. +// Skipped when domain is empty (e.g., missing endpoint context) or when the +// cardinality guard has tripped for this metric. +// +// The empty check runs BEFORE SanitizeDomainLabel deliberately: +// SanitizeDomainLabel("") returns DomainUnknown, which would turn "no endpoint +// context" into a real series and silently defeat the skip. +func RecordQoSFilterRejection(domain, serviceID, reason string) { + if domain == "" { return } - if !qosFilterRejectionGuard.allow(supplier, serviceID, reason) { + domain = SanitizeDomainLabel(domain) + if !qosFilterRejectionGuard.allow(domain, serviceID, reason) { return } - QoSFilterRejectionTotal.WithLabelValues(supplier, serviceID, reason).Inc() + QoSFilterRejectionTotal.WithLabelValues(domain, serviceID, reason).Inc() } // ============================================================================= -// Per-supplier reputation observability (Gauge + Counter) -// Labels: supplier, service_id (+ signal_type on counter) -// Purpose: Give operators of relay miners a metric-level view of why PATH is -// or isn't routing to them. Supplier already implies domain — no domain label. -// Signals are emitted from reputation/service.go::RecordSignal, so any QoS -// code that produces a Signal automatically feeds this counter. +// REMOVED: path_supplier_reputation_score (gauge, labels supplier/service_id/rpc_type) +// +// Removed 2026-08-12. It was the single worst churn source in the gateway: 4,510 +// series live in a 10-minute window against 74,639 distinct series minted over +// 7.7h on ONE pod (16.5×), ~232K series/pod/day, every one retained for the full +// 6-day Prometheus window. The churn was structural — the publisher Reset()s the +// whole gauge every snapshot cycle (correctly: a supplier that rotates out of a +// session must not stick at its last score via the 5-minute staleness window), +// so the live set is only ever the current sessions' suppliers while the +// cumulative set grows toward the whole chain. +// +// No cardinality guard could have bounded it. A guard caps the LIVE registry, +// and re-admitting an evicted tuple recreates the same label set — so the count +// of distinct series Prometheus must store is identical with or without +// eviction. Only fewer label values bound that, and `supplier` scales with chain +// growth rather than with our traffic. +// +// Nothing consumed it: zero references across our dashboards, zero Prometheus +// rules. Both readings it supported already exist and are cheaper: +// - per operator: path_reputation_mean_score{domain, service_id, rpc_type}, +// 403 series/pod, 1.0× churn (also Reset() per cycle, but bounded by the +// operator set instead of the supplier set). +// - per supplier, live and exact: GET /ready/?detailed=true, which +// returns score, strikes, latency, tier and cooldown per endpoint. That is +// the right shape for a per-supplier question — a point lookup, not 74K +// retained timeseries. +// +// Re-introducing a per-supplier metric means re-introducing that churn. Serve +// the question from the API instead. // ============================================================================= -var SupplierReputationScore = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Name: MetricPrefix + "supplier_reputation_score", - Help: "Per-supplier reputation score (0-100) by supplier, service_id, and rpc_type. Snapshotted every 10s. The rpc_type split keeps a supplier's websocket score from colliding with its json_rpc score (they are tracked and acted on separately).", - }, - []string{LabelSupplier, LabelServiceID, LabelRPCType}, -) - // Reasons an endpoint is dropped by the reputation filter, used as the `reason` // label on ReputationDisqualifiedTotal. const ( @@ -1037,6 +1141,31 @@ func RecordReputationRateCooldown(serviceID string) { ReputationRateCooldownTotal.WithLabelValues(serviceID).Inc() } +// ReputationInvalidRateCooldownTotal counts endpoints cooled down by the sustained +// protocol-violation-rate detector, by service_id. +// +// Distinct from ReputationRateCooldownTotal because the two answer different questions and +// share no threshold: that one fires at a 30% critical rate ("this endpoint is broken"), this +// one at 0.5% structurally-invalid responses ("this endpoint returns things that are never +// valid, at a low but sustained rate"). Folding them together would hide the second inside +// the first, which is the entire failure this detector exists to correct. +// +// Expected to be zero or near-zero fleet-wide: measured 2026-08-19, only two domains produced +// protocol violations above a 0.00003% noise floor. A broad nonzero rate means the threshold +// is wrong, not that the fleet degraded. +var ReputationInvalidRateCooldownTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricPrefix + "reputation_invalid_rate_cooldown_total", + Help: "Endpoints cooled down by the sustained protocol-violation-rate detector, by service_id. Expected near-zero; a broad nonzero rate means the threshold is mistuned.", + }, + []string{LabelServiceID}, +) + +// RecordReputationInvalidRateCooldown increments the invalid-rate cooldown counter. +func RecordReputationInvalidRateCooldown(serviceID string) { + ReputationInvalidRateCooldownTotal.WithLabelValues(serviceID).Inc() +} + // ReputationPoolCollapseGuardTotal counts how often reputation filtering would have removed // EVERY endpoint for a service (all in cooldown or below threshold) and the pool-collapse // guard instead kept the least-bad tier. A nonzero rate is the signal that a service is @@ -1056,48 +1185,43 @@ func RecordReputationPoolCollapseGuard(serviceID, rpcType string) { ReputationPoolCollapseGuardTotal.WithLabelValues(serviceID, rpcType).Inc() } -// Severity classes for supplier_signal_total. The reputation layer emits 8 -// distinct signal-type strings; carrying all 8 as a label multiplies this -// counter's cardinality 8× on top of the (supplier × service_id) base — the -// base already accumulates toward the full network supplier set as sessions -// rotate, so the extra 8× is what pushes the metric into the 25K guard within -// ~15 min of pod start. Collapsing to 3 severity classes cuts the fan to 3× -// while preserving the only distinction this per-supplier view needs: is the -// supplier working, degraded-but-serving, or failing. Full error taxonomy -// (5xx vs timeout vs config) remains available per-domain on relays_total -// (status_code + reputation_signal). -const ( - SupplierSeverityOK = "ok" // success, recovery_success - SupplierSeveritySlow = "slow" // slow_response, very_slow_response - SupplierSeverityError = "error" // minor/major/critical/fatal error -) - -var SupplierSignalTotal = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: MetricPrefix + "supplier_signal_total", - Help: "Reputation signals emitted by supplier and service_id, collapsed to a severity class (ok/slow/error). Full error taxonomy is available per-domain on relays_total.", - }, - []string{LabelSupplier, LabelServiceID, LabelSeverity}, -) - -// supplierSignalSeverity collapses a reputation signal-type string (the 8 -// reputation.SignalType wire values) into one of three severity classes. -// Unknown/new signal types fall through to "error" so a mis-added type shows -// up loudly rather than silently vanishing, and cardinality stays bounded. -func supplierSignalSeverity(signalType string) string { - switch signalType { - case "success", "recovery_success": - return SupplierSeverityOK - case "slow_response", "very_slow_response": - return SupplierSeveritySlow - default: // minor_error, major_error, critical_error, fatal_error, unknown - return SupplierSeverityError - } -} +// ============================================================================= +// REMOVED: path_supplier_signal_total (counter, labels supplier/service_id/severity) +// +// Removed 2026-08-12. Its cardinality had already been cut once — the 8 +// reputation signal types were collapsed to 3 severity classes to stop it +// tripping the 25K guard within ~15 min of pod start. That fixed the multiplier +// and left the base, which was the actual problem: the (supplier × service_id) +// base accumulates toward the whole network's supplier set as sessions rotate. +// +// Measured on one mainnet pod: 6,523 series live in a 10-minute window against +// 60,674 distinct series minted over 7.7h (9.3×), 79,617 series fleet-wide. It +// was one of the two largest sources of series in the entire gateway job while +// simultaneously sitting at ~26% of its own guard cap — the guard bounds the live +// registry, not the number of distinct series Prometheus retains, and eviction +// does not change that count at all (a re-admitted tuple is the same label set). +// +// Nothing consumed it: zero references across our dashboards, zero Prometheus +// rules. Everything it reported is already available, domain-keyed and 1.0× +// churn, on metrics we keep: +// - path_relays_total{domain, rpc_type, service_id, status_code, +// reputation_signal, request_type} — the FULL 8-value signal taxonomy, not +// the 3-class collapse, and it covers health-check and probation traffic via +// request_type. +// - path_health_check_status_total{domain, ..., reputation_signal} for the +// health-check path specifically. +// - /ready/?detailed=true for a live per-supplier point lookup. +// +// The severity collapse (ok/slow/error) went with it; nothing else used those +// constants. If a per-supplier counter is ever needed again, note that no guard +// setting makes it cheap — the cost is the label's value set, and that one grows +// with the chain. +// ============================================================================= // ============================================================================= // Relays (Counter + Histogram) -// Labels: domain, rpc_type, service_id, status_code, reputation_signal, request_type +// Labels (counter): domain, rpc_type, service_id, status_code, reputation_signal, request_type +// Labels (histogram): domain, rpc_type, service_id, request_type // Purpose: Track ALL outgoing relays from PATH to supplier endpoints // Includes: normal user requests, health checks, probation traffic // ============================================================================= @@ -1136,13 +1260,41 @@ var RelaysTotal = promauto.NewCounterVec( []string{LabelDomain, LabelRPCType, LabelServiceID, LabelStatusCode, LabelReputationSignal, "request_type"}, ) +// RelayLatency deliberately carries FEWER labels than RelaysTotal: it omits +// status_code and reputation_signal. +// +// A histogram costs ~12 series per label tuple (10 `le` buckets plus _sum and +// _count), so every label on it multiplies 12× what the same label costs on the +// counter beside it. status_code (5 values) × reputation_signal (4) is a 20× +// multiplier on the most expensive metric in the gateway: measured 2026-08-12, +// path_relay_latency_seconds_bucket was 341,840 series fleet-wide, 31.6% of the +// entire gateway job, and was the single largest source of ongoing series +// growth. +// +// The remaining labels are bounded by service topology rather than by request +// outcome. Measured on one mainnet pod: the live (domain, service_id, rpc_type) +// universe is 403 combinations, so this histogram's tuple ceiling is 403 × +// len(request_type) ≈ 1,600 → ~19K series. With status_code and +// reputation_signal it was 403 × 4 × 20 = 32,240 tuples → ~322K series/pod, and +// it was still climbing toward that at 7.6h of pod age (2,226 → 2,732 tuples +// over 6h). +// +// Nothing consumed the dropped labels HERE. Every dashboard query against this +// histogram aggregates to at most (domain, service_id, rpc_type, le) and uses +// request_type only as a selector; no Prometheus rule references it at all. The +// full outcome taxonomy remains on RelaysTotal, which carries all six labels at +// 1 series per tuple — join on (domain, rpc_type, service_id, request_type) to +// correlate a latency shift with the status codes behind it. +// +// If a per-status latency split is ever genuinely needed, add a SEPARATE +// narrow-labelled histogram rather than restoring these labels here. var RelayLatency = promauto.NewHistogramVec( prometheus.HistogramOpts{ Name: MetricPrefix + "relay_latency_seconds", - Help: "Outgoing relay latency in seconds by domain, rpc_type, service_id, status_code, reputation_signal, and request_type.", + Help: "Outgoing relay latency in seconds by domain, rpc_type, service_id, and request_type. No status_code/reputation_signal labels: on a histogram each label costs ~12 series per tuple, and that pair multiplied this metric 20× to 31.6% of all gateway series. The outcome taxonomy is on relays_total (same labels plus status_code and reputation_signal, 1 series per tuple).", Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, }, - []string{LabelDomain, LabelRPCType, LabelServiceID, LabelStatusCode, LabelReputationSignal, "request_type"}, + []string{LabelDomain, LabelRPCType, LabelServiceID, "request_type"}, ) // ============================================================================= @@ -1459,21 +1611,27 @@ var HedgeWinningLatency = promauto.NewHistogramVec( ) // ============================================================================= -// Hedge per-supplier outcome (Counter) + role latency (Histogram) +// Hedge per-operator outcome (Counter) + role latency (Histogram) // Purpose: Answers "am I losing races, and by how much?". // -// Split into two metrics to bound cardinality. The per-supplier signal is a +// Split into two metrics to bound cardinality. The per-operator signal is a // win-rate — a ratio of counts — which needs no buckets, so it lives on a plain -// COUNTER (supplier × role = 2 series/supplier, ~10K series network-wide, -// well under the 25K guard → complete, not first-seen-biased). The "by how -// much" is a latency distribution that does NOT need per-supplier resolution, -// so it lives on a HISTOGRAM keyed by role only (~2×12 series total). +// COUNTER (domain × role = 2 series/operator, ~26 series/pod). The "by how much" +// is a latency distribution that does NOT need per-operator resolution, so it +// lives on a HISTOGRAM keyed by role only (~2×12 series total). // // This replaces the earlier single per-supplier HistogramVec, which multiplied // ~8K supplier×role tuples by ~12 bucket series each = the largest single // metric family in the gateway (~585K series, ~39% of all gateway cardinality; // with service_id it was 945K — audit 2026-04-28). Win-rate = winner / // (winner + loser) on the counter; loser-vs-winner latency gap on the histogram. +// +// Keyed on domain rather than supplier since 2026-08-12: even as a bare counter, +// supplier × role was 49,231 series fleet-wide and 1.8× churn (4,306 live vs +// 7,840 distinct over 7.7h on one pod), because the supplier set rotates every +// session and scales with the chain rather than with our traffic. Hedge is an +// operator-level question — "did routing to a different operator help" — so the +// eTLD+1 is the granularity that answers it. // ============================================================================= const ( @@ -1481,23 +1639,24 @@ const ( HedgeRoleLoser = "loser" ) -// HedgeSupplierOutcomeTotal is the per-supplier win/loss counter. Bounded and -// complete: supplier × {winner,loser} stays well under the guard cap. +// HedgeSupplierOutcomeTotal is the per-operator win/loss counter. Bounded by the +// live operator set (13 domains measured 2026-08-12), not by the chain's +// supplier set. var HedgeSupplierOutcomeTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Name: MetricPrefix + "hedge_supplier_outcome_total", - Help: "Per-supplier hedge race outcomes. role=winner|loser. Win-rate = winner/(winner+loser).", + Help: "Per-operator (eTLD+1) hedge race outcomes. role=winner|loser. Win-rate = winner/(winner+loser).", }, - []string{LabelSupplier, "role"}, + []string{LabelDomain, "role"}, ) // HedgeRoleLatency is the hedge outcome latency distribution by role, aggregated -// across all suppliers (no supplier label → tiny, fixed cardinality). Pairs -// with HedgeSupplierOutcomeTotal for the per-supplier win-rate. +// across all operators (no domain label → tiny, fixed cardinality). Pairs with +// HedgeSupplierOutcomeTotal for the per-operator win-rate. var HedgeRoleLatency = promauto.NewHistogramVec( prometheus.HistogramOpts{ Name: MetricPrefix + "hedge_role_latency_seconds", - Help: "Hedge race outcome latency by role (winner|loser), aggregated across suppliers. Pairs with hedge_supplier_outcome_total for per-supplier win-rate.", + Help: "Hedge race outcome latency by role (winner|loser), aggregated across operators. Pairs with hedge_supplier_outcome_total for per-operator win-rate.", Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, }, []string{"role"}, @@ -1519,21 +1678,25 @@ func RecordHedgeLatencySavings(rpcType, serviceID string, savingsSeconds float64 // RecordHedgeSupplierOutcome records a hedge race outcome. role must be // HedgeRoleWinner or HedgeRoleLoser. The latency distribution is recorded per -// role (no supplier label, always). The per-supplier win/loss count is recorded -// only when supplier is known and the cardinality guard admits it. -func RecordHedgeSupplierOutcome(supplier, role string, latencySeconds float64) { +// role (no domain label, always). The per-operator win/loss count is recorded +// only when domain is known and the cardinality guard admits it. +// +// Empty check before SanitizeDomainLabel: the sanitizer maps "" to +// DomainUnknown, which would turn "no endpoint context" into a real series. +func RecordHedgeSupplierOutcome(domain, role string, latencySeconds float64) { // Role-only latency distribution: fixed, tiny cardinality — always recorded. HedgeRoleLatency.WithLabelValues(role).Observe(latencySeconds) - // Per-supplier win/loss counter: 2 series/supplier, still guarded as a - // backstop against a supplier-address label leak. - if supplier == "" { + // Per-operator win/loss counter: 2 series/operator, still guarded as a + // backstop against a domain label leak. + if domain == "" { return } - if !hedgeSupplierGuard.allow(supplier, role) { + domain = SanitizeDomainLabel(domain) + if !hedgeSupplierGuard.allow(domain, role) { return } - HedgeSupplierOutcomeTotal.WithLabelValues(supplier, role).Inc() + HedgeSupplierOutcomeTotal.WithLabelValues(domain, role).Inc() } // RecordBatchSize records a batch request with latency. @@ -1598,9 +1761,15 @@ func RecordProbationEvent(domain, rpcType, serviceID, event string) { // RecordSupplierBlacklist records a supplier being blacklisted with the specific reason // reason should be one of the BlacklistReason* constants -func RecordSupplierBlacklist(domain, supplier, serviceID, reason string) { +// +// supplier is accepted and ignored: it is no longer a label (see +// SupplierBlacklistTotal). The parameter is kept so call sites keep passing the +// address they already have, making a future re-introduction a one-line change +// rather than a hunt through callers — same pattern as RecordHealthCheck and +// RecordRPCTypeFallback. +func RecordSupplierBlacklist(domain, _ /* supplier */, serviceID, reason string) { domain = SanitizeDomainLabel(domain) - SupplierBlacklistTotal.WithLabelValues(domain, supplier, serviceID, reason).Inc() + SupplierBlacklistTotal.WithLabelValues(domain, serviceID, reason).Inc() } // RecordSupplierNilPubkey records when a supplier is found with a nil public key. @@ -1644,23 +1813,9 @@ func SetMeanScore(domain, serviceID, rpcType string, score float64) { ReputationMeanScore.WithLabelValues(domain, serviceID, rpcType).Set(score) } -// SetSupplierReputationScore sets the per-(supplier, service_id, rpc_type) reputation gauge. -// Skipped silently when supplier is empty (e.g., per-domain reputation key) -// or when the cardinality guard has tripped for this metric. -// -// The guard is keyed on all three labels, matching the gauge exactly. It used to -// key on (supplier, service_id) only, which let one admitted slot create one -// series per rpc_type — so the guard's tuple count under-reported the series it -// was capping, and eviction could not delete the series it reclaimed. -func SetSupplierReputationScore(supplier, serviceID, rpcType string, score float64) { - if supplier == "" { - return - } - if !supplierReputationGuard.allow(supplier, serviceID, rpcType) { - return - } - SupplierReputationScore.WithLabelValues(supplier, serviceID, rpcType).Set(score) -} +// SetSupplierReputationScore was removed with path_supplier_reputation_score. +// See the REMOVED block above SetMeanScore's metric for why, and use +// /ready/?detailed=true for per-supplier scores. // RecordReputationDisqualified increments the reputation-filter disqualification // counter for one dropped endpoint. reason is one of the @@ -1676,29 +1831,22 @@ func RecordHedgeSelfOperatorAvoided(serviceID string) { HedgeSelfOperatorAvoidedTotal.WithLabelValues(serviceID).Inc() } -// RecordSupplierSignal increments the per-supplier signal counter, collapsing -// the reputation signal type to a severity class (ok/slow/error) to bound -// cardinality. Skipped silently when supplier is empty (e.g., per-domain -// reputation key) or when the cardinality guard has tripped for this metric. -func RecordSupplierSignal(supplier, serviceID, signalType string) { - if supplier == "" { - return - } - severity := supplierSignalSeverity(signalType) - if !supplierSignalGuard.allow(supplier, serviceID, severity) { - return - } - SupplierSignalTotal.WithLabelValues(supplier, serviceID, severity).Inc() -} +// RecordSupplierSignal was removed with path_supplier_signal_total. The +// domain-keyed replacement is path_relays_total's reputation_signal label, which +// carries the full taxonomy rather than the 3-class collapse. See the REMOVED +// block where the metric was declared. // RecordRelay records an outgoing relay to a supplier endpoint with latency // relayType should be one of: RelayTypeNormal, RelayTypeHealthCheck, RelayTypeProbation // statusCode should be the HTTP status code category (2xx, 4xx, 5xx, etc.) // reputationSignal should be the signal recorded (ok, minor_error, major_error, etc.) +// +// statusCode and reputationSignal land on the counter only — see RelayLatency for +// why the histogram carries a narrower label set. func RecordRelay(domain, rpcType, serviceID, statusCode, reputationSignal, relayType string, latencySeconds float64) { domain = SanitizeDomainLabel(domain) RelaysTotal.WithLabelValues(domain, rpcType, serviceID, statusCode, reputationSignal, relayType).Inc() - RelayLatency.WithLabelValues(domain, rpcType, serviceID, statusCode, reputationSignal, relayType).Observe(latencySeconds) + RelayLatency.WithLabelValues(domain, rpcType, serviceID, relayType).Observe(latencySeconds) } // WebSocket Connection Metrics Helpers @@ -1835,6 +1983,17 @@ func GetLatencySignalWithThresholds(latencyMs float64, thresholds *LatencyThresh } // GetStatusCodeCategory returns the status code as a string, grouping 4xx and 5xx +// Status categories shared by path_requests_total and path_relays_total. +// +// StatusCategoryError covers relays that never produced an HTTP status at all — timeouts, +// refused/reset connections, unreachable hosts, signature and payload validation failures. +// It is deliberately NOT folded into 5xx: a backend that answers 500 is reachable and +// answering, and one that never answers is not. The two call for different responses. +const ( + StatusCategorySuccess = "200" + StatusCategoryError = "error" +) + func GetStatusCodeCategory(statusCode int) string { switch { case statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices: @@ -1847,3 +2006,31 @@ func GetStatusCodeCategory(statusCode int) string { return "other" } } + +// RequestSampleUniqueness and RequestSampleTop1Share describe the LAST COMPLETED request +// sampling window per service (see gateway.RequestSampler). Uniqueness = distinct +// fingerprints / sampled requests: 1.0 means every sampled request differed, near 0 means +// the same few requests repeated. Top1Share = the single most repeated fingerprint's share. +// Both are per service_id only — no method, no fingerprint — so cardinality is the service +// list. Sampled (1-in-N), so read as a ratio, never as a count. +var RequestSampleUniqueness = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: MetricPrefix + "request_sample_uniqueness", + Help: "distinct request fingerprints / sampled requests over the last completed sampling window, per service. 1.0 = all different; near 0 = the same few requests repeated. Sampled 1-in-N; see /admin/request-sample/{serviceId} for the fingerprints.", + }, + []string{LabelServiceID}, +) + +var RequestSampleTop1Share = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: MetricPrefix + "request_sample_top1_share", + Help: "share of sampled requests in the last completed sampling window that were the single most repeated fingerprint, per service.", + }, + []string{LabelServiceID}, +) + +// SetRequestSampleUniqueness publishes both gauges for a service's completed window. +func SetRequestSampleUniqueness(serviceID string, uniqueness, top1Share float64) { + RequestSampleUniqueness.WithLabelValues(serviceID).Set(uniqueness) + RequestSampleTop1Share.WithLabelValues(serviceID).Set(top1Share) +} diff --git a/metrics/prometheus_reporter.go b/metrics/prometheus_reporter.go index 1b0195914..b7ae70a57 100644 --- a/metrics/prometheus_reporter.go +++ b/metrics/prometheus_reporter.go @@ -100,17 +100,45 @@ func (pmr *PrometheusMetricsReporter) processEndpointObservation(serviceID strin latencyMs = float64(responseTime.AsTime().Sub(queryTime.AsTime()).Milliseconds()) } - // Get status code (returns 0 if not set, treat as 200 for success) + // Check if error was explicitly set (nil means no error, not UNSPECIFIED) + // This is important because UNSPECIFIED when explicitly set means "unknown error", + // while nil means "success with no error" + hasError := endpointObs.ErrorType != nil + errorType := endpointObs.GetErrorType() + + // Backend HTTP status, 0 when the relay never got one. statusCode := int(endpointObs.GetEndpointBackendServiceHttpResponseStatusCode()) - if statusCode == 0 { - statusCode = 200 // Default success - } // Determine RPC type from QoS observations rpcType := pmr.getRPCTypeFromQoS(qosObs) // Metric 5: Request count and latency - statusCodeStr := GetStatusCodeCategory(statusCode) + // + // A status of 0 means the backend never returned an HTTP response. That is TWO + // different outcomes and they must not share a label: + // + // - no error set -> the relay succeeded and the status simply was not recorded. + // - error set -> the relay failed BEFORE any HTTP status existed: a timeout, + // a refused/reset connection, an unreachable host, a signature + // or payload validation failure. + // + // This previously defaulted both to 200, so every transport failure was counted as a + // success against the very endpoint that failed. Measured on solana 2026-08-18: an + // operator producing 242 relay errors/s (5s timeouts, path_relay_latency P95 7.6s) + // reported 0/s non-200 here and read as ~95% successful, while an operator returning + // honest HTTP error codes at 50ms read as ~43%. The panel ranked them backwards, and + // any alert keyed on this metric was blind to exactly the failure mode that matters + // most — an endpoint that accepts the connection and then never answers. + // + // StatusCategoryError matches the vocabulary path_relays_total already uses for the + // same outcome, so the two metrics can be compared without a translation table. + statusCodeStr := StatusCategoryError + switch { + case statusCode != 0: + statusCodeStr = GetStatusCodeCategory(statusCode) + case !hasError: + statusCodeStr = StatusCategorySuccess + } latencySeconds := latencyMs / 1000.0 RecordRequest(domain, rpcType, serviceID, statusCodeStr, latencySeconds) @@ -118,12 +146,6 @@ func (pmr *PrometheusMetricsReporter) processEndpointObservation(serviceID strin latencySignal := GetLatencySignal(latencyMs) RecordLatencyReputation(domain, rpcType, serviceID, latencySignal) - // Check if error was explicitly set (nil means no error, not UNSPECIFIED) - // This is important because UNSPECIFIED when explicitly set means "unknown error", - // while nil means "success with no error" - hasError := endpointObs.ErrorType != nil - errorType := endpointObs.GetErrorType() - // Metric 3: Observation pipeline - determine signal from error type reputationSignal := pmr.getReputationSignalFromEndpoint(hasError, errorType, latencyMs) networkType := pmr.getNetworkType(serviceID, qosObs) @@ -178,8 +200,16 @@ func (pmr *PrometheusMetricsReporter) getReputationSignalFromEndpoint(hasError b // Error field is set - check the specific error type switch errorType { case protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_UNSPECIFIED: - // Error field was set but type is unknown - treat as major error - return SignalMajorError + // An error WAS recorded but the classifier left the type UNSPECIFIED. Every producer + // of that pair (protocol/shannon/error_classification.go: capability limitation, + // over-servicing, session mismatch, a heuristic verdict that was not a fault) records + // a SUCCESS reputation signal alongside it, and the protocol's own observation + // consumers (protocol.go, websocket_context.go) read UNSPECIFIED as success outright. + // This was the one place that read it as a MAJOR error, so a no-fault error — an + // endpoint honestly declining a request it cannot serve — showed on the observation + // pipeline as a penalty that reputation never applied. path_relays_total, which is + // labelled from the real signal, already reports the same relay as ok/error. + return SignalOK case protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_TIMEOUT, protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_HTTP_CONNECTION_TIMEOUT, diff --git a/metrics/prometheus_reporter_signal_label_test.go b/metrics/prometheus_reporter_signal_label_test.go new file mode 100644 index 000000000..690188bde --- /dev/null +++ b/metrics/prometheus_reporter_signal_label_test.go @@ -0,0 +1,30 @@ +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/require" + + protocolobs "github.com/pokt-network/path/observation/protocol" +) + +// The observation pipeline's reputation_signal label is re-derived from the observation's +// error type, not read from the signal reputation actually recorded. Every protocol path +// that records an error with type UNSPECIFIED pairs it with a SUCCESS signal — capability +// limitation, over-servicing, session mismatch — so labelling that pair "major_error" +// reports a penalty that was never applied. Seen in production the moment a new capability +// phrase was catalogued: the service's observation pipeline lit up with major_error while +// path_relays_total (labelled from the real signal) and the mean score said no penalty. +func TestGetReputationSignalFromEndpoint_UnspecifiedErrorIsNoFault(t *testing.T) { + pmr := &PrometheusMetricsReporter{} + + require.Equal(t, SignalOK, + pmr.getReputationSignalFromEndpoint(true, protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_UNSPECIFIED, 10), + "an error recorded with an UNSPECIFIED type is a no-fault error and must not read as a penalty") + + // Controls: a typed error keeps its severity, and no error at all is still ok. + require.Equal(t, SignalMajorError, + pmr.getReputationSignalFromEndpoint(true, protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_TIMEOUT, 10)) + require.Equal(t, SignalOK, + pmr.getReputationSignalFromEndpoint(false, protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_UNSPECIFIED, 10)) +} diff --git a/metrics/request_status_attribution_test.go b/metrics/request_status_attribution_test.go new file mode 100644 index 000000000..5bf863115 --- /dev/null +++ b/metrics/request_status_attribution_test.go @@ -0,0 +1,132 @@ +package metrics + +import ( + "testing" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + protocolobs "github.com/pokt-network/path/observation/protocol" +) + +// Test_RequestStatus_TransportFailureIsNotCountedAsSuccess is the regression test for a +// metric that reported the wrong operator as the healthy one. +// +// A relay that times out, is refused, or fails signature validation never receives an HTTP +// status from the backend, so the observation carries status 0. That was defaulted to 200, +// which counted every such failure as a success against the endpoint that produced it. +// +// Measured on solana 2026-08-18: an operator generating 242 relay errors/s — 5s timeouts, +// P95 relay latency 7.6s — reported 0/s non-200 on path_requests_total and read as ~95% +// successful, while an operator returning honest HTTP error codes in 50ms read as ~43%. The +// supplier-quality panel ranked them backwards. +func Test_RequestStatus_TransportFailureIsNotCountedAsSuccess(t *testing.T) { + const serviceID = "test-transport-failure" + + timeout := protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_TIMEOUT + reporter := &PrometheusMetricsReporter{Logger: polyzero.NewLogger()} + + // Drive the production recorder, not a hand-rolled label set: the defect was in how this + // function derives the label, so a test that passes its own status proves nothing. + reporter.processEndpointObservation(serviceID, &protocolobs.ShannonEndpointObservation{ + Supplier: "pokt1timeout", + EndpointUrl: "https://a001.op-alpha.example", + // No EndpointBackendServiceHttpResponseStatusCode: the backend never answered. + ErrorType: &timeout, + }, 0, nil) + + require.Equal(t, float64(0), requestCount(t, serviceID, StatusCategorySuccess), + "a relay that never received an HTTP status must not be counted as a success") + require.Equal(t, float64(1), requestCount(t, serviceID, StatusCategoryError), + "a transport failure must land on its own status category") +} + +// Test_RequestStatus_MissingStatusWithoutErrorStaysSuccess keeps the other half of the +// status-0 case intact. Not every missing status is a failure — when no error is set the +// relay succeeded and the status simply was not recorded, and turning those into errors +// would swap one wrong number for another. +func Test_RequestStatus_MissingStatusWithoutErrorStaysSuccess(t *testing.T) { + const serviceID = "test-missing-status-ok" + + reporter := &PrometheusMetricsReporter{Logger: polyzero.NewLogger()} + reporter.processEndpointObservation(serviceID, &protocolobs.ShannonEndpointObservation{ + Supplier: "pokt1quiet", + EndpointUrl: "https://b001.op-beta.example", + // Neither a status code nor an error. + }, 0, nil) + + require.Equal(t, float64(1), requestCount(t, serviceID, StatusCategorySuccess)) + require.Equal(t, float64(0), requestCount(t, serviceID, StatusCategoryError)) +} + +// Test_RequestStatus_RealBackendStatusIsPreserved guards the path that was already correct: +// a backend that answers keeps its own status category, error set or not. An endpoint +// answering 500 is reachable; one that never answers is not, and collapsing the two would +// destroy the distinction this fix exists to expose. +func Test_RequestStatus_RealBackendStatusIsPreserved(t *testing.T) { + const serviceID = "test-real-status" + + validationErr := protocolobs.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_RESPONSE_VALIDATION_ERR + reporter := &PrometheusMetricsReporter{Logger: polyzero.NewLogger()} + + for _, tc := range []struct { + name string + status int32 + errType *protocolobs.ShannonEndpointErrorType + expected string + }{ + {name: "429 from the backend", status: 429, expected: "4xx"}, + {name: "500 from the backend", status: 500, expected: "5xx"}, + {name: "200 answered but validation failed", status: 200, errType: &validationErr, expected: StatusCategorySuccess}, + } { + t.Run(tc.name, func(t *testing.T) { + svc := serviceID + "-" + tc.name + status := tc.status + reporter.processEndpointObservation(svc, &protocolobs.ShannonEndpointObservation{ + Supplier: "pokt1answers", + EndpointUrl: "https://c001.op-gamma.example", + EndpointBackendServiceHttpResponseStatusCode: &status, + ErrorType: tc.errType, + }, 0, nil) + + require.Equal(t, float64(1), requestCount(t, svc, tc.expected)) + require.Equal(t, float64(0), requestCount(t, svc, StatusCategoryError), + "an endpoint that returned an HTTP status must not be filed under 'no status at all'") + }) + } +} + +// requestCount reads one path_requests_total child back out of the registry. +// +// Gather() only reports children that exist, so an absent series reads as 0 — which is what +// the assertions above want, and why every test uses its own service_id rather than sharing +// a counter across cases. +func requestCount(t *testing.T, serviceID, statusCode string) float64 { + t.Helper() + + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + for _, family := range families { + if family.GetName() != MetricPrefix+"requests_total" { + continue + } + for _, metric := range family.GetMetric() { + if labelValue(metric, LabelServiceID) == serviceID && labelValue(metric, LabelStatusCode) == statusCode { + return metric.GetCounter().GetValue() + } + } + } + return 0 +} + +func labelValue(metric *dto.Metric, name string) string { + for _, label := range metric.GetLabel() { + if label.GetName() == name { + return label.GetValue() + } + } + return "" +} diff --git a/metrics/supplier_hedge_cardinality_test.go b/metrics/supplier_hedge_cardinality_test.go index e05ceca7a..99107ff1d 100644 --- a/metrics/supplier_hedge_cardinality_test.go +++ b/metrics/supplier_hedge_cardinality_test.go @@ -7,45 +7,15 @@ import ( "github.com/stretchr/testify/require" ) -// Test_supplierSignalSeverity locks the 8-signal-type → 3-severity-class -// collapse that bounds supplier_signal_total cardinality. The exact strings are -// the reputation.SignalType wire values (metrics cannot import reputation — -// import cycle — so they are asserted literally here as the contract). -func Test_supplierSignalSeverity(t *testing.T) { - cases := map[string]string{ - "success": SupplierSeverityOK, - "recovery_success": SupplierSeverityOK, - "slow_response": SupplierSeveritySlow, - "very_slow_response": SupplierSeveritySlow, - "minor_error": SupplierSeverityError, - "major_error": SupplierSeverityError, - "critical_error": SupplierSeverityError, - "fatal_error": SupplierSeverityError, - // Unknown / newly-added types must fall through to error, never leak a - // new label value (keeps cardinality bounded + surfaces the omission). - "some_future_signal": SupplierSeverityError, - "": SupplierSeverityError, - } - for in, want := range cases { - require.Equalf(t, want, supplierSignalSeverity(in), "signal %q", in) - } - - // Only three severity values may ever be emitted. - seen := map[string]struct{}{} - for in := range cases { - seen[supplierSignalSeverity(in)] = struct{}{} - } - require.LessOrEqual(t, len(seen), 3, "severity label must have at most 3 values") -} - -// Test_RecordSupplierSignal_EmptySupplierDropped guards the empty-supplier skip -// (per-domain reputation keys carry no supplier). -func Test_RecordSupplierSignal_EmptySupplierDropped(t *testing.T) { - before := testutil.CollectAndCount(SupplierSignalTotal) - RecordSupplierSignal("", "eth", "success") - require.Equal(t, before, testutil.CollectAndCount(SupplierSignalTotal), - "empty supplier must not create a series") -} +// Tests for the de-labeling of the per-supplier metric family. +// +// Two metrics that lived here — path_supplier_signal_total and +// path_supplier_reputation_score — were removed entirely on 2026-08-12, along +// with their severity-collapse and empty-supplier tests. Both were GUARDED and +// both HONORED their guard while remaining among the largest series sources in +// the gateway job, because a guard caps the live registry and not the number of +// distinct series Prometheus retains. See DefaultSeriesLimit for the measurements +// and Test_SupplierLabelIsGone below for the property that replaced them. // Test_RecordHealthCheck_CollapsesSuppliers locks the de-labeling of // path_health_check_status_total. The metric carried a `supplier` label with no @@ -71,29 +41,148 @@ func Test_RecordHealthCheck_CollapsesSuppliers(t *testing.T) { RecordHealthCheck(domain, "", rpcType, serviceID, checkName, SignalOK) require.Equal(t, before+3, testutil.ToFloat64(series), - "all suppliers behind one domain must land on the same series") + "all suppliers behind one domain must collapse onto the same series") +} + +// Test_RecordSupplierBlacklist_CollapsesSuppliers locks the `supplier` label drop +// on path_supplier_blacklist_total. The address is still accepted (and still +// logged at the call site); it must not reach a label. +func Test_RecordSupplierBlacklist_CollapsesSuppliers(t *testing.T) { + const ( + domain = "blacklist-delabel.example" + serviceID = "eth" + reason = BlacklistReasonSignatureError + ) + + // Arity assertion: 3 labels, no `supplier`. + series := SupplierBlacklistTotal.WithLabelValues(domain, serviceID, reason) + before := testutil.ToFloat64(series) + + RecordSupplierBlacklist(domain, "pokt1blacklistone", serviceID, reason) + RecordSupplierBlacklist(domain, "pokt1blacklisttwo", serviceID, reason) + + require.Equal(t, before+2, testutil.ToFloat64(series), + "two suppliers on one domain must collapse onto the same series") +} + +// Test_RecordQoSFilterRejection_KeysOnDomain locks the supplier→domain re-key. +// +// This metric fires ~9,500/s fleet-wide and, with a supplier label, had the worst +// churn of any gateway metric: 1,271 series live in a 10-minute window against +// 24,708 distinct minted over one pod's 7.7h life (19.4×). +func Test_RecordQoSFilterRejection_KeysOnDomain(t *testing.T) { + const ( + domain = "qosfilter-rekey.example" + serviceID = "eth" + reason = QoSFilterReasonBlockHeightLag + ) + + // Arity assertion: (domain, service_id, reason). + series := QoSFilterRejectionTotal.WithLabelValues(domain, serviceID, reason) + before := testutil.ToFloat64(series) + + RecordQoSFilterRejection(domain, serviceID, reason) + RecordQoSFilterRejection(domain, serviceID, reason) + require.Equal(t, before+2, testutil.ToFloat64(series)) + + // Empty target is skipped rather than recorded as DomainUnknown: the empty + // check MUST run before SanitizeDomainLabel, which maps "" to DomainUnknown + // and would turn "no endpoint context" into a real series. + unknown := QoSFilterRejectionTotal.WithLabelValues(DomainUnknown, serviceID, reason) + unknownBefore := testutil.ToFloat64(unknown) + RecordQoSFilterRejection("", serviceID, reason) + require.Equal(t, unknownBefore, testutil.ToFloat64(unknown), + "empty domain must be skipped, not collapsed onto DomainUnknown") + + // A supplier address reaching this metric collapses to the sentinel instead of + // expanding ~1:1 with the supplier set — the failure this re-key exists to + // prevent, in case a caller passes an EndpointAddr instead of a domain. + sentinel := QoSFilterRejectionTotal.WithLabelValues(DomainSupplierAddr, serviceID, reason) + sentinelBefore := testutil.ToFloat64(sentinel) + RecordQoSFilterRejection("pokt1qosfilterleakedaddress", serviceID, reason) + require.Equal(t, sentinelBefore+1, testutil.ToFloat64(sentinel), + "a leaked supplier address must land on the supplier_addr sentinel") } // Test_RecordHedgeSupplierOutcome_Split guards the histogram→(counter+role -// histogram) split: the role latency histogram is always recorded, but the -// per-supplier counter is skipped when supplier is empty. +// histogram) split, and the supplier→domain re-key of the counter: the role +// latency histogram is always recorded, but the per-operator counter is skipped +// when the domain is unknown. func Test_RecordHedgeSupplierOutcome_Split(t *testing.T) { - // Empty supplier: role histogram still observes, per-supplier counter does not. + // Unknown domain: role histogram still observes, per-operator counter does not. histBefore := testutil.CollectAndCount(HedgeRoleLatency) RecordHedgeSupplierOutcome("", HedgeRoleWinner, 0.12) - require.Greater(t, testutil.CollectAndCount(HedgeRoleLatency), histBefore-1, - "role latency histogram must record even without a supplier") + require.GreaterOrEqual(t, testutil.CollectAndCount(HedgeRoleLatency), histBefore, + "role latency histogram must record even without a domain") - // Known supplier: per-supplier counter increments for the right role. - const supplier = "pokt1testsupplierhedge" - winBefore := testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(supplier, HedgeRoleWinner)) - RecordHedgeSupplierOutcome(supplier, HedgeRoleWinner, 0.2) - winAfter := testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(supplier, HedgeRoleWinner)) - require.Equal(t, winBefore+1, winAfter, "winner count must increment for known supplier") + // Known operator: per-operator counter increments for the right role. + const domain = "hedge-outcome.example" + winBefore := testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(domain, HedgeRoleWinner)) + RecordHedgeSupplierOutcome(domain, HedgeRoleWinner, 0.2) + require.Equal(t, winBefore+1, + testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(domain, HedgeRoleWinner)), + "winner count must increment for a known operator") // Loser role is a distinct series. - loseBefore := testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(supplier, HedgeRoleLoser)) - RecordHedgeSupplierOutcome(supplier, HedgeRoleLoser, 0.5) + loseBefore := testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(domain, HedgeRoleLoser)) + RecordHedgeSupplierOutcome(domain, HedgeRoleLoser, 0.5) require.Equal(t, loseBefore+1, - testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(supplier, HedgeRoleLoser))) + testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(domain, HedgeRoleLoser))) + + // Two suppliers of the same operator must collapse, which is the whole point + // of the re-key: a raw supplier address hits the sentinel, not its own series. + sentinelBefore := testutil.ToFloat64( + HedgeSupplierOutcomeTotal.WithLabelValues(DomainSupplierAddr, HedgeRoleWinner)) + RecordHedgeSupplierOutcome("pokt1hedgeleakedaddressone", HedgeRoleWinner, 0.3) + RecordHedgeSupplierOutcome("pokt1hedgeleakedaddresstwo", HedgeRoleWinner, 0.3) + require.Equal(t, sentinelBefore+2, + testutil.ToFloat64(HedgeSupplierOutcomeTotal.WithLabelValues(DomainSupplierAddr, HedgeRoleWinner)), + "leaked supplier addresses must collapse onto one sentinel series") +} + +// Test_DomainFromEndpointAddr covers the EndpointAddr → operator-domain helper +// that the re-keyed call sites depend on. +// +// It returns "" rather than DomainUnknown on failure BY DESIGN: the Record* +// helpers check for empty before sanitizing, so returning a sentinel here would +// defeat their skip and mint a series for every context-less call. +func Test_DomainFromEndpointAddr(t *testing.T) { + cases := []struct { + name string + addr string + want string + }{ + { + name: "supplier-url form yields eTLD+1", + addr: "pokt1abc-https://relayminer.shannon-mainnet.eu.example.net", + want: "example.net", + }, + { + name: "subdomains collapse onto one operator", + addr: "pokt1def-https://other.host.example.net:8443", + want: "example.net", + }, + { + name: "no dash separator yields empty, not a sentinel", + addr: "pokt1abcnoseparator", + want: "", + }, + { + name: "empty input yields empty", + addr: "", + want: "", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, c.want, DomainFromEndpointAddr(c.addr)) + }) + } + + // Two suppliers behind the same operator must resolve to one domain — the + // property that turns a chain-sized label into an operator-sized one. + a := DomainFromEndpointAddr("pokt1one-https://a.example.net") + b := DomainFromEndpointAddr("pokt1two-https://b.example.net") + require.Equal(t, a, b, "different suppliers on one operator must share a domain") + require.NotEmpty(t, a) } diff --git a/pnf_path_rules.yaml b/pnf_path_rules.yaml index 9d5f0ef18..c9c389bde 100644 --- a/pnf_path_rules.yaml +++ b/pnf_path_rules.yaml @@ -905,7 +905,17 @@ # Block time: ~4s | Source: evm-sidechain.xrpl.org explorer # Formula: 300s / 4s = 75 blocks for 5 minutes sync_allowance: 75 - check_interval: 10s + # 30s, not 10s: websocket checks are the one check type NOT covered by backend-URL + # dedup (runEndpointChecks runs siblings with runWS=true so each is probed directly), + # so an operator holding N registrations behind ONE node URL takes N connect/close + # cycles per interval against the same socket server, per environment. Reported + # 2026-08-14 by an operator whose node logged our probe's own close reason + # ("health check complete", close 1000 Normal) as a flood — ethermint's websocket + # server logs every disconnect at Error level regardless of close code, so a clean + # probe still reads as an error on their side. + # Interval is the only lever available from config; the structural fix is extending + # backend-URL dedup to websocket probes whose siblings share an identical WS URL. + check_interval: 30s enabled: true checks: - name: eth_blockNumber @@ -943,8 +953,6 @@ archival: true reputation_signal: minor_error -# Hyperliquid is an EVM chain (not Cosmos) -# KNOWN ISSUE: eth_call ignores block number parameter - always returns latest block context # WebSocket data-path check. Deliberately sends a request rather than only # opening the socket: a supplier can complete the handshake, answer pings, and # still deliver nothing, so connect-only would score it healthy. @@ -961,6 +969,8 @@ timeout: 10s reputation_signal: major_error +# Hyperliquid is an EVM chain (not Cosmos) +# KNOWN ISSUE: eth_call ignores block number parameter - always returns latest block context - service_id: hyperliquid # Block time: ~1s | Source: hyperliquid docs, block explorer # Formula: 300s / 1s = 300 blocks for 5 minutes @@ -1976,7 +1986,9 @@ # Block time: ~4s | Source: Same as XRPL EVM mainnet # Formula: 300s / 4s = 75 blocks for 5 minutes sync_allowance: 75 - check_interval: 15s + # 30s, not 15s: same websocket probe noise as xrplevm mainnet — see that service for + # the dedup gap this works around. Both were reported in the same operator thread. + check_interval: 30s enabled: true checks: - name: eth_blockNumber @@ -2079,7 +2091,7 @@ expected_status_code: 200 expected_response_contains: '"ok"' timeout: 5s - reputation_signal: critical_error + reputation_signal: major_error - name: getBlockHeight type: json_rpc method: POST @@ -2477,28 +2489,77 @@ - service_id: kava # Block time: ~6s | Source: mintscan.io/kava, kava.io docs # Formula: 300s / 6s = 50 blocks for 5 minutes + # + # Kava is a dual chain: Cosmos SDK (comet_bft + rest) AND EVM (json_rpc), served by the + # same node on different ports. The `type` field is NOT cosmetic here - it selects which + # of the endpoint's per-rpc-type URLs the check is relayed to. + # + # FIXED 2026-08-14: `health` and `status` are CometBFT methods but were declared + # `type: json_rpc`, so every one was delivered to the endpoint's EVM JSON-RPC URL, which + # answered {"error":{"code":-32601,"message":"the method health does not exist/is not + # available"}}. `status` carries critical_error, so kava's json_rpc mean score sat at + # ~10.1 while rest read 85.7 - a rule bug, not an operator fault. Reported with the + # node-side capture showing the EVM port receiving `health`/`status`. + # + # comet_bft read 100 for the opposite reason: no check in this file was typed comet_bft, + # so that rpc type was never exercised. 100 means "never penalized", not "healthy". + # + # Verified against public Kava infrastructure 2026-08-14: + # evm.kava.io eth_chainId -> 0x8ae (2222); eth_blockNumber -> 22100394 + # rpc.data.kava.io status.sync_info.latest_block_height -> 22100395 + # EVM height tracks CometBFT height 1:1 (off by one block), so the single perceived + # height per service is valid for both sync checks. sync_allowance: 50 check_interval: 30s enabled: true checks: - - name: health + # --- EVM side (json_rpc) --- + - name: eth_blockNumber + type: json_rpc + method: POST + path: / + body: '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' + expected_status_code: 200 + timeout: 5s + reputation_signal: critical_error + sync_check: true + - name: eth_chainId type: json_rpc method: POST path: / + body: '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' + expected_status_code: 200 + # Chain-ID assertion: guards against a supplier serving a DIFFERENT chain + # under this service ID (see tron/Base incident). Trailing quote anchors the + # match so a short id like 0x1 cannot substring-match 0x1388 etc. + expected_response_contains: '0x8ae"' + timeout: 5s + reputation_signal: critical_error + # --- Cosmos side (comet_bft) --- + # major_error, not critical: comet_bft has never been measured on any service in this + # file, so its scores are all "never penalized" rather than proven. Opening at critical + # (-50, triggers cooldown) on an unexercised rpc type risks mass cooldowns for a rule + # defect rather than an endpoint one. Escalate once the false-positive rate is known - + # same staging as the websocket checks above. + - name: health + type: comet_bft + method: POST + path: / body: '{"jsonrpc":"2.0","id":1,"method":"health"}' expected_status_code: 200 timeout: 5s reputation_signal: minor_error - name: status - type: json_rpc + type: comet_bft method: POST path: / body: '{"jsonrpc":"2.0","id":1,"method":"status"}' expected_status_code: 200 expected_response_contains: '"node_info"' timeout: 5s - reputation_signal: critical_error + reputation_signal: major_error sync_check: true + # --- Cosmos side (rest) --- - name: syncing type: rest method: GET diff --git a/protocol/shannon/context.go b/protocol/shannon/context.go index c96134248..c587c9390 100644 --- a/protocol/shannon/context.go +++ b/protocol/shannon/context.go @@ -180,8 +180,15 @@ type requestContext struct { // outcome, so the Shannon layer SKIPS its RecordRelay for these — otherwise the same // relay was ALSO recorded as request_type="normal", leaving every "excludes health // checks" dashboard panel still containing 100% of health-check volume (phantom - // "normal" traffic on services with no user requests). Does not affect reputation - // signals or observations. + // "normal" traffic on services with no user requests). + // + // It is ALSO stamped onto every reputation signal this layer records. The rate + // detectors in reputation.RecordSignal are wrapped in `if !signal.IsHealthCheck` + // precisely so a probe cannot bench an endpoint on its own, but only the health-check + // executor's own three RecordSignal call sites set that flag — the relay path below + // left it false, so every health-check relay fed both rate EWMAs as if it were user + // traffic. The additive score is unaffected either way: a probe result still moves + // Value, exactly as before. Only the volume-independent rate detectors exclude it. isHealthCheck bool // tieredSelector provides access to tier-based selection and probation status. @@ -1282,6 +1289,13 @@ func (rc *requestContext) handleEndpointError( keyBuilder := rc.reputationService.KeyBuilderForService(rc.serviceID) endpointKey := keyBuilder.BuildKey(rc.serviceID, selectedEndpointAddr, rc.getCurrentRPCType()) + // Mark probe-originated signals so the volume-independent rate detectors skip them. + // See the isHealthCheck field doc: without this a health check's failure counts + // toward the very rates that decide whether to bench the endpoint, and a benched + // endpoint receives no user traffic — so probes become its only signal and it + // re-benches itself indefinitely. + signal.IsHealthCheck = rc.isHealthCheck + // Fire-and-forget: don't block request on reputation recording if err := rc.reputationService.RecordSignal(rc.context, endpointKey, signal); err != nil { rc.logger.Warn().Err(err).Msg("Failed to record reputation signal for error") @@ -1302,8 +1316,9 @@ func (rc *requestContext) handleEndpointError( rpcTypeStr := metrics.NormalizeRPCType(rc.getCurrentRPCType().String()) reputationSignal := mapSignalTypeToMetricSignal(signal.Type) - // Extract status code from error if possible, otherwise use "error" - statusCodeStr := "error" + // Extract status code from error if possible, otherwise use the shared + // "no HTTP status at all" category. + statusCodeStr := metrics.StatusCategoryError if statusCode, ok := extractHTTPStatusCode(endpointErr); ok { statusCodeStr = metrics.GetStatusCodeCategory(statusCode) } @@ -1423,6 +1438,12 @@ func (rc *requestContext) handleEndpointSuccess( relayType = metrics.RelayTypeNormal } + // Probe-originated signals are excluded from the rate detectors. This matters on + // the success path too: the EWMAs are ratios, so counting a probe's success in the + // denominator while its failures are excluded from the numerator would bias the + // measured rate downward instead of leaving it untouched. + signal.IsHealthCheck = rc.isHealthCheck + // Fire-and-forget: don't block request on reputation recording if err := rc.reputationService.RecordSignal(rc.context, endpointKey, signal); err != nil { rc.logger.Warn().Err(err).Msg("Failed to record reputation signal for success") @@ -1502,6 +1523,10 @@ func (rc *requestContext) recordLatencyPenaltySignalsIfNeeded( return // Unknown signal type, skip } + // A probe's latency is not user-experienced latency; exclude it from the rate + // detectors on the same grounds as the signals above. + penaltySignal.IsHealthCheck = rc.isHealthCheck + // Fire-and-forget: don't block request on reputation recording if err := rc.reputationService.RecordSignal(rc.context, endpointKey, penaltySignal); err != nil { rc.logger.Warn().Err(err). diff --git a/protocol/shannon/empty_payload_signal_test.go b/protocol/shannon/empty_payload_signal_test.go new file mode 100644 index 000000000..c258f07da --- /dev/null +++ b/protocol/shannon/empty_payload_signal_test.go @@ -0,0 +1,107 @@ +package shannon + +import ( + "fmt" + "testing" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/reputation" +) + +// Regression tests for the empty-payload severity gap found 2026-08-19. +// +// A supplier returning a signature-valid RelayResponse whose body is empty passed +// every check PATH performs — signature verification, ValidateBasic, and unmarshal. +// Only a content heuristic caught it, and that heuristic's signal was MINOR (-3), +// the same weight as a passing blockchain_error. One endpoint produced ~800 empty +// responses in a two-minute sample while holding a reputation score of 100: at a +// small fraction of total volume, successes outrun -3 indefinitely. +// +// An empty body on a body-bearing 2xx has no valid reading for any RPC type PATH +// forwards, and the relay is signed and settleable regardless of content — so it is +// a protocol violation and belongs at CRITICAL, alongside protocol_error. +// +// These assert through classifyErrorAsSignal, the function the relay path calls, +// rather than on the classifier branch directly. +func TestClassifyErrorAsSignal_EmptyResponseIsCritical(t *testing.T) { + logger := polyzero.NewLogger() + + // Shaped exactly as the production error is built at context.go:843. + err := fmt.Errorf("raw_payload: %s: heuristic detected %s (method=%s): %w", + "", "empty_response", "getTokenAccountsByOwner", errHeuristicDetectedBackendError) + + _, signal := classifyErrorAsSignal(logger, err, 250*time.Millisecond) + + require.Equal(t, reputation.SignalTypeCriticalError, signal.Type, + "an empty body on a body-bearing 2xx is a protocol violation, not a transient fault") + require.Equal(t, "empty_response", signal.Reason) + + // The severity alone cannot express this at production rates: at ~0.2% of traffic an + // additive score earns +998 against -25 per 1000 requests and returns to its ceiling. + // The flag is what routes it to the rate-based detector, so it is load-bearing and must + // be asserted here — without this line, deleting it breaks nothing visible. + require.True(t, signal.IsProtocolViolation, + "empty_response must be flagged as a protocol violation so it feeds the invalid-rate detector") +} + +// TestClassifyErrorAsSignal_SmallNoResultStaysMinor pins the deliberate split. A short +// response missing a "result" field is ambiguous — a truncated read or a terse upstream +// error — unlike a zero-length body, which has no valid reading. Raising both together +// would have been the easier edit and the wrong one. +func TestClassifyErrorAsSignal_SmallNoResultStaysMinor(t *testing.T) { + logger := polyzero.NewLogger() + + err := fmt.Errorf("raw_payload: %s: heuristic detected %s (method=%s): %w", + `{"jsonrpc":"2.0"}`, "small_no_result", "getSlot", errHeuristicDetectedBackendError) + + _, signal := classifyErrorAsSignal(logger, err, 250*time.Millisecond) + + require.Equal(t, reputation.SignalTypeMinorError, signal.Type, + "small_no_result is ambiguous and must not inherit the empty-body severity") + require.Equal(t, "small_no_result", signal.Reason) +} + +// TestClassifyErrorAsSignal_ReasonSuffixDoesNotDefeatMatching is the pin for the root +// cause. context.go builds the error as "heuristic detected %s (method=%s)", so the +// reason reaches the classifier as "empty_response (method=getSlot)". Every +// exact-equality case in classifyHeuristicErrorAsSignal therefore never matched for a +// JSON-RPC request and fell through to unknown_payload_error at MINOR. Only the +// HasPrefix("error_indicator_...") cases survived, which is why the gap stayed +// invisible — the surviving cases covered the common errors. +// +// Asserted per-reason with and without the suffix: the two must agree. +func TestClassifyErrorAsSignal_ReasonSuffixDoesNotDefeatMatching(t *testing.T) { + logger := polyzero.NewLogger() + + cases := []struct { + reason string + wantType reputation.SignalType + wantReason string + }{ + {"empty_response", reputation.SignalTypeCriticalError, "empty_response"}, + {"small_no_result", reputation.SignalTypeMinorError, "small_no_result"}, + {"html_error_page", reputation.SignalTypeCriticalError, "service_error"}, + {"bad_gateway", reputation.SignalTypeCriticalError, "service_error"}, + } + + for _, tc := range cases { + t.Run(tc.reason, func(t *testing.T) { + withMethod := fmt.Errorf("raw_payload: %s: heuristic detected %s (method=%s): %w", + "", tc.reason, "getSlot", errHeuristicDetectedBackendError) + bare := fmt.Errorf("raw_payload: %s: heuristic detected %s: %w", + "", tc.reason, errHeuristicDetectedBackendError) + + _, gotWith := classifyErrorAsSignal(logger, withMethod, 250*time.Millisecond) + _, gotBare := classifyErrorAsSignal(logger, bare, 250*time.Millisecond) + + require.Equal(t, tc.wantType, gotWith.Type, + "reason %q carrying a (method=...) suffix must classify the same as without it", tc.reason) + require.Equal(t, tc.wantReason, gotWith.Reason) + require.Equal(t, gotBare.Type, gotWith.Type, "suffix changed the signal type") + require.Equal(t, gotBare.Reason, gotWith.Reason, "suffix changed the signal reason") + }) + } +} diff --git a/protocol/shannon/error_classification.go b/protocol/shannon/error_classification.go index 1572bfacf..6654e47df 100644 --- a/protocol/shannon/error_classification.go +++ b/protocol/shannon/error_classification.go @@ -329,9 +329,32 @@ func classifyHeuristicErrorAsSignal( reputation.NewCriticalErrorSignal("protocol_error", latency) // Category: Empty Response (MINOR -3) - case reason == "empty_response", reason == "small_no_result": + // An empty body on a body-bearing 2xx is a protocol violation, not a transient + // fault: no RPC type PATH forwards has a valid response of zero length, and the + // relay was signed and is settleable regardless. It sat at MINOR (-3) — the same + // weight as a passing blockchain_error — which is why an endpoint returning ~800 + // empty responses in two minutes held a reputation score of 100: at a small + // fraction of total volume, successes outrun -3 indefinitely. Statuses that + // legitimately carry no body (204/205/304) never reach here; the heuristic + // exempts them, so this branch only ever sees a promise of content that was + // not delivered. + // Category: Supplier Protocol Violations (CRITICAL -25) + case reason == "empty_response": + // Also flagged as a protocol violation so it feeds the invalid-rate detector. + // The CRITICAL severity alone cannot express this: at the rates observed in + // production (~0.2% of all relays) an additive score is outvoted by successes and + // returns to 100. Only a rate-based signal reaches it. + emptySignal := reputation.NewCriticalErrorSignal("empty_response", latency) + emptySignal.IsProtocolViolation = true + return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_RAW_PAYLOAD_UNEXPECTED_EOF, + emptySignal + + // small_no_result stays MINOR: a short response missing a "result" field is + // ambiguous — it can be a truncated read or a terse upstream error — unlike a + // zero-length body, which has no valid reading. + case reason == "small_no_result": return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_RAW_PAYLOAD_UNEXPECTED_EOF, - reputation.NewMinorErrorSignal("empty_response") + reputation.NewMinorErrorSignal("small_no_result") // Default: Treat as unknown malformed payload default: @@ -395,6 +418,19 @@ func classifyMalformedPayloadAsSignal(logger polylog.Logger, payloadContent stri // Clean up body for logging and further analysis payloadContent = payloadContent[:idx] + // The producer appends " (method=)" to the reason + // (context.go: "heuristic detected %s (method=%s)"), so the reason arrives as + // e.g. "empty_response (method=getTokenAccountsByOwner)". Every exact-equality + // case in classifyHeuristicErrorAsSignal — empty_response, small_no_result, + // html_error_page, bad_gateway, rest_error_field, rest_code_message_error — + // therefore never matched for a JSON-RPC request, and those responses fell + // through to the default unknown_payload_error at MINOR. Only the + // HasPrefix("error_indicator_...") cases survived, because prefix matching + // tolerates the suffix, which is why the gap stayed invisible. + if paren := strings.Index(heuristicReason, " ("); paren != -1 { + heuristicReason = heuristicReason[:paren] + } + logger = logger.With( "payload_content_preview", payloadContent[:min(len(payloadContent), 200)], "heuristic_reason", heuristicReason, diff --git a/protocol/shannon/healthcheck_reputation_stamp_test.go b/protocol/shannon/healthcheck_reputation_stamp_test.go new file mode 100644 index 000000000..5c07a810f --- /dev/null +++ b/protocol/shannon/healthcheck_reputation_stamp_test.go @@ -0,0 +1,143 @@ +package shannon + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" + + protocolobservations "github.com/pokt-network/path/observation/protocol" + "github.com/pokt-network/path/protocol" + "github.com/pokt-network/path/reputation" +) + +// signalCapture records every signal the code under test hands to reputation, so a test can +// assert on what the PRODUCTION CALLER actually delivers rather than on a local variable. +// +// The embedded nil interface makes any method the code unexpectedly depends on panic rather +// than silently return a zero value. +type signalCapture struct { + reputation.ReputationService + + mu sync.Mutex + signals []reputation.Signal +} + +func (c *signalCapture) RecordSignal(_ context.Context, _ reputation.EndpointKey, signal reputation.Signal) error { + c.mu.Lock() + defer c.mu.Unlock() + c.signals = append(c.signals, signal) + return nil +} + +func (c *signalCapture) KeyBuilderForService(_ protocol.ServiceID) reputation.KeyBuilder { + return reputation.NewKeyBuilder(reputation.KeyGranularityEndpoint) +} + +func (c *signalCapture) GetLatencyConfigForService(_ protocol.ServiceID) reputation.LatencyConfig { + return reputation.LatencyConfig{} +} + +func (c *signalCapture) recorded() []reputation.Signal { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]reputation.Signal, len(c.signals)) + copy(out, c.signals) + return out +} + +// newStampTestContext builds a requestContext wired to a signal capture, with a fallback +// endpoint standing in as the selected endpoint (it satisfies the full `endpoint` interface +// with no session or chain state required). +func newStampTestContext(t *testing.T, cap *signalCapture) *requestContext { + t.Helper() + rc := &requestContext{ + logger: testLogger(), + context: context.Background(), + serviceID: protocol.ServiceID("solana"), + reputationService: cap, + selectedEndpoint: fallbackEndpoint{ + defaultURL: "https://probe.example.com/rpc", + }, + } + rc.currentRPCType.Store(int32(sharedtypes.RPCType_JSON_RPC)) + return rc +} + +// TestHealthCheckRelay_StampsReputationSignal_ErrorPath is the fix-1 test for the relay +// path, and it asserts on the signal reputation RECEIVES — not on rc.isHealthCheck, which +// was already true and told us nothing. +// +// A health-check relay reaches reputation through this handler because the executor calls +// MarkAsHealthCheck() and then HandleServiceRequest(). The `isHealthCheck` flag was +// consulted only to skip the relay METRIC; the field doc said outright that it "does not +// affect reputation signals", so every probe failure fed the volume-independent rate +// detectors as though a user had experienced it. +func TestHealthCheckRelay_StampsReputationSignal_ErrorPath(t *testing.T) { + cap := &signalCapture{} + rc := newStampTestContext(t, cap) + rc.MarkAsHealthCheck() + + _, _ = rc.handleEndpointError(time.Now(), protocol.Response{}, errors.New("connection refused")) + + sigs := cap.recorded() + require.Len(t, sigs, 1, "the error handler must record exactly one reputation signal") + require.True(t, sigs[0].IsHealthCheck, + "a health-check relay's failure must arrive at reputation stamped, or the rate detectors will act on a probe result") +} + +// TestUserRelay_DoesNotStampReputationSignal_ErrorPath is the control. Without it the test +// above passes on a change that stamps every signal unconditionally, which would make the +// rate detectors permanently inert — a far worse bug than the one being fixed. +func TestUserRelay_DoesNotStampReputationSignal_ErrorPath(t *testing.T) { + cap := &signalCapture{} + rc := newStampTestContext(t, cap) + + _, _ = rc.handleEndpointError(time.Now(), protocol.Response{}, errors.New("connection refused")) + + sigs := cap.recorded() + require.Len(t, sigs, 1) + require.False(t, sigs[0].IsHealthCheck, + "control: user traffic must reach the rate detectors") +} + +// TestHealthCheckObservations_StampReputationSignal is the fix-1 test for the SECOND path a +// probe reaches reputation by. +// +// One health check produces two reputation signals: the relay itself (above) and +// ApplyHTTPObservations, which the executor calls on the same relay's observations. Fixing +// only the relay path would have left half the probe volume feeding the EWMAs while looking +// fixed — the trap this repo has hit repeatedly, where one of several call sites is missed +// and the metric still reports success. +func TestHealthCheckObservations_StampReputationSignal(t *testing.T) { + for _, tc := range []struct { + name string + isHealthCheck bool + }{ + {"health check observations are stamped", true}, + {"user traffic observations are not", false}, + } { + t.Run(tc.name, func(t *testing.T) { + cap := &signalCapture{} + p := &Protocol{logger: testLogger(), reputationService: cap} + + p.recordSignalFromObservation( + protocol.ServiceID("solana"), + &protocolobservations.ShannonEndpointObservation{ + Supplier: "pokt1supplier", + EndpointUrl: "https://probe.example.com/rpc", + }, + tc.isHealthCheck, + ) + + sigs := cap.recorded() + require.Len(t, sigs, 1) + require.Equal(t, tc.isHealthCheck, sigs[0].IsHealthCheck, + "the observation path must carry the caller's health-check verdict through to reputation") + }) + } +} diff --git a/protocol/shannon/leaderboard.go b/protocol/shannon/leaderboard.go index 999f02e92..2040deded 100644 --- a/protocol/shannon/leaderboard.go +++ b/protocol/shannon/leaderboard.go @@ -313,99 +313,16 @@ func (p *Protocol) GetMeanScoreData(ctx context.Context) ([]metrics.MeanScoreEnt return entries, nil } -// GetSupplierScoreData implements the metrics.LeaderboardDataProvider interface. -// It returns the mean reputation score per (supplier, service_id, rpc_type) triple, -// averaging per-endpoint scores when the supplier has multiple endpoints of the same -// rpc_type. +// GetSupplierScoreData was removed with path_supplier_reputation_score +// (2026-08-12). It walked every service's active sessions, expanded every unique +// endpoint and did a reputation GetScore per endpoint, every 10 seconds, purely +// to feed a gauge that nothing queried and that minted ~232K distinct Prometheus +// series per pod per day. Per-supplier scores are served on demand by +// GET /ready/?detailed=true, which reads the same reputation state +// without a periodic fleet-wide walk. // -// Cardinality note: bounded by active suppliers × active services × rpc_types (a -// supplier typically serves 1-3 rpc types). The rpc_type split is deliberate: a -// supplier's websocket reputation is tracked and disqualified separately from its -// json_rpc reputation, and averaging the two together hid genuinely-broken websocket -// endpoints behind a healthy json_rpc score. -func (p *Protocol) GetSupplierScoreData(ctx context.Context) ([]metrics.SupplierScoreEntry, error) { - logger := p.logger.With("method", "GetSupplierScoreData") - - if p.unifiedServicesConfig == nil { - logger.Debug().Msg("No unified services config available, returning empty supplier scores") - return nil, nil - } - - if p.reputationService == nil { - logger.Debug().Msg("Reputation service not enabled, returning empty supplier scores") - return nil, nil - } - - type supplierKey struct { - Supplier string - ServiceID string - RPCType string - } - type aggregator struct { - Total float64 - Count int - } - aggregates := make(map[supplierKey]*aggregator) - - for _, serviceConfig := range p.unifiedServicesConfig.Services { - serviceID := serviceConfig.ID - - activeSessions, err := p.getCentralizedGatewayModeActiveSessions(ctx, serviceID, false) - if err != nil || len(activeSessions) == 0 { - continue - } - - rpcTypesToQuery := p.getServiceRPCTypesForLeaderboard(serviceID) - - for _, rpcType := range rpcTypesToQuery { - endpoints, actualRPCType, err := p.getUniqueEndpoints(ctx, serviceID, activeSessions, false, rpcType, nil, "") - if err != nil { - continue - } - - for endpointAddr := range endpoints { - supplier, err := endpointAddr.GetAddress() - if err != nil || supplier == "" { - continue - } - - keyBuilder := p.reputationService.KeyBuilderForService(serviceID) - key := keyBuilder.BuildKey(serviceID, endpointAddr, actualRPCType) - score, scoreErr := p.reputationService.GetScore(ctx, key) - if scoreErr != nil { - score = reputation.Score{Value: p.reputationService.GetInitialScoreForService(serviceID)} - } - - aggKey := supplierKey{ - Supplier: supplier, - ServiceID: string(serviceID), - RPCType: metrics.NormalizeRPCType(actualRPCType.String()), - } - if aggregates[aggKey] == nil { - aggregates[aggKey] = &aggregator{} - } - aggregates[aggKey].Total += score.Value - aggregates[aggKey].Count++ - } - } - } - - entries := make([]metrics.SupplierScoreEntry, 0, len(aggregates)) - for k, agg := range aggregates { - if agg.Count == 0 { - continue - } - entries = append(entries, metrics.SupplierScoreEntry{ - Supplier: k.Supplier, - ServiceID: k.ServiceID, - RPCType: k.RPCType, - Score: agg.Total / float64(agg.Count), - }) - } - - logger.Debug().Int("total_entries", len(entries)).Msg("Built supplier score data") - return entries, nil -} +// The per-operator equivalent is still published: see GetMeanScoreData above, +// which feeds path_reputation_mean_score{domain, service_id, rpc_type}. // GetCooldownCountData implements the metrics.LeaderboardDataProvider interface. // It returns per-(domain, service_id, rpc_type) counts of endpoints currently in diff --git a/protocol/shannon/protocol.go b/protocol/shannon/protocol.go index b90126dd9..371a05b39 100644 --- a/protocol/shannon/protocol.go +++ b/protocol/shannon/protocol.go @@ -960,7 +960,7 @@ func (p *Protocol) BuildHTTPRequestContextForEndpoint( // allowing endpoints to recover from failures via successful health checks. // // Implements gateway.Protocol interface. -func (p *Protocol) ApplyHTTPObservations(observations *protocolobservations.Observations) error { +func (p *Protocol) ApplyHTTPObservations(observations *protocolobservations.Observations, isHealthCheck bool) error { // Sanity check the input if observations == nil || observations.GetShannon() == nil { p.logger.ProbabilisticDebugInfo(polylog.ProbabilisticDebugInfoProb).Msg("SHOULD RARELY HAPPEN: ApplyHTTPObservations called with nil input or nil Shannon observation list.") @@ -976,7 +976,7 @@ func (p *Protocol) ApplyHTTPObservations(observations *protocolobservations.Obse // Record reputation signals from observations. // This allows health check results (from hydrator) to update endpoint reputation scores. if p.reputationService != nil { - p.recordReputationSignalsFromObservations(shannonObservations) + p.recordReputationSignalsFromObservations(shannonObservations, isHealthCheck) } return nil @@ -1523,7 +1523,7 @@ func (p *Protocol) HydrateDisqualifiedEndpointsResponse(serviceID protocol.Servi // recordReputationSignalsFromObservations maps protocol observations to reputation signals. // This is called by ApplyHTTPObservations to update endpoint reputation scores based on // health check results from the hydrator or any other observation source. -func (p *Protocol) recordReputationSignalsFromObservations(shannonObservations []*protocolobservations.ShannonRequestObservations) { +func (p *Protocol) recordReputationSignalsFromObservations(shannonObservations []*protocolobservations.ShannonRequestObservations, isHealthCheck bool) { for _, observationSet := range shannonObservations { httpObservations := observationSet.GetHttpObservations() if httpObservations == nil { @@ -1533,7 +1533,7 @@ func (p *Protocol) recordReputationSignalsFromObservations(shannonObservations [ serviceID := protocol.ServiceID(observationSet.GetServiceId()) for _, endpointObs := range httpObservations.GetEndpointObservations() { - p.recordSignalFromObservation(serviceID, endpointObs) + p.recordSignalFromObservation(serviceID, endpointObs, isHealthCheck) } } } @@ -1541,7 +1541,7 @@ func (p *Protocol) recordReputationSignalsFromObservations(shannonObservations [ // recordSignalFromObservation records a reputation signal for a single endpoint observation. // It maps the observation's error type directly to a reputation signal and records it. // Also records probation traffic metrics if the endpoint is in probation. -func (p *Protocol) recordSignalFromObservation(serviceID protocol.ServiceID, obs *protocolobservations.ShannonEndpointObservation) { +func (p *Protocol) recordSignalFromObservation(serviceID protocol.ServiceID, obs *protocolobservations.ShannonEndpointObservation, isHealthCheck bool) { // Reconstruct the FULL endpoint address (-). // // Using the bare URL here silently broke every websocket health check: the reputation @@ -1594,6 +1594,13 @@ func (p *Protocol) recordSignalFromObservation(serviceID protocol.ServiceID, obs } } + // Mark probe-originated signals so the volume-independent rate detectors skip them. + // + // This is the SECOND path a health check reaches reputation by — the relay itself + // already recorded a signal through requestContext — so leaving it unstamped left + // half the probe volume feeding the rate EWMAs even after the relay path was fixed. + signal.IsHealthCheck = isHealthCheck + // Record signal (fire-and-forget, non-blocking) ctx := context.Background() if err := p.reputationService.RecordSignal(ctx, key, signal); err != nil { diff --git a/protocol/shannon/reputation_test.go b/protocol/shannon/reputation_test.go index c3513e488..bfc568534 100644 --- a/protocol/shannon/reputation_test.go +++ b/protocol/shannon/reputation_test.go @@ -1447,7 +1447,7 @@ func TestReputationWebSocketRecording_RPCType(t *testing.T) { } // Record the WebSocket observation - p.recordSignalFromWebsocketConnectionObservation(serviceID, obs) + p.recordSignalFromWebsocketConnectionObservation(serviceID, obs, false) // Give it a moment to process (fire-and-forget) time.Sleep(50 * time.Millisecond) diff --git a/protocol/shannon/websocket_context.go b/protocol/shannon/websocket_context.go index 22a515c57..2d0dbe5d3 100644 --- a/protocol/shannon/websocket_context.go +++ b/protocol/shannon/websocket_context.go @@ -1036,7 +1036,7 @@ func (p *Protocol) websocketReconnectScoreFunc( // allowing endpoints to recover from failures via successful health checks. // // Implements gateway.Protocol interface. -func (p *Protocol) ApplyWebSocketObservations(observations *protocolobservations.Observations) error { +func (p *Protocol) ApplyWebSocketObservations(observations *protocolobservations.Observations, isHealthCheck bool) error { // Sanity check the input if observations == nil || observations.GetShannon() == nil { p.logger.ProbabilisticDebugInfo(polylog.ProbabilisticDebugInfoProb).Msg("SHOULD RARELY HAPPEN: ApplyWebSocketObservations called with nil input or nil Shannon observation list.") @@ -1052,7 +1052,7 @@ func (p *Protocol) ApplyWebSocketObservations(observations *protocolobservations // Record reputation signals from observations. // This allows health check results (from hydrator) to update endpoint reputation scores. if p.reputationService != nil { - p.recordReputationSignalsFromWebsocketObservations(shannonObservations) + p.recordReputationSignalsFromWebsocketObservations(shannonObservations, isHealthCheck) } return nil @@ -1061,20 +1061,20 @@ func (p *Protocol) ApplyWebSocketObservations(observations *protocolobservations // recordReputationSignalsFromWebsocketObservations maps websocket protocol observations to reputation signals. // This is called by ApplyWebSocketObservations to update endpoint reputation scores based on // health check results from the hydrator or any other observation source. -func (p *Protocol) recordReputationSignalsFromWebsocketObservations(shannonObservations []*protocolobservations.ShannonRequestObservations) { +func (p *Protocol) recordReputationSignalsFromWebsocketObservations(shannonObservations []*protocolobservations.ShannonRequestObservations, isHealthCheck bool) { for _, observationSet := range shannonObservations { serviceID := protocol.ServiceID(observationSet.GetServiceId()) // Process connection observations connObs := observationSet.GetWebsocketConnectionObservation() if connObs != nil { - p.recordSignalFromWebsocketConnectionObservation(serviceID, connObs) + p.recordSignalFromWebsocketConnectionObservation(serviceID, connObs, isHealthCheck) } } } // recordSignalFromWebsocketConnectionObservation records a reputation signal for a websocket connection observation. -func (p *Protocol) recordSignalFromWebsocketConnectionObservation(serviceID protocol.ServiceID, obs *protocolobservations.ShannonWebsocketConnectionObservation) { +func (p *Protocol) recordSignalFromWebsocketConnectionObservation(serviceID protocol.ServiceID, obs *protocolobservations.ShannonWebsocketConnectionObservation, isHealthCheck bool) { // Reconstruct the FULL endpoint address (-). // // Using the bare URL here silently broke every websocket health check: the reputation @@ -1111,6 +1111,13 @@ func (p *Protocol) recordSignalFromWebsocketConnectionObservation(serviceID prot signal = errorTypeToSignal(errorType, 0) } + // Probe-originated signals are excluded from the volume-independent rate detectors, + // exactly as on the HTTP observation path. A websocket check's FAILURE reaches + // reputation only through here (a passing check emits no observation and is recorded + // directly by the executor, already stamped), so this is the only site that can carry + // the verdict for a failing websocket probe. + signal.IsHealthCheck = isHealthCheck + // Record signal (fire-and-forget, non-blocking) ctx := context.Background() if err := p.reputationService.RecordSignal(ctx, key, signal); err != nil { diff --git a/qos/evm/archival_ttl_test.go b/qos/evm/archival_ttl_test.go new file mode 100644 index 000000000..c75e09f5c --- /dev/null +++ b/qos/evm/archival_ttl_test.go @@ -0,0 +1,56 @@ +package evm + +import ( + "testing" + "time" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/gateway" + "github.com/pokt-network/path/protocol" + qostypes "github.com/pokt-network/path/qos/types" +) + +// Test_ArchivalTTL_UnverifiedPathDoesNotOutliveVerifiedPath pins the invariant that both +// sources of archival status share one lifetime, asserted through the production caller +// rather than on the constant. +// +// The two had drifted 16x apart, in the worst direction. The health-check path asserts an +// exact expected historical value from the rules file, so a node that ignores the block +// parameter and answers from current state FAILS it — and that verified mark expired in 30 +// minutes. The user-traffic path cannot assert a value, since the query is whatever a +// client happened to send, so it granted archival status on any successful archival-method +// call — and that UNVERIFIED mark lasted 8 hours. +// +// Measured in production 2026-08-20: four endpoints on one operator were marked archival +// while returning current state for every block asked, including one 256x past the chain +// tip. The health-check rule for the service they served had been deleted, leaving only +// the unverified 8h path to promote them. +// +// The old code carried a comment asserting the two values matched. They did not. A comment +// cannot hold this invariant; reading the stored expiry back can. +func Test_ArchivalTTL_UnverifiedPathDoesNotOutliveVerifiedPath(t *testing.T) { + qos := NewSimpleQoSInstance(polyzero.NewLogger(), protocol.ServiceID("eth")) + endpointAddr := protocol.EndpointAddr("pokt1supplier-https://archival.example.com/rpc") + + before := time.Now() + require.NoError(t, qos.UpdateFromExtractedData(endpointAddr, &qostypes.ExtractedData{ + ArchivalCheckPerformed: true, + IsArchival: true, + })) + + qos.endpointStore.endpointsMu.RLock() + stored := qos.endpointStore.endpoints[endpointAddr] + qos.endpointStore.endpointsMu.RUnlock() + + require.True(t, stored.checkArchival.isArchival, "precondition: the endpoint must have been promoted") + + granted := stored.checkArchival.expiresAt.Sub(before) + require.InDelta(t, gateway.ArchivalStatusTTL.Seconds(), granted.Seconds(), 30, + "a user-traffic archival confirmation must grant the SAME lifetime as a verified "+ + "health-check confirmation; weaker evidence must never outlive stronger evidence") + require.LessOrEqual(t, granted, time.Hour, + "an unverified archival mark lasting hours lets one lucky response hold an endpoint "+ + "in the archival pool long after it stopped being re-confirmed") +} diff --git a/qos/evm/endpoint_selection.go b/qos/evm/endpoint_selection.go index 4cd36d59a..d3e857c07 100644 --- a/qos/evm/endpoint_selection.go +++ b/qos/evm/endpoint_selection.go @@ -475,7 +475,12 @@ func (ss *serviceState) categorizeValidationFailure(err error) qosobservations.E // // Note: This function is lock-free - perceivedBlockNumber uses atomic operations. func (ss *serviceState) basicEndpointValidation(endpointAddr protocol.EndpointAddr, endpoint endpoint, requiresArchival bool) error { - supplier, _ := endpointAddr.GetAddress() + // Operator domain, not the supplier address: path_qos_filter_rejection_total is + // keyed on domain since 2026-08-12. With a supplier label it had the worst + // churn of any gateway metric (1,271 live series vs 24,708 distinct over one + // pod's 7.7h life) because rejections are sporadic per supplier while the + // supplier set rotates every session. Memoized — see DomainFromEndpointAddr. + domain := metrics.DomainFromEndpointAddr(string(endpointAddr)) serviceID := string(ss.serviceQoSConfig.GetServiceID()) // Check if the endpoint has returned an empty response within the timeout period. @@ -483,13 +488,13 @@ func (ss *serviceState) basicEndpointValidation(endpointAddr protocol.EndpointAd if endpoint.hasReturnedEmptyResponse && endpoint.invalidResponseLastObserved != nil { timeSinceEmptyResponse := time.Since(*endpoint.invalidResponseLastObserved) if timeSinceEmptyResponse < invalidResponseTimeout { - metrics.RecordQoSFilterRejection(supplier, serviceID, metrics.QoSFilterReasonEmptyResponse) + metrics.RecordQoSFilterRejection(domain, serviceID, metrics.QoSFilterReasonEmptyResponse) return fmt.Errorf("recent empty response validation failed (%.0f minutes ago): %w", timeSinceEmptyResponse.Minutes(), errEmptyResponseObs) } } else if endpoint.hasReturnedEmptyResponse { // Fallback for cases where hasReturnedEmptyResponse is true but invalidResponseLastObserved is nil - metrics.RecordQoSFilterRejection(supplier, serviceID, metrics.QoSFilterReasonEmptyResponse) + metrics.RecordQoSFilterRejection(domain, serviceID, metrics.QoSFilterReasonEmptyResponse) return fmt.Errorf("empty response validation failed: %w", errEmptyResponseObs) } @@ -497,7 +502,7 @@ func (ss *serviceState) basicEndpointValidation(endpointAddr protocol.EndpointAd if endpoint.hasReturnedInvalidResponse && endpoint.invalidResponseLastObserved != nil { timeSinceInvalidResponse := time.Since(*endpoint.invalidResponseLastObserved) if timeSinceInvalidResponse < invalidResponseTimeout { - metrics.RecordQoSFilterRejection(supplier, serviceID, metrics.QoSFilterReasonInvalidResponse) + metrics.RecordQoSFilterRejection(domain, serviceID, metrics.QoSFilterReasonInvalidResponse) return fmt.Errorf("recent invalid response validation failed (%.0f minutes ago): %w. Empty response: %t. Response validation error: %s", timeSinceInvalidResponse.Minutes(), errRecentInvalidResponseObs, endpoint.hasReturnedEmptyResponse, endpoint.invalidResponseError) } @@ -510,13 +515,13 @@ func (ss *serviceState) basicEndpointValidation(endpointAddr protocol.EndpointAd if errors.Is(err, errNoBlockNumberObs) { reason = metrics.QoSFilterReasonBlockHeightUnknown } - metrics.RecordQoSFilterRejection(supplier, serviceID, reason) + metrics.RecordQoSFilterRejection(domain, serviceID, reason) return fmt.Errorf("block number validation failed: %w", err) } // Check if the endpoint's EVM chain ID matches the expected chain ID. if err := ss.isChainIDValid(endpoint.checkChainID); err != nil { - metrics.RecordQoSFilterRejection(supplier, serviceID, metrics.QoSFilterReasonChainIDMismatch) + metrics.RecordQoSFilterRejection(domain, serviceID, metrics.QoSFilterReasonChainIDMismatch) return fmt.Errorf("chain ID validation failed: %w", err) } @@ -558,7 +563,7 @@ func (ss *serviceState) basicEndpointValidation(endpointAddr protocol.EndpointAd } // Both checks failed - return structured error with diagnostic details - metrics.RecordQoSFilterRejection(supplier, serviceID, metrics.QoSFilterReasonArchivalRequired) + metrics.RecordQoSFilterRejection(domain, serviceID, metrics.QoSFilterReasonArchivalRequired) return NewArchivalFilterError(string(endpointAddr), archivalDetails, errEndpointNotArchival) } diff --git a/qos/evm/extractor.go b/qos/evm/extractor.go index b878e1803..5570807e7 100644 --- a/qos/evm/extractor.go +++ b/qos/evm/extractor.go @@ -209,14 +209,38 @@ func (e *EVMDataExtractor) IsArchival(request []byte, response []byte) (bool, er errMsg := strings.ToLower(gjson.GetBytes(response, "error.message").String()) archivalErrorIndicators := []string{ "missing trie node", + // geth's path-based state scheme (PBSS) reports unavailable historical + // state as "metadata is not found, ". Without this entry the + // check fell through to the "some other error" branch below, returned + // an error rather than false, and the endpoint was never demoted out + // of the archival pool that had just failed it. + "metadata is not found", "pruned", "ancient block", "block not found", "header not found", "state not available", - "state histories", // "state histories haven't been fully indexed yet" - "not fully indexed", // catch variations - "historical data", // "historical data not available" + "state histories", // "state histories haven't been fully indexed yet" + "not fully indexed", // catch variations + "historical data", // "historical data not available" + // Measured against production 2026-08-20 by probing archival-marked + // endpoints with a deep historical block. Two live wordings landed here + // and BOTH missed every entry above by a single word: + // + // gnosis: "historical state is not available" -- "state not available" + // misses because the real text is "state IS not available", and + // "historical data" misses because it is "historical STATE". + // poly: "historical state " + // + // Both fell through to the "some other error" branch, which returns an + // error rather than false, so the endpoint was never demoted and kept + // receiving archival requests it cannot serve. Same failure the PBSS + // entry above fixed, on a different vendor's wording. + // + // The bare prefix covers both observed forms. It is also already present + // in qos/heuristic/indicators.go, where this error IS recognised — the two + // catalogues had drifted, and this realigns them. + "historical state", } for _, indicator := range archivalErrorIndicators { @@ -234,6 +258,16 @@ func (e *EVMDataExtractor) IsArchival(request []byte, response []byte) (bool, er // Check for result field resultField := gjson.GetBytes(response, "result") if resultField.Exists() && resultField.Type != gjson.Null { + // A success only proves archival capability when the request actually asked + // for historical state. Every method in archivalMethods is also the ordinary + // way to read CURRENT state -- eth_getBalance(addr, "latest") is among the + // most common calls on the network -- and a pruned node answers those + // perfectly. Treating any success as proof promoted pruned nodes into the + // archival pool, which then handed them the historical queries they cannot + // serve. + if !targetsHistoricalBlock(request, method) { + return false, fmt.Errorf("archival check inconclusive: %s did not target a historical block", method) + } // The archival query succeeded - endpoint is archival-capable return true, nil } @@ -241,6 +275,48 @@ func (e *EVMDataExtractor) IsArchival(request []byte, response []byte) (bool, er return false, fmt.Errorf("archival check response missing both result and error") } +// targetsHistoricalBlock reports whether the request's block parameter names a +// historical block rather than the chain tip. +// +// False for an omitted parameter (EVM defaults it to "latest"), for the +// latest/pending/safe/finalized tags, and for anything that is not a parseable +// block number -- a block hash, or an EIP-1898 object, carries no depth here. +// +// Known limitation: without the perceived chain tip this cannot separate a deep +// historical block from one a few blocks back, so a numeric parameter near the tip +// still reads as archival. The DataExtractor interface carries only +// (request, response), so the tip is not reachable from here. The tag case is the +// one that mattered in practice: it is the shape of nearly all current-state +// traffic, and it was the whole of the pollution. +func targetsHistoricalBlock(request []byte, method string) bool { + info, ok := evmMethodBlockParams[method] + if !ok { + return false + } + + path := "params." + strconv.Itoa(info.paramIndex) + if info.isObject { + path += "." + info.objectKey + } + + param := gjson.GetBytes(request, path) + if !param.Exists() { + // An omitted block parameter defaults to "latest". + return false + } + + blockParam := param.String() + switch BlockTag(strings.ToLower(blockParam)) { + case BlockTagEarliest: + return true + case BlockTagLatest, BlockTagPending, BlockTagSafe, BlockTagFinalized: + return false + } + + _, err := parseBlockNumber(blockParam) + return err == nil +} + // IsValidResponse checks if the response is a valid JSON-RPC 2.0 response. // This performs basic structural validation without extracting specific data. // diff --git a/qos/evm/extractor_archival_gate_test.go b/qos/evm/extractor_archival_gate_test.go new file mode 100644 index 000000000..12ff27cd8 --- /dev/null +++ b/qos/evm/extractor_archival_gate_test.go @@ -0,0 +1,111 @@ +package evm + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// IsArchival decides whether an endpoint enters the archival pool. Two independent +// defects let pruned nodes in, and both are reproduced here. +func Test_IsArchival_LatestTagIsNotProofOfArchival(t *testing.T) { + e := NewEVMDataExtractor() + + // eth_getBalance at "latest" is ordinary current-state traffic and a pruned node + // answers it perfectly. Treating the success as proof of archival capability + // promoted pruned nodes into the archival pool, which then handed them the + // historical queries they cannot serve. + request := []byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x56Eddb7aa87536c09CCc2793473599fD21A8b17F","latest"],"id":1}`) + response := []byte(`{"jsonrpc":"2.0","id":1,"result":"0x13570a9"}`) + + isArchival, err := e.IsArchival(request, response) + require.Error(t, err, "a success at \"latest\" must be inconclusive, not archival") + require.False(t, isArchival) +} + +func Test_IsArchival_CurrentStateTagsAndOmittedParam(t *testing.T) { + e := NewEVMDataExtractor() + response := []byte(`{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + + for _, req := range []string{ + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","pending"],"id":1}`, + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","safe"],"id":1}`, + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","finalized"],"id":1}`, + // Omitted block parameter defaults to "latest" in every EVM client. + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc"],"id":1}`, + // A block hash carries no depth. + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","0x4e3a3754410177e6937ef1f84bba68ea139e8d1a2258c5f85db9f1cd715a1bdd"],"id":1}`, + } { + isArchival, err := e.IsArchival([]byte(req), response) + require.Error(t, err, "request must be inconclusive: %s", req) + require.False(t, isArchival) + } +} + +func Test_IsArchival_HistoricalBlockStillProvesArchival(t *testing.T) { + e := NewEVMDataExtractor() + response := []byte(`{"jsonrpc":"2.0","id":1,"result":"0x0"}`) + + for _, req := range []string{ + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","0x13570a9"],"id":1}`, + `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","earliest"],"id":1}`, + `{"jsonrpc":"2.0","method":"eth_getStorageAt","params":["0xabc","0x0","0x1"],"id":1}`, + } { + isArchival, err := e.IsArchival([]byte(req), response) + require.NoError(t, err, "request must prove archival capability: %s", req) + require.True(t, isArchival) + } +} + +// Geth PBSS reports unavailable historical state as "metadata is not found, ". +// Unrecognised, it fell through to the "some other error" branch, which returns an +// error -- so the endpoint that had just failed an archival query was never demoted +// out of the archival pool and kept receiving them. +func Test_IsArchival_PBSSPrunedStateDemotes(t *testing.T) { + e := NewEVMDataExtractor() + + request := []byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x56Eddb7aa87536c09CCc2793473599fD21A8b17F","0x13570a9"],"id":338498041}`) + response := []byte(`{"jsonrpc":"2.0","id":338498041,"error":{"code":-32000,"message":"metadata is not found, 12114132"}}`) + + isArchival, err := e.IsArchival(request, response) + require.NoError(t, err, "PBSS pruned-state error must be a definitive not-archival result") + require.False(t, isArchival) +} + +// Test_IsArchival_HistoricalStateWordingsDemote covers two live prunedstate wordings +// measured in production on 2026-08-20 by sending a deep historical block to endpoints +// PATH had marked archival. +// +// Both missed every pattern in archivalErrorIndicators by a single word: "state not +// available" does not match "state IS not available", and "historical data" does not +// match "historical STATE". So both fell through to the "some other error" branch, which +// returns an error rather than false, and the endpoint stayed in the archival pool that +// had just failed it -- the exact failure the PBSS entry was added to fix, on a different +// vendor's wording. +// +// Table-driven because the discriminating detail is the exact string; a single case would +// pass on a pattern that only covers one of the two. +func Test_IsArchival_HistoricalStateWordingsDemote(t *testing.T) { + // A request that genuinely asks for historical state, so a false result can only come + // from the error classification and not from the targetsHistoricalBlock gate. + request := []byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x0000000000000000000000000000000000000000","0x1312D00"],"id":1}`) + + for _, tc := range []struct { + name string + message string + }{ + {"gnosis wording", "historical state is not available"}, + {"poly wording", "historical state 654f28d19b44239d1012f27038f1f"}, + } { + t.Run(tc.name, func(t *testing.T) { + e := NewEVMDataExtractor() + response := []byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"` + tc.message + `"}}`) + + isArchival, err := e.IsArchival(request, response) + require.NoError(t, err, + "an unavailable-historical-state error must be a DEFINITIVE not-archival result; "+ + "returning an error instead leaves the endpoint in the archival pool") + require.False(t, isArchival) + }) + } +} diff --git a/qos/evm/extractor_test.go b/qos/evm/extractor_test.go index 34945cb32..2bd47747b 100644 --- a/qos/evm/extractor_test.go +++ b/qos/evm/extractor_test.go @@ -221,7 +221,11 @@ func TestEVMDataExtractor_IsArchival(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - request := []byte(`{"jsonrpc":"2.0","method":"eth_getBalance","id":1}`) + // The block parameter is load-bearing: a success only proves archival + // capability when the request actually asked for historical state. + // This case previously omitted params entirely, which EVM clients + // default to "latest" — the shape a pruned node answers perfectly. + request := []byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0xabc","0x13570a9"],"id":1}`) isArchival, err := extractor.IsArchival(request, []byte(tt.response)) if tt.expectError { assert.Error(t, err) diff --git a/qos/evm/qos.go b/qos/evm/qos.go index db77dc5b6..7479d746c 100644 --- a/qos/evm/qos.go +++ b/qos/evm/qos.go @@ -199,8 +199,11 @@ func (qos *QoS) UpdateFromExtractedData(endpointAddr protocol.EndpointAddr, data // - Health checks validate archival capability via eth_getBlockByNumber for ancient blocks // - User requests can confirm (IsArchival=true) or invalidate (IsArchival=false) archival status if data.ArchivalCheckPerformed { - // Default TTL for archival status (8 hours - matches health check archival TTL) - archivalTTL := 8 * time.Hour + // Shared with the health-check path so the two sources cannot drift; see + // gateway.ArchivalStatusTTL for why that matters. This path is the WEAKER of the + // two — it cannot assert an expected value, only that some archival-method call + // succeeded — so it must not grant a longer-lived mark than the verified one. + archivalTTL := gateway.ArchivalStatusTTL expiresAt := time.Now().Add(archivalTTL) storedEndpoint.checkArchival = endpointCheckArchival{ @@ -670,9 +673,9 @@ func (qos *QoS) StartBackgroundSync(ctx context.Context, syncInterval time.Durat // IMPORTANT: Performs immediate refresh on startup to ensure cache is warm before serving requests. // // Recommended startup sequence for full cross-replica sync: -// 1. qos.SetReputationService(svc) -// 2. qos.StartBackgroundSync(ctx, 5*time.Second) // Perceived block number -// 3. qos.StartArchivalCacheRefreshWorker(ctx, 2*time.Hour) // Archival status +// 1. qos.SetReputationService(svc) +// 2. qos.StartBackgroundSync(ctx, 5*time.Second) // Perceived block number +// 3. qos.StartArchivalCacheRefreshWorker(ctx, 2*time.Hour) // Archival status func (qos *QoS) StartArchivalCacheRefreshWorker(ctx context.Context, refreshInterval time.Duration) { if qos.reputationSvc == nil { qos.logger.Warn().Msg("Cannot start archival cache refresh: reputation service not set") @@ -736,8 +739,8 @@ func (qos *QoS) StartArchivalCacheRefreshWorker(ctx context.Context, refreshInte func (qos *QoS) refreshArchivalCacheFromRedis(parentCtx context.Context) { serviceID := qos.serviceQoSConfig.GetServiceID() - // Use 8-hour TTL matching archival status expiry - const archivalTTL = 8 * time.Hour + // Matches archival status expiry; see gateway.ArchivalStatusTTL. + const archivalTTL = gateway.ArchivalStatusTTL var refreshed, failed int diff --git a/qos/evm/qos_filter_rejection_label_test.go b/qos/evm/qos_filter_rejection_label_test.go new file mode 100644 index 000000000..c9f88aa93 --- /dev/null +++ b/qos/evm/qos_filter_rejection_label_test.go @@ -0,0 +1,81 @@ +package evm + +import ( + "testing" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/metrics" + "github.com/pokt-network/path/protocol" +) + +// Test_BasicEndpointValidation_RejectionKeysOnDomain asserts the CALL SITE, not +// the metric. +// +// path_qos_filter_rejection_total was re-keyed from `supplier` to `domain` on +// 2026-08-12 (1,271 series live in a 10-minute window vs 24,708 distinct minted +// over one pod's 7.7h life — 19.4× churn, the worst of any gateway metric). The +// re-key compiles silently if a caller keeps passing the supplier address: both +// parameters are plain strings. +// +// So the label NAME being `domain` proves nothing. This asserts on the label +// VALUE the production caller emits: an operator domain, never the +// DomainSupplierAddr sentinel that SanitizeDomainLabel produces from a bech32 +// address. That distinction is the entire fix — the metric's whole purpose is to +// collapse many suppliers onto one operator. +func Test_BasicEndpointValidation_RejectionKeysOnDomain(t *testing.T) { + metrics.QoSFilterRejectionTotal.Reset() + + const ( + supplier = "pokt1ylsjqcl0yunve78etutw660a327avc26fxrlfr" + // Two endpoints, same operator, different suppliers and subdomains: the + // pair that must collapse onto ONE series. + addrA = protocol.EndpointAddr(supplier + "-https://relayminer.eu.operator-under-test.example") + addrB = protocol.EndpointAddr("pokt1othersupplieraddresshere0000000000000-https://other.us.operator-under-test.example") + ) + + ss := &serviceState{ + logger: polyzero.NewLogger(), + serviceQoSConfig: NewEVMServiceQoSConfig("test-service", "1", nil), + } + + // An endpoint with no block-number observation is rejected with + // QoSFilterReasonBlockHeightUnknown, which is the shortest production path + // into RecordQoSFilterRejection. + require.Error(t, ss.basicEndpointValidation(addrA, endpoint{}, false)) + require.Error(t, ss.basicEndpointValidation(addrB, endpoint{}, false)) + + domains := emittedLabelValues(t, metrics.QoSFilterRejectionTotal, "domain") + + require.NotContains(t, domains, supplier, + "the call site is passing the supplier address where a domain is expected") + require.NotContains(t, domains, metrics.DomainSupplierAddr, + "the call site passed a bech32 address; the sanitizer caught it, but the label is "+ + "now a sentinel instead of the operator it is supposed to name") + require.Equal(t, map[string]struct{}{"operator-under-test.example": {}}, domains, + "two suppliers of one operator must collapse onto exactly one domain series") +} + +// emittedLabelValues reads back the values the Prometheus collector actually +// received for one label, across every child series of a vec. +func emittedLabelValues(t *testing.T, c prometheus.Collector, labelName string) map[string]struct{} { + t.Helper() + ch := make(chan prometheus.Metric, 1<<12) + c.Collect(ch) + close(ch) + + out := map[string]struct{}{} + for m := range ch { + var pb dto.Metric + require.NoError(t, m.Write(&pb)) + for _, lp := range pb.GetLabel() { + if lp.GetName() == labelName { + out[lp.GetValue()] = struct{}{} + } + } + } + return out +} diff --git a/qos/heuristic/analyzer.go b/qos/heuristic/analyzer.go index 12429259d..e9aecacc3 100644 --- a/qos/heuristic/analyzer.go +++ b/qos/heuristic/analyzer.go @@ -2,6 +2,7 @@ package heuristic import ( "bytes" + "fmt" sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) @@ -45,6 +46,21 @@ func (ra *ResponseAnalyzer) Analyze(responseBytes []byte, httpStatusCode int, rp return statusResult } + // A body-less status carries no body by definition, so an empty payload is the + // correct response rather than a fault. Without this, 204/205/304 were flagged + // as empty_response — harmless while that signal was MINOR, but it becomes a + // CRITICAL reputation penalty for correct behaviour once it is weighted as the + // protocol violation it is on a body-bearing status. + if len(responseBytes) == 0 && isBodylessHTTPStatus(httpStatusCode) { + return AnalysisResult{ + ShouldRetry: false, + Confidence: 0.0, + Reason: "no_body_expected", + Structure: StructureValid, + Details: fmt.Sprintf("HTTP %d carries no body; empty payload is correct", httpStatusCode), + } + } + // Level 1: Structural Analysis structResult := StructuralAnalysis(responseBytes) if structResult.ShouldRetry { @@ -229,6 +245,21 @@ func (ra *ResponseAnalyzer) ShouldRetry(responseBytes []byte, httpStatusCode int return result.ShouldRetry } +// isBodylessHTTPStatus reports whether a status code is defined to carry no message +// body, making an empty payload the correct response rather than a supplier fault. +// - 204 No Content, 205 Reset Content: RFC 9110 forbids a body. +// - 304 Not Modified: RFC 9110 forbids a body. +// +// 1xx responses never reach the analyzer (they are not final), so they are omitted. +func isBodylessHTTPStatus(statusCode int) bool { + switch statusCode { + case 204, 205, 304: + return true + default: + return false + } +} + // Package-level convenience functions using default analyzer var defaultAnalyzer = NewDefaultAnalyzer() diff --git a/qos/heuristic/analyzer_test.go b/qos/heuristic/analyzer_test.go index 773e9f99e..650e890e5 100644 --- a/qos/heuristic/analyzer_test.go +++ b/qos/heuristic/analyzer_test.go @@ -685,6 +685,30 @@ func TestProtocolAnalysis_MethodAwareEmptyArray(t *testing.T) { expectedRetry: false, expectedReason: "jsonrpc_success", }, + { + name: "getProgramAccounts + empty array = valid (no accounts match the filters)", + method: "getProgramAccounts", + expectedRetry: false, + expectedReason: "jsonrpc_success", + }, + { + name: "getInflationReward + empty array = valid", + method: "getInflationReward", + expectedRetry: false, + expectedReason: "jsonrpc_success", + }, + { + name: "getSlotLeaders + empty array = valid", + method: "getSlotLeaders", + expectedRetry: false, + expectedReason: "jsonrpc_success", + }, + { + name: "getConfirmedBlocksWithLimit + empty array = valid", + method: "getConfirmedBlocksWithLimit", + expectedRetry: false, + expectedReason: "jsonrpc_success", + }, } for _, tt := range tests { @@ -915,45 +939,45 @@ func TestRESTEmptyObjectPathWhitelist(t *testing.T) { emptyObject := []byte(`{}`) tests := []struct { - name string - path string - expectedRetry bool + name string + path string + expectedRetry bool expectedReason string }{ { - name: "No path — still flagged", - path: "", - expectedRetry: true, + name: "No path — still flagged", + path: "", + expectedRetry: true, expectedReason: "rest_empty_object", }, { - name: "Tron /wallet/getaccount — whitelisted", - path: "/wallet/getaccount", - expectedRetry: false, + name: "Tron /wallet/getaccount — whitelisted", + path: "/wallet/getaccount", + expectedRetry: false, expectedReason: "rest_no_error_indicator", }, { - name: "Tron /wallet/gettransactionbyid — whitelisted", - path: "/wallet/gettransactionbyid", - expectedRetry: false, + name: "Tron /wallet/gettransactionbyid — whitelisted", + path: "/wallet/gettransactionbyid", + expectedRetry: false, expectedReason: "rest_no_error_indicator", }, { - name: "Tron /walletsolidity/getaccount — whitelisted", - path: "/walletsolidity/getaccount", - expectedRetry: false, + name: "Tron /walletsolidity/getaccount — whitelisted", + path: "/walletsolidity/getaccount", + expectedRetry: false, expectedReason: "rest_no_error_indicator", }, { - name: "Cosmos REST path — whitelisted", - path: "/cosmos/base/tendermint/v1beta1/blocks/latest", - expectedRetry: false, + name: "Cosmos REST path — whitelisted", + path: "/cosmos/base/tendermint/v1beta1/blocks/latest", + expectedRetry: false, expectedReason: "rest_no_error_indicator", }, { - name: "Root path — not whitelisted", - path: "/", - expectedRetry: true, + name: "Root path — not whitelisted", + path: "/", + expectedRetry: true, expectedReason: "rest_empty_object", }, } @@ -1453,24 +1477,24 @@ func BenchmarkAnalyze_LargeResponse(b *testing.B) { func TestCheckRequestIDMismatch(t *testing.T) { tests := []struct { - name string - response []byte - requestID string - expectFlag bool + name string + response []byte + requestID string + expectFlag bool expectReason string }{ { - name: "ID mismatch — response null, request had integer ID", - response: []byte(`{"jsonrpc":"2.0","id":null,"error":{"code":-32700,"message":"parse error"}}`), - requestID: "1", - expectFlag: true, + name: "ID mismatch — response null, request had integer ID", + response: []byte(`{"jsonrpc":"2.0","id":null,"error":{"code":-32700,"message":"parse error"}}`), + requestID: "1", + expectFlag: true, expectReason: "jsonrpc_id_mismatch", }, { - name: "ID mismatch — response null, request had string ID", - response: []byte(`{"jsonrpc":"2.0","id":null,"error":{"code":-32603,"message":"service unavailable"}}`), - requestID: `"abc"`, - expectFlag: true, + name: "ID mismatch — response null, request had string ID", + response: []byte(`{"jsonrpc":"2.0","id":null,"error":{"code":-32603,"message":"service unavailable"}}`), + requestID: `"abc"`, + expectFlag: true, expectReason: "jsonrpc_id_mismatch", }, { diff --git a/qos/heuristic/bodyless_status_test.go b/qos/heuristic/bodyless_status_test.go new file mode 100644 index 000000000..2826fbe10 --- /dev/null +++ b/qos/heuristic/bodyless_status_test.go @@ -0,0 +1,61 @@ +package heuristic + +import ( + "testing" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/assert" +) + +// An empty body is correct on a status defined to carry none. Without this exemption +// 204/205/304 were reported as empty_response — harmless while that signal was MINOR, +// but a CRITICAL reputation penalty for correct behaviour once empty_response is +// weighted as the protocol violation it is on a body-bearing status. +func TestAnalyze_EmptyBodyOnBodylessStatusIsNotAFault(t *testing.T) { + for _, code := range []int{204, 205, 304} { + t.Run(http(code), func(t *testing.T) { + result := Analyze([]byte{}, code, sharedtypes.RPCType_JSON_RPC, "getSlot") + + assert.False(t, result.ShouldRetry, "HTTP %d carries no body; empty is correct", code) + assert.Equal(t, "no_body_expected", result.Reason) + }) + } +} + +// The exemption must not swallow the real case: on a body-bearing 2xx an empty payload +// is still a fault, and it is the one this whole change exists to score correctly. +func TestAnalyze_EmptyBodyOnBodyBearingStatusIsStillAFault(t *testing.T) { + for _, code := range []int{200, 201, 202} { + t.Run(http(code), func(t *testing.T) { + result := Analyze([]byte{}, code, sharedtypes.RPCType_JSON_RPC, "getTokenAccountsByOwner") + + assert.True(t, result.ShouldRetry, "HTTP %d promises a body; empty is a violation", code) + assert.Equal(t, "empty_response", result.Reason) + }) + } +} + +// A populated body on a bodyless status is not something the exemption should hide — +// the guard is keyed on the body actually being empty, not on the status alone. +func TestAnalyze_BodylessStatusWithBodyIsNotExempted(t *testing.T) { + result := Analyze([]byte(`{"jsonrpc":"2.0","result":1,"id":1}`), 204, sharedtypes.RPCType_JSON_RPC, "getSlot") + assert.NotEqual(t, "no_body_expected", result.Reason, "exemption must require an empty body") +} + +func http(code int) string { + switch code { + case 200: + return "200_OK" + case 201: + return "201_Created" + case 202: + return "202_Accepted" + case 204: + return "204_NoContent" + case 205: + return "205_ResetContent" + case 304: + return "304_NotModified" + } + return "other" +} diff --git a/qos/heuristic/empty_array_solana_test.go b/qos/heuristic/empty_array_solana_test.go new file mode 100644 index 000000000..b49741acf --- /dev/null +++ b/qos/heuristic/empty_array_solana_test.go @@ -0,0 +1,112 @@ +package heuristic + +import ( + "fmt" + "testing" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/assert" +) + +// Regression tests for the 2026-08-19 solana incident. +// +// getProgramAccounts returns a top-level array and legitimately returns [] whenever +// its filters match nothing — routine with dataSize/memcmp filters. It was missing +// from emptyArrayValidMethods, so every correct empty response was classified as a +// broken supplier, retried twice, and returned to the client as a 500. +// +// The tell in production was that all five solana operators failed at once at nearly +// identical rates (72-81%) on both gateway builds, while health checks — which never +// call getProgramAccounts — kept succeeding. Independent operators do not fail +// identically at the same second; the common factor was this classifier. +// +// These assert through Analyze, the function the four production call sites use +// (protocol/shannon/context.go, gateway/hedge.go, gateway/health_check_executor.go, +// gateway/http_request_context_handle_request.go), not through ProtocolAnalysis. +func TestAnalyze_SolanaArrayReturningMethods_EmptyArrayIsSuccess(t *testing.T) { + // Captured verbatim from a mainnet gateway error log. + emptyArray := []byte(`{"jsonrpc":"2.0","result":[],"id":"a50875a8-38d3-467a-8758-072f0271b49e"}`) + + methods := []struct { + method string + why string + }{ + {"getProgramAccounts", "array of {pubkey, account}; [] when the filters match nothing"}, + {"getInflationReward", "array of reward objects, entries may be null"}, + {"getSlotLeaders", "array of validator pubkeys"}, + {"getConfirmedBlocksWithLimit", "deprecated alias of getBlocksWithLimit, array of slots"}, + } + + for _, tc := range methods { + t.Run(tc.method, func(t *testing.T) { + result := Analyze(emptyArray, 200, sharedtypes.RPCType_JSON_RPC, tc.method) + + assert.False(t, result.ShouldRetry, + "%s returns %s — an empty array is a valid success, not a supplier fault", tc.method, tc.why) + assert.Equal(t, "jsonrpc_success", result.Reason) + }) + } +} + +// TestAnalyze_EmptyArrayStillFlaggedForNonArrayMethods proves the fix did not blanket +// disable the detection. These methods never return a top-level array, so "result":[] +// from them really is a broken or canned response and must still be retried. +// +// getMultipleAccounts and getTokenAccountsByOwner are here deliberately: both appeared +// in the production logs alongside getProgramAccounts, but they wrap their array in +// {context, value}, so a bare "result":[] from them is genuinely malformed. Adding them +// to the allowlist would have blinded a true detection. +func TestAnalyze_EmptyArrayStillFlaggedForNonArrayMethods(t *testing.T) { + emptyArray := []byte(`{"jsonrpc":"2.0","result":[],"id":1}`) + + for _, method := range []string{ + "getSlot", + "getBalance", + "getLatestBlockhash", + "getMultipleAccounts", + "getTokenAccountsByOwner", + "eth_blockNumber", + "eth_getBalance", + } { + t.Run(method, func(t *testing.T) { + result := Analyze(emptyArray, 200, sharedtypes.RPCType_JSON_RPC, method) + + assert.True(t, result.ShouldRetry, + "%s never returns a top-level array — empty array must stay a detection", method) + assert.Equal(t, "jsonrpc_invalid_empty_array", result.Reason) + }) + } +} + +// TestAnalyze_PopulatedResultsAreSuccess guards the ordinary path: a non-empty result +// was never affected by the bug, and must not be affected by the fix either. +func TestAnalyze_PopulatedResultsAreSuccess(t *testing.T) { + cases := map[string][]byte{ + "getProgramAccounts": []byte(`{"jsonrpc":"2.0","result":[{"pubkey":"5tzF...","account":{"lamports":1}}],"id":1}`), + "getSlot": []byte(`{"jsonrpc":"2.0","result":361234567,"id":1}`), + } + + for method, body := range cases { + t.Run(method, func(t *testing.T) { + result := Analyze(body, 200, sharedtypes.RPCType_JSON_RPC, method) + + assert.False(t, result.ShouldRetry, "populated %s result must not be flagged", method) + assert.Equal(t, "jsonrpc_success", result.Reason) + }) + } +} + +// TestEmptyArrayValidMethods_CoversObservedSolanaTraffic pins the allowlist entries the +// incident added, so a future edit that drops one fails here with the reason attached +// rather than silently re-opening the outage. +func TestEmptyArrayValidMethods_CoversObservedSolanaTraffic(t *testing.T) { + for _, method := range []string{ + "getProgramAccounts", + "getInflationReward", + "getSlotLeaders", + "getConfirmedBlocksWithLimit", + } { + assert.True(t, emptyArrayValidMethods[method], + fmt.Sprintf("%s returns a top-level array; removing it re-opens the 2026-08-19 solana incident", method)) + } +} diff --git a/qos/heuristic/indicators.go b/qos/heuristic/indicators.go index fbf80acde..466f3af4d 100644 --- a/qos/heuristic/indicators.go +++ b/qos/heuristic/indicators.go @@ -157,15 +157,17 @@ var errorPatterns = []errorPattern{ // Blockchain-Specific Errors (EVM) // ONLY include errors that indicate supplier/node problems, NOT application-level errors - {[]byte("mdbx_panic"), CategoryBlockchainError, 0.98}, // Erigon MDBX database corruption/disk full - {[]byte("missing trie node"), CategoryBlockchainError, 0.95}, // Data corruption/sync issue - {[]byte("failed to call fallback"), CategoryBlockchainError, 0.95}, // Node's internal fallback for archival data failed - {[]byte("state has been pruned"), CategoryBlockchainError, 0.95}, // Archival data not available - {[]byte("is pruned"), CategoryBlockchainError, 0.95}, // Generic pruned error (e.g., "state at block #X is pruned") - {[]byte("state not available"), CategoryBlockchainError, 0.90}, // Node sync issue - {[]byte("haven't been fully indexed"), CategoryBlockchainError, 0.95}, // Archival indexing not complete (BSC) - {[]byte("not been fully indexed"), CategoryBlockchainError, 0.95}, // Archival indexing not complete (variant) - {[]byte("historical state"), CategoryBlockchainError, 0.85}, // Historical state not available + {[]byte("mdbx_panic"), CategoryBlockchainError, 0.98}, // Erigon MDBX database corruption/disk full + {[]byte("missing trie node"), CategoryBlockchainError, 0.95}, // Data corruption/sync issue + {[]byte("metadata is not found"), CategoryBlockchainError, 0.95}, // geth PBSS pruned state: "metadata is not found, " + {[]byte("excluded from account secondary indexes"), CategoryBlockchainError, 0.95}, // Solana -32010: node has no secondary index for this key (config, not fault) + {[]byte("failed to call fallback"), CategoryBlockchainError, 0.95}, // Node's internal fallback for archival data failed + {[]byte("state has been pruned"), CategoryBlockchainError, 0.95}, // Archival data not available + {[]byte("is pruned"), CategoryBlockchainError, 0.95}, // Generic pruned error (e.g., "state at block #X is pruned") + {[]byte("state not available"), CategoryBlockchainError, 0.90}, // Node sync issue + {[]byte("haven't been fully indexed"), CategoryBlockchainError, 0.95}, // Archival indexing not complete (BSC) + {[]byte("not been fully indexed"), CategoryBlockchainError, 0.95}, // Archival indexing not complete (variant) + {[]byte("historical state"), CategoryBlockchainError, 0.85}, // Historical state not available // Blockchain-Specific Errors (Solana) {[]byte("node is behind"), CategoryBlockchainError, 0.90}, // Sync issue @@ -273,6 +275,11 @@ func IsArchivalRelatedError(pattern string) bool { "haven't been fully indexed", "not been fully indexed", "missing trie node", + // geth's path-based state scheme (PBSS) reports unavailable historical + // state as "metadata is not found, ". The trie-node and pruned + // wordings above are all hash-based-scheme (HBSS) messages, so a PBSS + // node's honest "I do not retain that state" matched nothing. + "metadata is not found", "block has been pruned", "height is not available": return true @@ -292,6 +299,13 @@ func IsCapabilityLimitationError(pattern string) bool { switch pattern { case "capability_limitation": // Tron lite fullnode, "api not supported" plain text responses return true + // Solana -32010 " excluded from account secondary indexes; this RPC method + // unavailable for key": the node was started without a secondary account index for + // this program, so getProgramAccounts for it cannot be served here while another + // operator serves it from its index. Node configuration, not a fault — and not + // archival either, so it is deliberately not in IsArchivalRelatedError. + case "excluded from account secondary indexes": + return true default: return false } @@ -388,6 +402,8 @@ var capabilityLimitationSubstrings = []string{ "haven't been fully indexed", "not been fully indexed", "missing trie node", + // geth PBSS wording; see IsArchivalRelatedError. + "metadata is not found", "block has been pruned", "height is not available", // CometBFT's real pruned-height message is "height %d is not available, lowest @@ -400,6 +416,8 @@ var capabilityLimitationSubstrings = []string{ // Capability limitation (e.g., Tron lite fullnodes) "lite fullnode", "api is not supported", + // Solana -32010 account-index exclusion; see IsCapabilityLimitationError. + "excluded from account secondary indexes", // rest_protocol_mismatch_error: heuristic-detected honest JSON-RPC error // returned to a REST-shaped request (supplier's backend doesn't speak REST). // The structured AnalysisResult is lost when this surfaces through the diff --git a/qos/heuristic/pbss_pruned_state_test.go b/qos/heuristic/pbss_pruned_state_test.go new file mode 100644 index 000000000..fd621c5e7 --- /dev/null +++ b/qos/heuristic/pbss_pruned_state_test.go @@ -0,0 +1,46 @@ +package heuristic + +import ( + "testing" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" +) + +// Geth's path-based state scheme (PBSS) reports unavailable historical state with +// wording none of the hash-based-scheme patterns match: +// +// {"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"metadata is not found, 12114132"}} +// +// Reported from production against eth_getBalance at block 0x13570a9 (20,279,465). +// Before this was recognised the analyzer classified it as jsonrpc_valid_error and +// returned ShouldRetry=false, so the request was never re-tried on an endpoint that +// actually retains the state and the client received the -32000 verbatim. +const pbssPrunedStateResponse = `{"jsonrpc":"2.0","id":338498041,"error":{"code":-32000,"message":"metadata is not found, 12114132"}}` + +func Test_PBSSPrunedState_IsRetried(t *testing.T) { + result := Analyze([]byte(pbssPrunedStateResponse), 200, sharedtypes.RPCType_JSON_RPC, "eth_getBalance") + + require.True(t, result.ShouldRetry, + "PBSS pruned-state error must retry on a different endpoint; got reason %q", result.Reason) + require.Equal(t, "metadata is not found", result.MatchedPattern) +} + +func Test_PBSSPrunedState_IsCapabilityLimitation(t *testing.T) { + result := Analyze([]byte(pbssPrunedStateResponse), 200, sharedtypes.RPCType_JSON_RPC, "eth_getBalance") + + // Retrying is only half the requirement. A node that honestly reports it does + // not retain historical state is capability-limited, not broken: circuit-breaking + // its whole domain for a capability mismatch is the death-spiral this guards. + require.True(t, IsArchivalRelatedError(result.MatchedPattern), + "pattern %q must be archival-related", result.MatchedPattern) + require.True(t, IsCapabilityLimitationError(result.MatchedPattern), + "pattern %q must be a capability limitation", result.MatchedPattern) +} + +// The structured AnalysisResult is lost on the hedge_failed path, where only the +// error string survives; that fallback must recognise the wording too. +func Test_PBSSPrunedState_SubstringFallback(t *testing.T) { + require.True(t, ErrorContainsArchivalPattern( + `relay failed: {"code":-32000,"message":"Metadata is not found, 12114132"}`)) +} diff --git a/qos/heuristic/protocol.go b/qos/heuristic/protocol.go index bb99091b6..c82987aff 100644 --- a/qos/heuristic/protocol.go +++ b/qos/heuristic/protocol.go @@ -154,11 +154,15 @@ var emptyArrayValidMethods = map[string]bool{ "getBlocks": true, // array of slot numbers in range "getBlocksWithLimit": true, // array of slot numbers "getConfirmedBlocks": true, // deprecated, same as getBlocks + "getConfirmedBlocksWithLimit": true, // deprecated, same as getBlocksWithLimit "getSignaturesForAddress": true, // array of signature info objects "getConfirmedSignaturesForAddress2": true, // deprecated, same as above "getRecentPerformanceSamples": true, // array of performance samples "getClusterNodes": true, // array of node info "getRecentPrioritizationFees": true, // array of fee objects + "getProgramAccounts": true, // array of {pubkey, account}; [] whenever the filters match nothing + "getInflationReward": true, // array of reward objects, entries may be null + "getSlotLeaders": true, // array of validator pubkeys } // Tier 2: Protocol-Specific Success Checks diff --git a/qos/heuristic/solana_account_index_test.go b/qos/heuristic/solana_account_index_test.go new file mode 100644 index 000000000..1afc3eeb3 --- /dev/null +++ b/qos/heuristic/solana_account_index_test.go @@ -0,0 +1,44 @@ +package heuristic + +import ( + "testing" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" +) + +// Solana's -32010 is "account index unavailable for this key": the node was started +// without a secondary account index for this program (or with it excluded), so it cannot +// serve getProgramAccounts for it. Observed in production, HTTP 200, signature-valid: +// +// {"jsonrpc":"2.0","error":{"code":-32010,"message":" excluded from account secondary indexes; this RPC method unavailable for key"},"id":1} +// +// Node configuration, not a fault — another operator serves the same call from an index. +// Before this was recognised the analyzer classified it as a generic JSON-RPC error, the +// request was retried (correct) and the domain was charged a circuit-breaker failure and a +// reputation penalty (wrong) on every poll of a query that one dapp sends continuously. +const solanaAccountIndexExcludedResponse = `{"jsonrpc":"2.0","error":{"code":-32010,"message":"7rAgHPLDc9NryZmNdeEzyDui6D9PHkvTxMjKhNSa7w3a excluded from account secondary indexes; this RPC method unavailable for key"},"id":1}` + +func Test_SolanaAccountIndexExcluded_IsRetried(t *testing.T) { + result := Analyze([]byte(solanaAccountIndexExcludedResponse), 200, sharedtypes.RPCType_JSON_RPC, "getProgramAccounts") + + require.True(t, result.ShouldRetry, + "an index-excluded key must retry on an endpoint that indexes it; got reason %q", result.Reason) + require.Equal(t, "excluded from account secondary indexes", result.MatchedPattern) +} + +func Test_SolanaAccountIndexExcluded_IsCapabilityLimitationNotArchival(t *testing.T) { + result := Analyze([]byte(solanaAccountIndexExcludedResponse), 200, sharedtypes.RPCType_JSON_RPC, "getProgramAccounts") + + require.True(t, IsCapabilityLimitationError(result.MatchedPattern), + "pattern %q must be a capability limitation: no circuit break, no reputation penalty", result.MatchedPattern) + require.False(t, IsArchivalRelatedError(result.MatchedPattern), + "an index exclusion is not an archival condition and must not route through the archival filters") +} + +// The structured AnalysisResult is lost on the hedge_failed path, where only the error +// string survives; that fallback must recognise the wording too. +func Test_SolanaAccountIndexExcluded_SubstringFallback(t *testing.T) { + require.True(t, ErrorContainsArchivalPattern( + `relay failed: {"code":-32010,"message":"Tokenkeg excluded from account secondary indexes; this RPC method unavailable for key"}`)) +} diff --git a/qos/solana/endpoint.go b/qos/solana/endpoint.go index 6bb8680fa..b147d92c1 100644 --- a/qos/solana/endpoint.go +++ b/qos/solana/endpoint.go @@ -10,6 +10,11 @@ import ( // Expected value of the `result` field to a `getHealth` request. const resultGetHealthOK = "ok" +// resultGetHealthSyncing is recorded when a getHealth probe reports the node is behind or +// unhealthy. Any value other than resultGetHealthOK fails validateBasic; this one names the +// reason instead of leaving the field empty, which would read as "never observed". +const resultGetHealthSyncing = "syncing" + const ( // TODO_TECHDEBT(@adshmh): Add sanctions mechanism for dishonest endpoints (e.g., using public RPCs). // The sanctions store will apply to all QoS packages via PR #253 (JUDGE framework). @@ -25,11 +30,9 @@ const ( // The errors below list all the possible basic validation errors on an endpoint. var ( - errNoGetHealthObs = fmt.Errorf("endpoint has not had an observation of its response to a %q request", methodGetHealth) errInvalidGetHealthObs = fmt.Errorf("endpoint responded incorrectly to a %q request, expected: %q", methodGetHealth, resultGetHealthOK) errNoGetEpochInfoObs = fmt.Errorf("endpoint has not had an observation of its response to a %q request", methodGetEpochInfo) errInvalidGetEpochInfoHeightZeroObs = fmt.Errorf("endpoint responded with blockHeight of 0 to a %q request, expected a blockHeight of > 0", methodGetEpochInfo) - errInvalidGetEpochInfoEpochZeroObs = fmt.Errorf("endpoint responded with epoch of 0 to a %q request, expected an epoch of > 0", methodGetEpochInfo) errRecentJSONRPCValidationError = fmt.Errorf("endpoint has recent JSON-RPC validation errors") ) @@ -62,10 +65,26 @@ func (e endpoint) validateBasic() error { } switch { - case e.SolanaGetHealthResponse == nil: - return errNoGetHealthObs - - case e.Result != resultGetHealthOK: + // A MISSING health observation is not a fault — it means the getHealth probe has not + // landed yet. Solana's two health-check probes (getHealth and getBlockHeight) arrive at + // different moments, so every endpoint sits in this state briefly after every restart and + // between check cycles. + // + // Rejecting it inverted the previous behaviour: before block heights were stored at all, + // an unprobed endpoint was simply ABSENT from the store, and filterValidEndpoints waves + // absent endpoints through as fresh. Once the block-height probe began populating the + // store, that same endpoint became present-but-incomplete and was rejected — so learning + // MORE about an endpoint made it LESS selectable. Measured on canary 2026-08-18: + // rejections went from ~1.1k/s to ~12.6k/s and the selectable pool halved. + // + // Same principle as the Epoch case below: absence of a measurement is not evidence of + // badness. An observation that says the node is unhealthy still rejects, immediately + // below — that is a measurement, and it fails. + // + // The nil guard is load-bearing for the next case, not just for this one: Result is + // promoted from the embedded *SolanaGetHealthResponse, so reading it without the guard + // nil-dereferences. + case e.SolanaGetHealthResponse != nil && e.Result != resultGetHealthOK: return fmt.Errorf("❌Invalid solana health response: %s :%w", e.Result, errInvalidGetHealthObs) case e.SolanaGetEpochInfoResponse == nil: @@ -74,8 +93,15 @@ func (e endpoint) validateBasic() error { case e.BlockHeight == 0: return errInvalidGetEpochInfoHeightZeroObs - case e.Epoch == 0: - return errInvalidGetEpochInfoEpochZeroObs + // Epoch 0 is deliberately NOT fatal: it means "not observed", not "wrong". + // + // The only source of a real epoch is a getEpochInfo response from user traffic — the + // health-check path builds a SolanaGetEpochInfoResponse carrying just a block height, so + // its Epoch is 0 by construction. Treating that as invalid would re-create the trap this + // file's health-observation fix just closed: an endpoint kept out of selection for a field + // nothing routinely supplies, and therefore never given the traffic that would supply it. + // + // ValidateEndpoint skips the epoch comparison when Epoch is 0 for the same reason. default: return nil diff --git a/qos/solana/extractor.go b/qos/solana/extractor.go index 707c3d2bd..71d5713bd 100644 --- a/qos/solana/extractor.go +++ b/qos/solana/extractor.go @@ -71,7 +71,25 @@ func (e *SolanaDataExtractor) ExtractBlockHeight(request []byte, response []byte return blockHeight.Int(), nil } - return 0, fmt.Errorf("could not extract block height: getEpochInfo result missing numeric blockHeight field") + // getBlockHeight answers with a bare numeric result rather than an object. + // + // Gated on the request method, which is what separates this from the absoluteSlot fallback + // the comment above forbids: that one guessed at a field inside a getEpochInfo result and + // guessed the slot; this one reads the documented return value of a method whose entire + // purpose is to report block height. Accepting a bare number from ANY response would + // re-open the poisoning hole, since getSlot answers with a bare number too. + // + // This shape was previously unparseable, and getBlockHeight is one of the two probes + // solana's health checks actually run — so health checks contributed no block height at + // all, and every endpoint whose observations came only from health checks was rejected as + // never-observed. + if isMethod(request, methodGetBlockHeight) { + if result := gjson.GetBytes(response, "result"); result.Exists() && result.Type == gjson.Number { + return result.Int(), nil + } + } + + return 0, fmt.Errorf("could not extract block height: no numeric blockHeight in getEpochInfo result and not a %q response", methodGetBlockHeight) } // ExtractChainID extracts the cluster identifier from a Solana response. @@ -134,6 +152,17 @@ func (e *SolanaDataExtractor) ExtractChainID(request []byte, response []byte) (s // - false if endpoint is healthy (not syncing) // - Error if sync status cannot be determined func (e *SolanaDataExtractor) IsSyncing(request []byte, response []byte) (bool, error) { + // Only a getHealth response carries sync status. + // + // This gate is load-bearing in two directions. Without it every response was run through + // the "result == ok" test below, so a getBlockHeight response — whose result is a number — + // was reported as SYNCING. And with SyncCheckPerformed now derived from whether this + // returns an error, an ungated version would claim a health observation for responses that + // contain none, which is worse than having no observation at all. + if !isMethod(request, methodGetHealth) { + return false, fmt.Errorf("request is not a %q request: sync status not derivable", methodGetHealth) + } + // If getHealth returns an error, the node is unhealthy (possibly syncing) errorResult := gjson.GetBytes(response, "error") if errorResult.Exists() && errorResult.Type != gjson.Null { @@ -251,3 +280,12 @@ func (e *SolanaDataExtractor) IsValidResponse(request []byte, response []byte) ( return true, nil } + +// isMethod reports whether the JSON-RPC request body names the given method. +// +// The rest of this extractor identifies responses by shape alone, which works when the shapes +// are distinctive and fails when they are not — "ok" versus a bare number being the case that +// bit us. Where the method matters, read it. +func isMethod(request []byte, method jsonrpc.Method) bool { + return gjson.GetBytes(request, "method").String() == string(method) +} diff --git a/qos/solana/extractor_test.go b/qos/solana/extractor_test.go index 28fb80b80..f30dbd02f 100644 --- a/qos/solana/extractor_test.go +++ b/qos/solana/extractor_test.go @@ -147,40 +147,62 @@ func TestSolanaDataExtractor_ExtractChainID(t *testing.T) { func TestSolanaDataExtractor_IsSyncing(t *testing.T) { extractor := NewSolanaDataExtractor() + const getHealthReq = `{"jsonrpc":"2.0","id":1,"method":"getHealth"}` + tests := []struct { name string + request string response string expectedSync bool expectError bool }{ { name: "healthy node (not syncing)", + request: getHealthReq, response: `{"jsonrpc":"2.0","id":1,"result":"ok"}`, expectedSync: false, expectError: false, }, { name: "node behind (syncing)", + request: getHealthReq, response: `{"jsonrpc":"2.0","id":1,"error":{"code":-32005,"message":"Node is behind by 42 slots"}}`, expectedSync: true, expectError: false, }, { name: "unhealthy node", + request: getHealthReq, response: `{"jsonrpc":"2.0","id":1,"error":{"code":-32005,"message":"Node is unhealthy"}}`, expectedSync: true, expectError: false, }, { name: "other error", + request: getHealthReq, response: `{"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"Invalid params"}}`, expectError: true, }, + { + // The bug the method gate closes. getBlockHeight's result is a bare number, so + // the "result == ok" test reported every block-height response as SYNCING — and + // getBlockHeight is one of the two probes solana's health checks actually run. + name: "getBlockHeight response is not a sync signal", + request: `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + response: `{"jsonrpc":"2.0","id":1,"result":418160000}`, + expectError: true, + }, + { + name: "no request body: sync status is not derivable", + request: "", + response: `{"jsonrpc":"2.0","id":1,"result":"ok"}`, + expectError: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - isSyncing, err := extractor.IsSyncing(nil, []byte(tt.response)) + isSyncing, err := extractor.IsSyncing([]byte(tt.request), []byte(tt.response)) if tt.expectError { assert.Error(t, err) } else { diff --git a/qos/solana/health_observation_test.go b/qos/solana/health_observation_test.go new file mode 100644 index 000000000..15caa88ca --- /dev/null +++ b/qos/solana/health_observation_test.go @@ -0,0 +1,252 @@ +package solana + +import ( + "context" + "testing" + + "github.com/pokt-network/poktroll/pkg/polylog" + "github.com/stretchr/testify/require" + + "github.com/pokt-network/path/protocol" + qostypes "github.com/pokt-network/path/qos/types" +) + +const healthCheckedAddr = protocol.EndpointAddr("pokt1hc-https://a001.op-alpha.example") + +// newQoSForHealthTest builds a QoS whose endpoint store starts empty, the way a pod's does +// after a restart. Solana's store cannot be rebuilt from Redis (it needs live health and epoch +// data), so this is the state every deploy starts from. +func newQoSForHealthTest(t *testing.T) *QoS { + t.Helper() + logger := polylog.Ctx(context.Background()) + + serviceState := &ServiceState{logger: logger, serviceID: "solana"} + endpointStore := &EndpointStore{ + logger: logger, + serviceState: serviceState, + endpoints: map[protocol.EndpointAddr]endpoint{}, + } + return &QoS{logger: logger, ServiceState: serviceState, EndpointStore: endpointStore} +} + +// feedHealthCheck replays what the health-check pipeline produces: the executor hands the raw +// response to the extractor via ExtractedData.ExtractAll, then the result reaches QoS through +// UpdateFromExtractedData. Going through ExtractAll rather than hand-filling the struct is the +// point — the defect lived in which fields that pipeline populates. +func feedHealthCheck(t *testing.T, q *QoS, addr protocol.EndpointAddr, request, response string) { + t.Helper() + + data := qostypes.NewExtractedData(addr, 200, []byte(response), 0) + data.ExtractAll(NewSolanaDataExtractor(), []byte(request)) + require.NoError(t, q.UpdateFromExtractedData(addr, data)) +} + +// Test_HealthCheckAlone_MakesEndpointSelectable is the regression test for an endpoint that +// health checks could never rescue. +// +// Solana's validator requires a getHealth observation (errNoGetHealthObs). The health-check +// path wrote only a block height, so an endpoint whose observations came solely from health +// checks stayed permanently invalid — the configured getHealth probe ran, passed, and +// populated nothing the validator reads. Permanently invalid means no user traffic, and user +// traffic was the only other source of a health observation. +func Test_HealthCheckAlone_MakesEndpointSelectable(t *testing.T) { + q := newQoSForHealthTest(t) + + // Exactly the two probes configured for solana in pnf_path_rules.yaml. + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getHealth"}`, + `{"jsonrpc":"2.0","id":1,"result":"ok"}`) + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + + // Assert through the production selection caller. The endpoint IS in the store, so the + // "not found, treat as fresh" bypass cannot account for a pass here. + stored, found := q.endpoints[healthCheckedAddr] + require.True(t, found, "health checks must put the endpoint in the store") + require.NotNil(t, stored.SolanaGetHealthResponse, "health check must record a health observation") + + picked, err := q.SelectMultipleWithArchival(protocol.EndpointAddrList{healthCheckedAddr}, 1, false) + require.NoError(t, err) + require.Equal(t, protocol.EndpointAddrList{healthCheckedAddr}, picked) + + require.NoError(t, q.ValidateEndpoint(healthCheckedAddr, stored), + "an endpoint fed only by health checks must be valid") +} + +// Test_HealthCheckAlone_UnhealthyIsStillRejected guards the other direction: the fix must not +// turn "we now record health" into "everything is healthy". A node reporting it is behind has +// been observed, and observed-bad is not the same as observed-good. +func Test_HealthCheckAlone_UnhealthyIsStillRejected(t *testing.T) { + q := newQoSForHealthTest(t) + + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getHealth"}`, + `{"jsonrpc":"2.0","id":1,"error":{"code":-32005,"message":"Node is behind by 42 slots"}}`) + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + + stored := q.endpoints[healthCheckedAddr] + require.NotNil(t, stored.SolanaGetHealthResponse) + require.Equal(t, resultGetHealthSyncing, stored.Result) + require.Error(t, q.ValidateEndpoint(healthCheckedAddr, stored), + "an endpoint that reported itself behind must stay invalid") +} + +// Test_BlockHeightResponse_DoesNotForgeAHealthObservation is the narrow guard on the method +// gate in IsSyncing. getBlockHeight returns a bare number; the pre-gate code ran that through +// a `result == "ok"` test. With SyncCheckPerformed now derived from whether IsSyncing errors, +// an ungated version would mint a health observation out of a block-height response — claiming +// evidence nobody gathered, which is worse than having none. +func Test_BlockHeightResponse_DoesNotForgeAHealthObservation(t *testing.T) { + q := newQoSForHealthTest(t) + + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + + stored := q.endpoints[healthCheckedAddr] + require.Nil(t, stored.SolanaGetHealthResponse, + "a block-height response must not be recorded as a health observation") + require.Equal(t, uint64(418160000), stored.BlockHeight, "the block height must still be recorded") +} + +// Test_HealthOnlyObservation_DoesNotClobberBlockHeight covers the write ordering. A health-only +// observation carries block height 0; writing that would erase a real height locally and, via +// the per-endpoint Redis write, across every replica. +func Test_HealthOnlyObservation_DoesNotClobberBlockHeight(t *testing.T) { + q := newQoSForHealthTest(t) + + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getHealth"}`, + `{"jsonrpc":"2.0","id":1,"result":"ok"}`) + + require.Equal(t, uint64(418160000), q.endpoints[healthCheckedAddr].BlockHeight, + "a health-only observation must leave the block height untouched") +} + +// Test_UnobservedEpoch_DoesNotInvalidate covers the epoch half. +// +// Epoch is only ever set by a getEpochInfo response from user traffic — the health-check path +// builds a SolanaGetEpochInfoResponse carrying just a block height, leaving Epoch at 0. When 0 +// was treated as invalid it re-created the same trap: an endpoint benched for a field nothing +// routinely supplies, and therefore never given the traffic that would supply it. +func Test_UnobservedEpoch_DoesNotInvalidate(t *testing.T) { + q := newQoSForHealthTest(t) + q.perceivedEpoch = 1018 + + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getHealth"}`, + `{"jsonrpc":"2.0","id":1,"result":"ok"}`) + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + + stored := q.endpoints[healthCheckedAddr] + require.Zero(t, stored.Epoch, "health checks supply no epoch — this is the case under test") + require.NoError(t, q.ValidateEndpoint(healthCheckedAddr, stored), + "an unobserved epoch means 'not measured', never 'behind'") +} + +// Test_EpochLag_ToleratesOneEpoch covers the tolerance and its boundary. +// +// perceivedEpoch is a max over observations, so at a rollover whichever endpoint reports first +// puts every other endpoint an epoch behind through no fault of its own — the same +// max-versus-strict shape as the block height check. Two epochs behind is real staleness. +func Test_EpochLag_ToleratesOneEpoch(t *testing.T) { + for _, tc := range []struct { + name string + epoch uint64 + expectValid bool + }{ + {name: "current epoch", epoch: 1018, expectValid: true}, + {name: "one epoch behind (rollover skew)", epoch: 1017, expectValid: true}, + {name: "two epochs behind", epoch: 1016, expectValid: false}, + } { + t.Run(tc.name, func(t *testing.T) { + q := newQoSForHealthTest(t) + q.perceivedEpoch = 1018 + + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getHealth"}`, + `{"jsonrpc":"2.0","id":1,"result":"ok"}`) + // A real epoch only ever arrives via getEpochInfo from user traffic; health checks + // supply none. Feed one directly to exercise the comparison. + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getEpochInfo"}`, + `{"jsonrpc":"2.0","id":1,"result":{"blockHeight":418160000,"absoluteSlot":440100000,"epoch":1018}}`) + + stored := q.endpoints[healthCheckedAddr] + require.NotNil(t, stored.SolanaGetEpochInfoResponse) + stored.Epoch = tc.epoch + + err := q.ValidateEndpoint(healthCheckedAddr, stored) + if tc.expectValid { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +// Test_PartiallyProbedEndpoint_StaysSelectable is the regression test for a behaviour +// inversion introduced by the health-observation fix itself. +// +// Solana's health checks run two probes (getHealth and getBlockHeight) that land at +// different moments, so between them an endpoint holds a block height and no health +// observation. That intermediate state is not rare — every endpoint passes through it after +// every restart, and there are two of them per check cycle. +// +// Before the health-observation fix, ExtractBlockHeight could not parse a getBlockHeight +// response at all, so nothing was stored and the endpoint stayed ABSENT from the store — +// where filterValidEndpoints waves it through as fresh. After the fix the block height IS +// stored, which puts the endpoint IN the store, where a missing getHealth observation was +// fatal. Net effect: learning MORE about an endpoint made it LESS selectable. +// +// That is the same "absence of a measurement is not evidence of badness" mistake already +// fixed for Epoch in the same commit; it just was not carried across to the health field two +// lines above it. +func Test_PartiallyProbedEndpoint_StaysSelectable(t *testing.T) { + q := newQoSForHealthTest(t) + + // Only the block-height probe has landed. No getHealth observation yet. + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + + stored, found := q.endpoints[healthCheckedAddr] + require.True(t, found, "the block-height probe must have put the endpoint in the store") + require.Nil(t, stored.SolanaGetHealthResponse, "no health observation yet — the state under test") + + require.NoError(t, q.ValidateEndpoint(healthCheckedAddr, stored), + "an endpoint awaiting its first health probe must not be rejected: "+ + "unobserved is not unhealthy, and it was selectable before it entered the store") + + picked, err := q.SelectMultipleWithArchival(protocol.EndpointAddrList{healthCheckedAddr}, 1, false) + require.NoError(t, err) + require.Equal(t, protocol.EndpointAddrList{healthCheckedAddr}, picked) +} + +// Test_ObservedUnhealthy_IsStillRejected pins the other side of the boundary. Dropping the +// "no observation" rejection must NOT also drop the "observed bad" one — otherwise a node +// reporting itself behind becomes selectable, which is the opposite failure. +func Test_ObservedUnhealthy_IsStillRejected(t *testing.T) { + q := newQoSForHealthTest(t) + + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getBlockHeight"}`, + `{"jsonrpc":"2.0","id":1,"result":418160000}`) + feedHealthCheck(t, q, healthCheckedAddr, + `{"jsonrpc":"2.0","id":1,"method":"getHealth"}`, + `{"jsonrpc":"2.0","id":1,"error":{"code":-32005,"message":"Node is behind by 42 slots"}}`) + + stored := q.endpoints[healthCheckedAddr] + require.NotNil(t, stored.SolanaGetHealthResponse) + require.Error(t, q.ValidateEndpoint(healthCheckedAddr, stored), + "an endpoint that reported itself behind must stay rejected") +} diff --git a/qos/solana/methods.go b/qos/solana/methods.go index ead18f410..11059ca0c 100644 --- a/qos/solana/methods.go +++ b/qos/solana/methods.go @@ -9,6 +9,10 @@ const ( // Reference: https://docs.solana.com/developing/clients/jsonrpc-api#getepochinfo methodGetEpochInfo = jsonrpc.Method("getEpochInfo") + // methodGetBlockHeight is the JSON-RPC method for getting the block height directly. + // Its result is a bare number, unlike getEpochInfo's object — see ExtractBlockHeight. + methodGetBlockHeight = jsonrpc.Method("getBlockHeight") + // methodGetHealth is the JSON-RPC method for checking the health of the node. // Reference: https://docs.solana.com/developing/clients/jsonrpc-api#gethealth methodGetHealth = jsonrpc.Method("getHealth") diff --git a/qos/solana/solana.go b/qos/solana/solana.go index 18e93b616..4a5034f0d 100644 --- a/qos/solana/solana.go +++ b/qos/solana/solana.go @@ -113,7 +113,16 @@ func (q *QoS) UpdateFromExtractedData(endpointAddr protocol.EndpointAddr, data * hasBlock = true } - if !hasBlock { + // A getHealth observation is as load-bearing as a block height: ValidateEndpoint rejects + // any endpoint without one (errNoGetHealthObs), and this is the ONLY path by which a + // health check can supply it. + // + // Before this, health checks wrote block height and nothing else, so an endpoint whose + // observations came only from health checks sat in the store permanently invalid — the + // configured getHealth probe ran, passed, and populated nothing the validator reads. That + // is self-reinforcing: invalid means no user traffic, and user traffic was the only other + // source of a health observation. + if !hasBlock && !data.SyncCheckPerformed { return nil } @@ -129,12 +138,28 @@ func (q *QoS) UpdateFromExtractedData(endpointAddr protocol.EndpointAddr, data * storedEndpoint := q.endpoints[endpointAddr] + // Record the health observation when this response was a getHealth response. + // SyncCheckPerformed distinguishes "observed healthy" from "never asked" — without it the + // zero value would silently assert health nobody measured. + if data.SyncCheckPerformed { + result := resultGetHealthOK + if data.IsSyncing { + result = resultGetHealthSyncing + } + if storedEndpoint.SolanaGetHealthResponse == nil { + storedEndpoint.SolanaGetHealthResponse = &qosobservations.SolanaGetHealthResponse{} + } + storedEndpoint.Result = result + } + // Update the endpoint's block height observation (Solana uses block height from getEpochInfo) // Create or update the SolanaGetEpochInfoResponse with just the block height - if storedEndpoint.SolanaGetEpochInfoResponse == nil { - storedEndpoint.SolanaGetEpochInfoResponse = &qosobservations.SolanaGetEpochInfoResponse{} + if hasBlock { + if storedEndpoint.SolanaGetEpochInfoResponse == nil { + storedEndpoint.SolanaGetEpochInfoResponse = &qosobservations.SolanaGetEpochInfoResponse{} + } + storedEndpoint.BlockHeight = blockHeight } - storedEndpoint.BlockHeight = blockHeight // Store the updated endpoint back q.endpoints[endpointAddr] = storedEndpoint @@ -171,8 +196,10 @@ func (q *QoS) UpdateFromExtractedData(endpointAddr protocol.EndpointAddr, data * } } - // Write per-endpoint block height to Redis for cross-replica sync (async, non-blocking) - if q.reputationSvc != nil { + // Write per-endpoint block height to Redis for cross-replica sync (async, non-blocking). + // Guarded on hasBlock: a health-only observation carries blockHeight 0, and writing that + // would clobber a real height across every replica. + if hasBlock && q.reputationSvc != nil { go func(addr protocol.EndpointAddr, bn uint64, svcID protocol.ServiceID) { rCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() diff --git a/qos/solana/state.go b/qos/solana/state.go index cb0e3bb7d..e16e0525a 100644 --- a/qos/solana/state.go +++ b/qos/solana/state.go @@ -1,15 +1,45 @@ package solana import ( + "errors" "fmt" "sync" + "sync/atomic" "github.com/pokt-network/poktroll/pkg/polylog" + "github.com/pokt-network/path/metrics" "github.com/pokt-network/path/protocol" "github.com/pokt-network/path/qos" ) +// defaultSolanaBlockNumberSyncAllowance is the sync allowance used when configuration has +// not supplied one (startup, or external health-check rules failed to load). +// +// Unlike EVM and CosmosSDK — both of which default to 0 — Solana cannot default to a strict +// comparison. Solana produces a block roughly every 400ms, and perceivedBlockHeight is a MAX +// over endpoint observations, so with zero tolerance only the most recently observed endpoint +// can ever be valid: every other endpoint's newest observation is, by construction, older +// than the one that just raised the bar. +// +// That is not hypothetical. It locked Solana onto a single operator in production +// (2026-08-18): endpoints carrying user traffic refreshed their block height continuously and +// stayed valid, while endpoints refreshed only by health checks (~0.35/s per endpoint against +// ~2.5 blocks/s) sat permanently behind and were filtered out — which kept them from +// receiving the traffic that would have refreshed them. No health-check rate fixes that; it +// is a race re-lost every block. +// +// 1500 blocks ≈ 10 minutes of Solana, matching the sync_allowance configured for solana in +// pnf_path_rules.yaml, so an unloaded config behaves like a loaded one. +// +// Note that value was sized as a health-check gate, where being generous only risks probing +// a stale endpoint. It now also governs which endpoints are selectable, so tightening it is +// a routing change: lower it and endpoints leave the pool. Do not tune it below the margin +// the starvation loop needs — an endpoint refreshed only by health checks trails the +// perceived height by several blocks at all times, purely because something else reported +// more recently. +const defaultSolanaBlockNumberSyncAllowance = 1500 + // ServiceState keeps the expected current state of the Solana blockchain // based on the endpoints' responses to different requests. type ServiceState struct { @@ -27,25 +57,117 @@ type ServiceState struct { // Used by observations of Synthetic requests. chainID string serviceID protocol.ServiceID + + // syncAllowance is how many blocks an endpoint may trail perceivedBlockHeight and still + // be considered valid. 0 (the zero value) means "not configured" and falls back to + // defaultSolanaBlockNumberSyncAllowance — it does NOT disable the check, matching the + // CosmosSDK QoS. Set dynamically from configuration via QoS.SetSyncAllowance. + // + // Atomic rather than guarded by serviceStateLock: it is written by the health-check + // config refresh, not by the observation path, and ValidateEndpoint must not take a + // write lock to read it. + syncAllowance atomic.Uint64 +} + +// getSyncAllowance returns the configured block-height sync allowance, falling back to the +// Solana default when configuration has not supplied one. +func (s *ServiceState) getSyncAllowance() uint64 { + if v := s.syncAllowance.Load(); v > 0 { + return v + } + return defaultSolanaBlockNumberSyncAllowance +} + +// SetSyncAllowance dynamically updates the block-height sync allowance for this QoS instance. +// +// Called by the health check executor when external rules are loaded or refreshed, via the +// `interface{ SetSyncAllowance(uint64) }` assertion. Solana did not implement that interface +// before, so the `sync_allowance` configured for the service reached the health check's own +// sync check and was silently dropped on the endpoint-selection path. +// +// Promoted to the Solana QoS via its embedded *ServiceState. +func (s *ServiceState) SetSyncAllowance(syncAllowance uint64) { + s.syncAllowance.Store(syncAllowance) } // TODO_FUTURE: add an endpoint ranking method which can be used to assign a rank/score to a valid endpoint to guide endpoint selection. // // ValidateEndpoint returns an error if the supplied endpoint is not valid based on the perceived state of Solana blockchain. -func (s *ServiceState) ValidateEndpoint(endpoint endpoint) error { +func (s *ServiceState) ValidateEndpoint(endpointAddr protocol.EndpointAddr, endpoint endpoint) error { s.serviceStateLock.RLock() - defer s.serviceStateLock.RUnlock() + perceivedEpoch := s.perceivedEpoch + perceivedBlockHeight := s.perceivedBlockHeight + s.serviceStateLock.RUnlock() + + // Rejections are recorded lazily: this runs for every endpoint on every selection pass + // (~2000/s × the session's endpoint count on solana), and parsing the address out to a + // domain on the passing path would be pure waste. + // + // Operator domain, not the supplier address: path_qos_filter_rejection_total is keyed on + // domain because the supplier set rotates every session while rejections are sporadic per + // supplier — with a supplier label it had the worst churn of any gateway metric. + recordRejection := func(reason string) { + metrics.RecordQoSFilterRejection( + metrics.DomainFromEndpointAddr(string(endpointAddr)), + string(s.serviceID), + reason, + ) + } if err := endpoint.validateBasic(); err != nil { + // Split the reason so "we have never observed this endpoint" is distinguishable from + // "this endpoint answered badly" — the two call for opposite responses, and lumping + // them together is what made the pre-fix exclusions unreadable. + // One reason per distinct cause. An earlier version of this mapping folded several + // causes into a single block_height_unknown bucket, which meant telling them apart + // required inferring from the ABSENCE of a sibling series — the diagnosis that + // actually mattered rested on a negative. + // + // There is no health_unknown case any more: a missing health observation is no longer + // a rejection at all (see validateBasic). QoSFilterReasonHealthUnknown is kept in the + // metrics vocabulary because it earned its keep — splitting it out is what made the + // pool collapse legible within minutes of the deploy that caused it. + reason := metrics.QoSFilterReasonInvalidResponse + switch { + case errors.Is(err, errInvalidGetHealthObs): + reason = metrics.QoSFilterReasonUnhealthy + case errors.Is(err, errNoGetEpochInfoObs), errors.Is(err, errInvalidGetEpochInfoHeightZeroObs): + reason = metrics.QoSFilterReasonBlockHeightUnknown + case errors.Is(err, errRecentJSONRPCValidationError): + reason = metrics.QoSFilterReasonInvalidResponse + } + recordRejection(reason) return err } - if endpoint.Epoch < s.perceivedEpoch { - return fmt.Errorf("solana endpoint epoch is less than chain perceived epoch: %d < %d", endpoint.Epoch, s.perceivedEpoch) + // Epoch comparison, skipped when either side is unobserved. + // + // Epoch 0 means "never observed" — the health-check path supplies a block height but no + // epoch — and rejecting on it would bench endpoints for a field nothing routinely writes. + // + // One epoch of tolerance because this is the same max-versus-strict shape as the block + // height check: perceivedEpoch is raised by whichever endpoint reports first, and at a + // rollover every other endpoint is briefly an epoch behind through no fault of its own. + // Solana epochs last ~2.5 days, so this costs almost nothing and removes a cliff that + // would otherwise empty the pool for a few seconds every couple of days. + if endpoint.Epoch > 0 && perceivedEpoch > 0 && endpoint.Epoch+1 < perceivedEpoch { + recordRejection(metrics.QoSFilterReasonEpochLag) + return fmt.Errorf("solana endpoint epoch is more than one epoch behind chain perceived epoch: %d < %d", endpoint.Epoch, perceivedEpoch) } - if endpoint.BlockHeight < s.perceivedBlockHeight { - return fmt.Errorf("solana endpoint block height is less than chain perceived block height: %d < %d", endpoint.BlockHeight, s.perceivedBlockHeight) + // An endpoint may trail the perceived height by up to the sync allowance. + // + // A strict comparison here is unusable on Solana: perceivedBlockHeight is a MAX over + // observations of a chain producing ~2.5 blocks/s, so it is raised by whichever endpoint + // reported last, above every other endpoint's most recent report. See + // defaultSolanaBlockNumberSyncAllowance for what that cost in production. + minAllowedBlockHeight := qos.MinAllowedBlockNumber(perceivedBlockHeight, s.getSyncAllowance()) + if endpoint.BlockHeight < minAllowedBlockHeight { + recordRejection(metrics.QoSFilterReasonBlockHeightLag) + return fmt.Errorf( + "solana endpoint block height is outside the sync allowance: %d < %d (perceived %d, allowance %d)", + endpoint.BlockHeight, minAllowedBlockHeight, perceivedBlockHeight, s.getSyncAllowance(), + ) } return nil diff --git a/qos/solana/store.go b/qos/solana/store.go index f3d86e768..592d244e5 100644 --- a/qos/solana/store.go +++ b/qos/solana/store.go @@ -277,7 +277,7 @@ func (es *EndpointStore) filterValidEndpoints(allAvailableEndpoints protocol.End continue } - if err := es.serviceState.ValidateEndpoint(endpoint); err != nil { + if err := es.serviceState.ValidateEndpoint(availableEndpointAddr, endpoint); err != nil { logger.Warn().Err(err).Msgf("⚠️ SKIPPING endpoint because it failed validation: %s", availableEndpointAddr) continue } diff --git a/qos/solana/sync_allowance_test.go b/qos/solana/sync_allowance_test.go new file mode 100644 index 000000000..3c53b2c31 --- /dev/null +++ b/qos/solana/sync_allowance_test.go @@ -0,0 +1,198 @@ +package solana + +import ( + "context" + "testing" + + "github.com/pokt-network/poktroll/pkg/polylog" + "github.com/stretchr/testify/require" + + qosobservations "github.com/pokt-network/path/observation/qos" + "github.com/pokt-network/path/protocol" +) + +// Endpoint addresses shaped like production ones: -. The domain is what +// path_qos_filter_rejection_total is keyed on, and what an operator reads in a dashboard. +const ( + freshEndpointAddr = protocol.EndpointAddr("pokt1fresh-https://a001.op-alpha.example") + trailingEndpointAddr = protocol.EndpointAddr("pokt1trail-https://b001.op-beta.example") +) + +// makeValidatableStore builds a store whose endpoints all pass validateBasic, so the only +// thing separating them is how far their block height trails the perceived height. +// +// makeEndpointStore (store_test.go) deliberately omits the getHealth observation so every +// endpoint fails validateBasic — that is what the least-stale fallback tests need, and the +// opposite of what these tests need. +func makeValidatableStore(t *testing.T, perceived uint64, heights map[protocol.EndpointAddr]uint64) *EndpointStore { + t.Helper() + logger := polylog.Ctx(context.Background()) + + endpoints := make(map[protocol.EndpointAddr]endpoint, len(heights)) + for addr, h := range heights { + endpoints[addr] = endpoint{ + SolanaGetHealthResponse: &qosobservations.SolanaGetHealthResponse{ + Result: resultGetHealthOK, + }, + SolanaGetEpochInfoResponse: &qosobservations.SolanaGetEpochInfoResponse{ + BlockHeight: h, + Epoch: 1, + }, + } + } + + ss := &ServiceState{ + logger: logger, + serviceID: "solana", + perceivedBlockHeight: perceived, + perceivedEpoch: 1, + } + return &EndpointStore{ + logger: logger, + serviceState: ss, + endpoints: endpoints, + } +} + +// selectedSet runs the production selection caller and returns what it handed back. +// +// Asserting here rather than on filterValidEndpoints or on the ServiceState fields: the only +// question that matters is whether selection still returns the endpoint. Three drain bugs +// shipped with passing tests that asserted on a helper's own output. +func selectedSet(t *testing.T, es *EndpointStore, available protocol.EndpointAddrList) map[protocol.EndpointAddr]bool { + t.Helper() + picked, err := es.SelectMultipleWithArchival(available, uint(len(available)), false) + require.NoError(t, err) + + out := make(map[protocol.EndpointAddr]bool, len(picked)) + for _, addr := range picked { + out[addr] = true + } + return out +} + +// Test_SyncAllowance_TrailingEndpointStaysSelectable reproduces the production failure. +// +// perceivedBlockHeight is a MAX over observations. An endpoint carrying user traffic re-reports +// continuously and sits at the perceived height; an endpoint refreshed only by health checks +// trails it by a handful of blocks. Under the pre-fix strict comparison the trailing endpoint +// was filtered out, which denied it the traffic that would have refreshed it — Solana locked +// onto a single operator with the alternatives scoring 100 and receiving nothing. +func Test_SyncAllowance_TrailingEndpointStaysSelectable(t *testing.T) { + const perceived = uint64(418_160_000) + + es := makeValidatableStore(t, perceived, map[protocol.EndpointAddr]uint64{ + freshEndpointAddr: perceived, + // One block behind — the entire margin the old code needed to exclude it. + trailingEndpointAddr: perceived - 1, + }) + + selected := selectedSet(t, es, protocol.EndpointAddrList{freshEndpointAddr, trailingEndpointAddr}) + + require.True(t, selected[freshEndpointAddr], "endpoint at the perceived height must be selectable") + require.True(t, selected[trailingEndpointAddr], + "endpoint 1 block behind must be selectable: at ~2.5 blocks/s no endpoint can hold the max") +} + +// Test_SyncAllowance_BoundsHowFarAnEndpointMayTrail proves the allowance is genuinely consulted +// rather than the check having been removed. +// +// Without this the fix is indistinguishable from deleting the block-height filter, and a revert +// of the allowance plumbing would leave Test_SyncAllowance_TrailingEndpointStaysSelectable green. +func Test_SyncAllowance_BoundsHowFarAnEndpointMayTrail(t *testing.T) { + const perceived = uint64(418_160_000) + + es := makeValidatableStore(t, perceived, map[protocol.EndpointAddr]uint64{ + freshEndpointAddr: perceived, + trailingEndpointAddr: perceived - 10, + }) + es.serviceState.SetSyncAllowance(5) + + available := protocol.EndpointAddrList{freshEndpointAddr, trailingEndpointAddr} + selected := selectedSet(t, es, available) + + // The fresh endpoint survives, so the filtered set is non-empty and the least-stale + // fallback does NOT run. Absence below is therefore a real exclusion, not a fallback + // happening to rank the trailing endpoint last. + require.True(t, selected[freshEndpointAddr]) + require.False(t, selected[trailingEndpointAddr], + "10 blocks behind against an allowance of 5 must be excluded") +} + +// Test_SyncAllowance_DefaultExcludesGenuinelyStale confirms the default is a real bound and not +// an effective disable. +func Test_SyncAllowance_DefaultExcludesGenuinelyStale(t *testing.T) { + const perceived = uint64(418_160_000) + + es := makeValidatableStore(t, perceived, map[protocol.EndpointAddr]uint64{ + freshEndpointAddr: perceived, + // Well past defaultSolanaBlockNumberSyncAllowance — roughly 20 minutes behind. + trailingEndpointAddr: perceived - 3000, + }) + + selected := selectedSet(t, es, protocol.EndpointAddrList{freshEndpointAddr, trailingEndpointAddr}) + + require.True(t, selected[freshEndpointAddr]) + require.False(t, selected[trailingEndpointAddr], + "3000 blocks behind must still be excluded under the default allowance") +} + +// Test_SyncAllowance_SurvivesTheConfiguredValue checks the boundary in both directions using +// the value solana actually carries in pnf_path_rules.yaml. +func Test_SyncAllowance_SurvivesTheConfiguredValue(t *testing.T) { + const perceived = uint64(418_160_000) + const configured = uint64(1500) + + atLimit := protocol.EndpointAddr("pokt1atlimit-https://c001.op-gamma.example") + pastLimit := protocol.EndpointAddr("pokt1pastlimit-https://d001.op-delta.example") + + es := makeValidatableStore(t, perceived, map[protocol.EndpointAddr]uint64{ + freshEndpointAddr: perceived, + atLimit: perceived - configured, + pastLimit: perceived - configured - 1, + }) + es.serviceState.SetSyncAllowance(configured) + + selected := selectedSet(t, es, protocol.EndpointAddrList{freshEndpointAddr, atLimit, pastLimit}) + + require.True(t, selected[atLimit], "exactly at the allowance is inside it") + require.False(t, selected[pastLimit], "one block past the allowance is outside it") +} + +// Test_SetSyncAllowance_IsReachableFromHealthCheckConfig is the regression test for the actual +// defect. +// +// The health check executor applies a service's configured sync_allowance by asserting the QoS +// instance to `interface{ SetSyncAllowance(uint64) }`. Solana did not implement that method, so +// `sync_allowance: 750` was read, applied to the health check's own sync check, and silently +// dropped for endpoint selection — a configured value that reached one consumer and not the +// other, with nothing anywhere reporting the gap. +func Test_SetSyncAllowance_IsReachableFromHealthCheckConfig(t *testing.T) { + logger := polylog.Ctx(context.Background()) + serviceState := &ServiceState{logger: logger, serviceID: "solana"} + qosInstance := any(&QoS{ + logger: logger, + ServiceState: serviceState, + EndpointStore: &EndpointStore{logger: logger, serviceState: serviceState}, + }) + + // Byte-for-byte the assertion in gateway/health_check_executor.go. + setter, ok := qosInstance.(interface{ SetSyncAllowance(uint64) }) + require.True(t, ok, "solana QoS must satisfy the interface the health check executor asserts on") + + setter.SetSyncAllowance(750) + require.Equal(t, uint64(750), serviceState.getSyncAllowance(), + "the configured value must reach the state that ValidateEndpoint reads") +} + +// Test_SyncAllowance_UnconfiguredFallsBackToDefault covers the startup window and the case where +// external rules fail to load: 0 means "not configured", never "strict". +func Test_SyncAllowance_UnconfiguredFallsBackToDefault(t *testing.T) { + ss := &ServiceState{logger: polylog.Ctx(context.Background()), serviceID: "solana"} + + require.Equal(t, uint64(defaultSolanaBlockNumberSyncAllowance), ss.getSyncAllowance()) + + ss.SetSyncAllowance(0) + require.Equal(t, uint64(defaultSolanaBlockNumberSyncAllowance), ss.getSyncAllowance(), + "0 must not re-enable the strict comparison this fix exists to remove") +} diff --git a/qos/types/extractor.go b/qos/types/extractor.go index dbb9e7545..e1a9c173e 100644 --- a/qos/types/extractor.go +++ b/qos/types/extractor.go @@ -113,8 +113,18 @@ type ExtractedData struct { ChainID string // IsSyncing indicates if the endpoint is syncing. + // Only meaningful when SyncCheckPerformed is true. IsSyncing bool + // SyncCheckPerformed indicates whether a sync/health check was actually performed. + // When true, IsSyncing contains a definitive result. When false, IsSyncing should be + // ignored — the response simply was not one that carries sync status. + // + // Mirrors ArchivalCheckPerformed. Without this flag, "not syncing" (the zero value) is + // indistinguishable from "never checked", which is exactly the distinction a QoS needs + // in order to record a health observation rather than assume one. + SyncCheckPerformed bool + // IsArchival indicates if the endpoint supports archival queries. // Only meaningful when ArchivalCheckPerformed is true. IsArchival bool @@ -182,6 +192,7 @@ func (ed *ExtractedData) ExtractAll(extractor DataExtractor, request []byte) { // Check sync status if isSyncing, err := extractor.IsSyncing(request, ed.RawResponse); err == nil { ed.IsSyncing = isSyncing + ed.SyncCheckPerformed = true // Mark that we got a definitive result } else { ed.ExtractionErrors["is_syncing"] = err.Error() } @@ -261,6 +272,7 @@ func (ed *ExtractedData) ExtractWithConfig(extractor DataExtractor, request []by if config.CheckSyncStatus { if isSyncing, err := extractor.IsSyncing(request, ed.RawResponse); err == nil { ed.IsSyncing = isSyncing + ed.SyncCheckPerformed = true // Mark that we got a definitive result } else { ed.ExtractionErrors["is_syncing"] = err.Error() } diff --git a/reputation/cooldown_escalation_isolation_test.go b/reputation/cooldown_escalation_isolation_test.go new file mode 100644 index 000000000..84da7a5b0 --- /dev/null +++ b/reputation/cooldown_escalation_isolation_test.go @@ -0,0 +1,190 @@ +package reputation + +import ( + "context" + "testing" + "time" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" +) + +// seedScore installs a starting Score directly, to reach a state that otherwise needs more +// than an hour of wall-clock to reproduce (the escalation windows are DefaultMaxCooldown +// wide and the service reads time.Now() with no injectable clock). +// +// This is a PRECONDITION only. Every assertion below is on the bench the endpoint receives +// afterwards, never on a field written here. +func seedScore(t *testing.T, svc *service, key EndpointKey, score Score) { + t.Helper() + svc.mu.Lock() + svc.setScoreLocked(key, score) + svc.mu.Unlock() +} + +// remaining returns how long the endpoint's bench still has to run. +func remaining(t *testing.T, svc *service, ctx context.Context, key EndpointKey) time.Duration { + t.Helper() + score, err := svc.GetScore(ctx, key) + require.NoError(t, err) + return time.Until(score.CooldownUntil) +} + +// driveUntilTrip feeds a 1-in-`oneIn` protocol-violation stream ONE SIGNAL AT A TIME and +// stops the instant `stop` reports the trip under test has happened. +// +// Feeding a fixed-size batch does not work: a trip resets the detector's EWMA to zero and +// the loop keeps going, so a single 4000-request run at 1% trips FIVE times and escalates +// on each one. Any assertion about a first offence has to stop at the first offence. +func driveUntilTrip(t *testing.T, svc *service, ctx context.Context, key EndpointKey, oneIn int, stop func(Score) bool) Score { + t.Helper() + const maxSignals = 200000 + for i := 0; i < maxSignals; i++ { + var err error + if oneIn > 0 && i%oneIn == 0 { + err = svc.RecordSignal(ctx, key, violationSignal()) + } else { + err = svc.RecordSignal(ctx, key, NewSuccessSignal(100*time.Millisecond)) + } + require.NoError(t, err) + + score, err := svc.GetScore(ctx, key) + require.NoError(t, err) + if stop(score) { + return score + } + } + t.Fatalf("detector never tripped within %d signals", maxSignals) + return Score{} +} + +// tripCountAtLeast builds a stop predicate for the invalid-rate detector. Only valid on a +// key whose count starts at zero. +func tripCountAtLeast(n int) func(Score) bool { + return func(s Score) bool { return s.InvalidRateCooldownCount >= n } +} + +// benchExtendedBeyond stops as soon as the endpoint's bench runs longer than d — i.e. a +// rate detector has extended it past whatever was already in force. Used where the +// escalation counters are seeded non-zero and so cannot themselves signal a fresh trip. +func benchExtendedBeyond(d time.Duration) func(Score) bool { + return func(s Score) bool { return time.Until(s.CooldownUntil) > d } +} + +// TestInvalidRateEscalation_IgnoresCooldownsOtherDetectorsSet is the fix-3 test. +// +// Score.InvalidRateCooldownCount is documented as "kept separate from RateCooldownCount so +// the two detectors escalate independently and a trip of one cannot be misread as a trip of +// the other". The COUNTER was separate; the timestamp it escalated against was not — both +// detectors compared against the shared Score.CooldownUntil, which the strike system also +// writes. So the intent in that comment was never actually implemented. +// +// Note the sign: time.Since() on a cooldown that is still in force is NEGATIVE, hence +// always below DefaultMaxCooldown, so ANY bench in force made the very next trip read as +// consecutive. On a service where endpoints are benched often for unrelated reasons — the +// exact population this detector is aimed at — a first offence was benched at the escalated +// duration immediately. +// +// Asserted on the bench the endpoint actually receives, not on the counter: the counter is +// what the author wrote, the duration is what selection lives with. +func TestInvalidRateEscalation_IgnoresCooldownsOtherDetectorsSet(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("solana", "stale-history-plus-foreign-bench", sharedtypes.RPCType_JSON_RPC) + + // The production shape this reproduces: an endpoint on a churn-heavy service whose + // invalid-rate history is old enough that its escalation must RESET, but which is + // benched right now by an unrelated mechanism (here the strike system). + seedScore(t, svc, key, Score{ + Value: 80, + LastUpdated: time.Now(), + SuccessCount: 5000, + ErrorCount: 50, + + // Ran clean for this detector far longer than DefaultMaxCooldown, so its next trip + // is a first offence again. + InvalidRateCooldownCount: 3, + InvalidRateCooldownUntil: time.Now().Add(-2 * time.Hour), + + // A bench earned by the strike system, still in force. + CooldownUntil: time.Now().Add(5 * time.Minute), + }) + + // Stop the moment the bench is extended past the seeded 5m — the seeded count of 3 + // means the counter itself cannot mark a fresh trip. + score := driveUntilTrip(t, svc, ctx, key, 100, benchExtendedBeyond(6*time.Minute)) // 1 in 100 = 1% + require.True(t, score.IsInCooldown(), "the invalid-rate detector must have tripped") + + // A first offence benches for one DefaultRateCooldown (10m), which exceeds the 5m strike + // bench and is therefore visible in CooldownUntil. Escalating off the foreign bench + // instead resumes the stale count at 4 and benches for 40m. + require.InDelta(t, DefaultRateCooldown.Seconds(), remaining(t, svc, ctx, key).Seconds(), 30, + "a cooldown earned by another detector must not resume this detector's stale escalation") + require.Equal(t, 1, score.InvalidRateCooldownCount, + "this detector had run clean past DefaultMaxCooldown, so its count must reset to 1") +} + +// TestInvalidRateEscalation_StillEscalatesOnItsOwnRepeatTrips is the other half. A fix that +// simply stopped escalating would also pass the test above while removing the backoff that +// benches a persistent offender progressively longer. +func TestInvalidRateEscalation_StillEscalatesOnItsOwnRepeatTrips(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("solana", "two-consecutive-violations", sharedtypes.RPCType_JSON_RPC) + + first := driveUntilTrip(t, svc, ctx, key, 100, tripCountAtLeast(1)) + require.True(t, first.IsInCooldown()) + require.Equal(t, 1, first.InvalidRateCooldownCount) + require.InDelta(t, DefaultRateCooldown.Seconds(), remaining(t, svc, ctx, key).Seconds(), 30) + + // The trip reset RecentInvalidRate to 0, so the endpoint has to re-accumulate the same + // sustained rate before it can trip again. + second := driveUntilTrip(t, svc, ctx, key, 100, tripCountAtLeast(2)) + require.Equal(t, 2, second.InvalidRateCooldownCount, + "a repeat trip of this detector, landing within DefaultMaxCooldown of its own previous bench, must escalate") + require.InDelta(t, (2 * DefaultRateCooldown).Seconds(), remaining(t, svc, ctx, key).Seconds(), 30, + "the second consecutive invalid-rate trip must bench for 2 x DefaultRateCooldown") +} + +// TestCriticalRateEscalation_IgnoresCooldownsOtherDetectorsSet is the symmetric case. The +// critical-rate detector had the same defect first; the invalid-rate detector inherited it +// by copying the block. Fixing only the newer one would leave the older one escalating off +// benches it did not earn. +func TestCriticalRateEscalation_IgnoresCooldownsOtherDetectorsSet(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("eth", "stale-rate-history-plus-foreign-bench", sharedtypes.RPCType_JSON_RPC) + + // Mirror image of the test above: stale critical-rate history that must reset, plus a + // bench in force that this detector did not earn (here from the invalid-rate detector). + seedScore(t, svc, key, Score{ + Value: 80, + LastUpdated: time.Now(), + SuccessCount: 5000, + ErrorCount: 50, + RateCooldownCount: 3, + RateCooldownUntil: time.Now().Add(-2 * time.Hour), + CooldownUntil: time.Now().Add(5 * time.Minute), + }) + + // Drive a sustained 50% critical rate, carrying no protocol violations, so only the + // critical-rate detector can fire. + var score Score + for i := 0; i < 2000; i++ { + var sig Signal + if i%2 == 0 { + sig = NewCriticalErrorSignal("timeout", 100*time.Millisecond) + } else { + sig = NewSuccessSignal(100 * time.Millisecond) + } + require.NoError(t, svc.RecordSignal(ctx, key, sig)) + + var err error + score, err = svc.GetScore(ctx, key) + require.NoError(t, err) + if time.Until(score.CooldownUntil) > 6*time.Minute { + break + } + } + require.Equal(t, 1, score.RateCooldownCount, + "a bench earned by the invalid-rate detector is not a trip of the critical-rate detector") + require.InDelta(t, DefaultRateCooldown.Seconds(), remaining(t, svc, ctx, key).Seconds(), 30, + "a first critical-rate trip must bench for one DefaultRateCooldown, not the escalated duration") +} diff --git a/reputation/healthcheck_rate_exclusion_test.go b/reputation/healthcheck_rate_exclusion_test.go new file mode 100644 index 000000000..fdd045d9a --- /dev/null +++ b/reputation/healthcheck_rate_exclusion_test.go @@ -0,0 +1,153 @@ +package reputation + +import ( + "context" + "testing" + "time" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" +) + +// healthCheckViolationSignal is the signal a health-check probe produces when an endpoint +// returns a zero-length payload: identical to the user-traffic one except that it is +// stamped as probe-originated. +func healthCheckViolationSignal() Signal { + s := violationSignal() + s.IsHealthCheck = true + return s +} + +// healthCheckSuccessSignal is a probe that passed. +func healthCheckSuccessSignal() Signal { + s := NewSuccessSignal(100 * time.Millisecond) + s.IsHealthCheck = true + return s +} + +// runAtRateHealthCheck mirrors runAtRate but stamps every signal as probe-originated. +func runAtRateHealthCheck(t *testing.T, svc *service, ctx context.Context, key EndpointKey, n, oneIn int) Score { + t.Helper() + for i := 0; i < n; i++ { + var err error + if oneIn > 0 && i%oneIn == 0 { + err = svc.RecordSignal(ctx, key, healthCheckViolationSignal()) + } else { + err = svc.RecordSignal(ctx, key, healthCheckSuccessSignal()) + } + require.NoError(t, err) + } + score, err := svc.GetScore(ctx, key) + require.NoError(t, err) + return score +} + +// TestHealthCheckSignals_CannotTripInvalidRateDetector is the outcome test for the +// contamination bug. +// +// Both rate detectors are wrapped in `if !signal.IsHealthCheck` so a probe can never bench +// an endpoint on its own — a strict or flaky check (a Solana getBlockHeight sync check, a +// CometBFT probe with the wrong payload shape) must not cool an endpoint that serves user +// reads perfectly. That guard was intact; what was missing was the STAMP. Only the +// health-check executor's own three RecordSignal call sites set the flag, while a probe +// ALSO reaches reputation twice through the protocol layer — once from the relay itself +// and once from ApplyHTTPObservations — and neither of those stamped it. +// +// The consequence is a self-sustaining loop, not a one-off penalty: a benched endpoint +// receives no user traffic, so probes become its ONLY signal, so its rate EWMAs are 100% +// probe-derived, so it re-benches itself on the next probe failure. Measured on canary +// 2026-08-20: every solana endpoint tripping roughly twice an hour against a ~12-endpoint +// pool, with the pool-collapse guard firing 19.3x the control environment to keep the +// service served at all. +// +// The identical stream at the identical rate is asserted twice — once stamped, once not — +// so the control proves the harness actually reaches the detector. +func TestHealthCheckSignals_CannotTripInvalidRateDetector(t *testing.T) { + svc, ctx := newRateTestService(t) + + probeKey := NewEndpointKey("solana", "probe-only-violations", sharedtypes.RPCType_JSON_RPC) + probeScore := runAtRateHealthCheck(t, svc, ctx, probeKey, 4000, 100) // 1 in 100 = 1% + + require.False(t, probeScore.IsInCooldown(), + "a 1%% violation rate seen ONLY by health-check probes must not bench the endpoint: "+ + "a benched endpoint gets no user traffic, so probes become its only signal and it re-benches itself forever") + + // Control: the same stream, unstamped, must bench. Without this the test above passes + // on any harness that never reaches the detector at all. + userKey := NewEndpointKey("solana", "user-traffic-violations", sharedtypes.RPCType_JSON_RPC) + userScore := runAtRate(t, svc, ctx, userKey, 4000, 100) + require.True(t, userScore.IsInCooldown(), + "control: the same violation rate on USER traffic must still trip the detector") +} + +// TestHealthCheckSignals_CannotTripCriticalRateDetector is the same assertion for the +// older volume-independent critical-rate detector. That detector shipped with the +// `!IsHealthCheck` guard and has been silently contaminated by the protocol layer for as +// long as it has existed; the invalid-rate detector only made the contamination visible by +// tripping at a threshold three orders of magnitude lower. +func TestHealthCheckSignals_CannotTripCriticalRateDetector(t *testing.T) { + svc, ctx := newRateTestService(t) + + // A sustained 50% critical rate — far above CriticalRateThreshold (0.30) — seen only by + // probes. Alternating so the strike counter (which decays 3 per success) never reaches + // DefaultStrikeThreshold and cannot be the thing doing the benching. + probeKey := NewEndpointKey("eth", "probe-only-criticals", sharedtypes.RPCType_JSON_RPC) + for i := 0; i < 200; i++ { + var sig Signal + if i%2 == 0 { + sig = NewCriticalErrorSignal("timeout", 100*time.Millisecond) + sig.IsHealthCheck = true + } else { + sig = healthCheckSuccessSignal() + } + require.NoError(t, svc.RecordSignal(ctx, probeKey, sig)) + } + probeScore, err := svc.GetScore(ctx, probeKey) + require.NoError(t, err) + require.False(t, probeScore.IsInCooldown(), + "a 50%% critical rate seen ONLY by health-check probes must not bench the endpoint") + + // Control: identical stream, unstamped. + userKey := NewEndpointKey("eth", "user-traffic-criticals", sharedtypes.RPCType_JSON_RPC) + for i := 0; i < 200; i++ { + var sig Signal + if i%2 == 0 { + sig = NewCriticalErrorSignal("timeout", 100*time.Millisecond) + } else { + sig = NewSuccessSignal(100 * time.Millisecond) + } + require.NoError(t, svc.RecordSignal(ctx, userKey, sig)) + } + userScore, err := svc.GetScore(ctx, userKey) + require.NoError(t, err) + require.True(t, userScore.IsInCooldown(), + "control: the same critical rate on USER traffic must still trip the detector") +} + +// TestHealthCheckSignals_StillMoveTheAdditiveScore guards the scope of the exclusion. +// +// Only the volume-independent RATE detectors ignore probes. A probe result must still move +// Value — that is how an endpoint recovers from a bench when it is receiving no user +// traffic, and removing it would strand every benched endpoint permanently. A fix that +// made health checks inert everywhere would look like this test failing. +func TestHealthCheckSignals_StillMoveTheAdditiveScore(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("eth", "probe-moves-score", sharedtypes.RPCType_JSON_RPC) + + // An endpoint reputation has never seen has no stored score, so the baseline is the + // configured initial score rather than a read. + const initialScore = 80.0 + + for i := 0; i < 10; i++ { + sig := NewCriticalErrorSignal("timeout", 100*time.Millisecond) + sig.IsHealthCheck = true + require.NoError(t, svc.RecordSignal(ctx, key, sig)) + } + + after, err := svc.GetScore(ctx, key) + require.NoError(t, err) + require.Less(t, after.Value, initialScore, + "health-check failures must still penalise the additive score — the rate detectors are the only thing that excludes them") + require.Equal(t, int64(10), after.ErrorCount, + "probe results must still be counted; excluding them from the counters would corrupt every rate's denominator") +} diff --git a/reputation/invalid_rate_cooldown_test.go b/reputation/invalid_rate_cooldown_test.go new file mode 100644 index 000000000..d7f5fe592 --- /dev/null +++ b/reputation/invalid_rate_cooldown_test.go @@ -0,0 +1,135 @@ +package reputation + +import ( + "context" + "testing" + "time" + + sharedtypes "github.com/pokt-network/poktroll/x/shared/types" + "github.com/stretchr/testify/require" +) + +// violationSignal builds the signal the protocol classifier produces for a zero-length +// payload: CRITICAL severity plus the protocol-violation flag. +func violationSignal() Signal { + s := NewCriticalErrorSignal("empty_response", 100*time.Millisecond) + s.IsProtocolViolation = true + return s +} + +// run drives n requests at the given violation rate (1 in every `oneIn`), returning the score. +func runAtRate(t *testing.T, svc *service, ctx context.Context, key EndpointKey, n, oneIn int) Score { + t.Helper() + for i := 0; i < n; i++ { + var err error + if oneIn > 0 && i%oneIn == 0 { + err = svc.RecordSignal(ctx, key, violationSignal()) + } else { + err = svc.RecordSignal(ctx, key, NewSuccessSignal(100*time.Millisecond)) + } + require.NoError(t, err) + } + score, err := svc.GetScore(ctx, key) + require.NoError(t, err) + return score +} + +// TestInvalidRate_CatchesTheRateAdditiveScoringCannotReach is the reason this detector exists. +// +// Measured in production 2026-08-19: an endpoint returning zero-length payloads at ~0.2-0.9% +// of its traffic held a reputation score of 100 all day. Additive scoring cannot express that +// rate — at 1 violation per 1000 requests the endpoint earns +998 and loses -25, so the score +// pins at its ceiling no matter how long the behaviour continues. Raising the per-event +// penalty to FATAL (-50) does not change the sign either. +// +// A ~1% sustained violation rate must therefore trip a cooldown even though the endpoint is +// succeeding 99% of the time and its additive score is pegged at maximum. +func TestInvalidRate_CatchesTheRateAdditiveScoringCannotReach(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("solana", "empty-payloads-1pct", sharedtypes.RPCType_JSON_RPC) + + score := runAtRate(t, svc, ctx, key, 4000, 100) // 1 in 100 = 1% + + require.True(t, score.IsInCooldown(), + "a sustained 1%% protocol-violation rate must trip the invalid-rate detector") + require.Less(t, score.CriticalStrikes, DefaultStrikeThreshold, + "strikes must stay below threshold — this bench must come from the RATE detector, not the burst counter") + require.GreaterOrEqual(t, score.Value, float64(90), + "the additive score should still be near its ceiling: that is precisely why the rate detector is needed") +} + +// TestInvalidRate_QuietEndpointNotPenalized pins the noise floor. Every other domain on the +// fleet sat at ~0.00003% violations or exactly zero on 2026-08-19; none of them may be +// benched by this detector. +func TestInvalidRate_QuietEndpointNotPenalized(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("solana", "clean-endpoint", sharedtypes.RPCType_JSON_RPC) + + // 1 violation in 5000 requests = 0.02%, still ~250x the observed fleet noise floor. + score := runAtRate(t, svc, ctx, key, 5000, 5000) + + require.False(t, score.IsInCooldown(), + "an endpoint far below the threshold must never be benched by the invalid-rate detector") +} + +// TestInvalidRate_RequiresConvergedSample guards against benching on a short unlucky burst. +// The EWMA needs ~1/alpha observations before it means anything; below that the detector must +// stay silent no matter what it sees. +func TestInvalidRate_RequiresConvergedSample(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("solana", "new-endpoint-bad-start", sharedtypes.RPCType_JSON_RPC) + + // 100% violations, but far fewer than InvalidRateMinObservations. + score := runAtRate(t, svc, ctx, key, InvalidRateMinObservations/4, 1) + + require.Less(t, score.SuccessCount+score.ErrorCount, int64(InvalidRateMinObservations), + "precondition: sample must be under the minimum") + require.Equal(t, 0, score.InvalidRateCooldownCount, + "the invalid-rate detector must not trip before the EWMA has converged") +} + +// TestInvalidRate_HealthCheckProbesExcluded mirrors the critical-rate detector's exclusion. +// A hard bench must reflect what users receive; a probe that is stricter than user impact +// must not be able to cool an endpoint out of rotation on its own. +func TestInvalidRate_HealthCheckProbesExcluded(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("solana", "probe-only-violations", sharedtypes.RPCType_JSON_RPC) + + for i := 0; i < 4000; i++ { + sig := NewSuccessSignal(100 * time.Millisecond) + if i%10 == 0 { // 10% violation rate — far above threshold + sig = violationSignal() + } + sig.IsHealthCheck = true + require.NoError(t, svc.RecordSignal(ctx, key, sig)) + } + + score, err := svc.GetScore(ctx, key) + require.NoError(t, err) + require.Equal(t, 0, score.InvalidRateCooldownCount, + "health-check probes must never trip the invalid-rate detector") +} + +// TestInvalidRate_CriticalErrorsAloneDoNotTrip proves the two detectors stay separate. A +// sustained 5xx rate below CriticalRateThreshold is a transient the network absorbs; it must +// not reach the far lower invalid-rate threshold, or every endpoint with a 1% error rate +// would be benched. +func TestInvalidRate_CriticalErrorsAloneDoNotTrip(t *testing.T) { + svc, ctx := newRateTestService(t) + key := NewEndpointKey("eth", "ordinary-5xx", sharedtypes.RPCType_JSON_RPC) + + for i := 0; i < 4000; i++ { + if i%20 == 0 { // 5% critical rate — well under CriticalRateThreshold (0.30) + require.NoError(t, svc.RecordSignal(ctx, key, NewCriticalErrorSignal("5xx", 100*time.Millisecond))) + } else { + require.NoError(t, svc.RecordSignal(ctx, key, NewSuccessSignal(100*time.Millisecond))) + } + } + + score, err := svc.GetScore(ctx, key) + require.NoError(t, err) + require.Equal(t, 0, score.InvalidRateCooldownCount, + "plain critical errors must not feed the protocol-violation detector") + require.False(t, score.IsInCooldown(), + "a 5%% 5xx rate is below both detectors' thresholds and must not bench") +} diff --git a/reputation/reputation.go b/reputation/reputation.go index a10cf6ae9..db59f614b 100644 --- a/reputation/reputation.go +++ b/reputation/reputation.go @@ -128,6 +128,35 @@ type Score struct { // DefaultMaxCooldown. Mirrors the strike system's escalation, for the rate detector. RateCooldownCount int + // RateCooldownUntil is the end of the last cooldown THIS detector set, and is the + // reference point RateCooldownCount escalates against. + // + // It exists because CooldownUntil is shared: the strike system, this detector and the + // invalid-rate detector all write it. Escalating against the shared field made a + // cooldown earned by any other mechanism read as "a consecutive trip of this one", so + // on an endpoint that is benched often for unrelated reasons the count ratcheted to the + // DefaultMaxCooldown cap on what was really a first offence. Selection still gates on + // CooldownUntil alone; only the escalation arithmetic reads this. + RateCooldownUntil time.Time + + // RecentInvalidRate is an EWMA of the per-request protocol-violation indicator, the + // structural-validity counterpart to RecentCriticalRate. It uses a much longer memory and + // a much lower threshold: a violation rate that would be unremarkable for 5xx is damning + // for responses that are never legitimate. See InvalidRateEWMAAlpha / InvalidRateThreshold. + RecentInvalidRate float64 + + // InvalidRateCooldownCount counts consecutive invalid-rate trips for escalating backoff. + // Kept separate from RateCooldownCount so the two detectors escalate independently and a + // trip of one cannot be misread as a trip of the other. + InvalidRateCooldownCount int + + // InvalidRateCooldownUntil is the end of the last cooldown THIS detector set — the + // invalid-rate counterpart to RateCooldownUntil, and for the same reason. The separate + // COUNTER above was not sufficient on its own: both counters escalated against the + // shared CooldownUntil, so a critical-error burst still lengthened a protocol-violation + // bench, which is exactly what keeping the counters separate was supposed to prevent. + InvalidRateCooldownUntil time.Time + // IsArchival indicates whether the endpoint has passed archival health checks. // When true, the endpoint can serve historical blockchain data. // This is shared across all replicas via Redis storage. @@ -313,6 +342,29 @@ const ( // offense sits under one Shannon session (~20 min), so a transient spike recovers within // the same session; only a persistent offender ramps toward a full hour. DefaultRateCooldown = 10 * time.Minute + + // InvalidRateEWMAAlpha is the smoothing factor for Score.RecentInvalidRate. + // Effective memory ≈ 1/alpha requests, so 0.001 ≈ a 1000-request window. + // + // The long window is load-bearing, not conservatism: CriticalRateEWMAAlpha's ~20-request + // memory CANNOT REPRESENT a sub-1% rate at all — the EWMA can only be 0 or ~0.05 there, so + // the quantity we care about is not measurable at that alpha regardless of threshold. + InvalidRateEWMAAlpha = 0.001 + + // InvalidRateThreshold is the sustained protocol-violation rate (0..1) at or above which + // an endpoint is cooled down. 0.005 = 0.5% of responses structurally invalid. + // + // Sized from production, not intuition (2026-08-19, one hour, fleet-wide): the two + // offending domains ran 0.216% and 0.065% of ALL their relays, and ~0.85% of the affected + // service; every other domain on the fleet sat at 0.00003% or exactly zero. 0.5% sits far + // above the noise floor and below the observed offender on the service where it happens. + // Raise it if healthy endpoints trip; do not lower it without fresh per-domain data. + InvalidRateThreshold = 0.005 + + // InvalidRateMinObservations is the minimum lifetime observations before the invalid-rate + // detector may trip. It must be at least ~1/alpha or the EWMA has not converged and a + // short unlucky burst on a new endpoint reads as a sustained rate. + InvalidRateMinObservations = 1000 ) // Key granularity options determine how endpoints are grouped for scoring. diff --git a/reputation/service.go b/reputation/service.go index 295a87f77..f80d6042f 100644 --- a/reputation/service.go +++ b/reputation/service.go @@ -4,7 +4,6 @@ import ( "cmp" "context" "slices" - "strings" "sync" "time" @@ -202,7 +201,14 @@ func (s *service) RecordSignal(ctx context.Context, key EndpointKey, signal Sign // for longer than that starts fresh at one session. Mirrors the strike system's // exponential backoff so a persistently broken endpoint is benched progressively // longer while a one-off transient spike costs only a single session. - prevCooldownUntil := score.CooldownUntil + // + // Escalate against the end of THIS detector's own previous cooldown, not the + // shared CooldownUntil. The shared field is also written by the strike system + // and by the invalid-rate detector, so reading it here counted a bench earned + // by an unrelated mechanism as a consecutive trip of this one — an endpoint + // cooled often for other reasons reached the DefaultMaxCooldown cap on its + // first actual rate offence. + prevCooldownUntil := score.RateCooldownUntil if !prevCooldownUntil.IsZero() && time.Since(prevCooldownUntil) < DefaultMaxCooldown { score.RateCooldownCount++ } else { @@ -217,6 +223,7 @@ func (s *service) RecordSignal(ctx context.Context, key EndpointKey, signal Sign } rateCooldownUntil := time.Now().Add(cooldownDuration) + score.RateCooldownUntil = rateCooldownUntil if rateCooldownUntil.After(score.CooldownUntil) { score.CooldownUntil = rateCooldownUntil } @@ -233,6 +240,62 @@ func (s *service) RecordSignal(ctx context.Context, key EndpointKey, signal Sign Dur("cooldown_duration", cooldownDuration). Msg("[RATE_COOLDOWN] Endpoint cooled down due to sustained critical error rate (volume-independent)") } + + } + + // Protocol-violation rate detector. + // + // Separate from the critical-rate detector above because the two measure different + // things. A 5xx is a transient the network is expected to absorb, so its threshold is + // 30%. A structurally invalid response — today, a zero-length payload on a body-bearing + // 2xx — is never legitimate at any rate, so it warrants a threshold three orders of + // magnitude lower and a correspondingly longer EWMA window. + // + // Without this, such a violation is unreachable by reputation: the additive score is + // outvoted by successes (at 0.2%, +998 against -50 per 1000 requests, so the score + // returns to 100), and the critical-rate EWMA's ~20-request memory cannot represent a + // sub-1% rate at all. Raising the per-event penalty does not help; only a rate does. + invalidIndicator := 0.0 + if signal.IsProtocolViolation { + invalidIndicator = 1.0 + } + score.RecentInvalidRate = score.RecentInvalidRate*(1-InvalidRateEWMAAlpha) + InvalidRateEWMAAlpha*invalidIndicator + + if score.RecentInvalidRate >= InvalidRateThreshold && + (score.SuccessCount+score.ErrorCount) >= InvalidRateMinObservations { + trippedInvalidRate := score.RecentInvalidRate + + // Escalate against this detector's own previous cooldown end. Reading the + // shared CooldownUntil here defeated the point of keeping the two counters + // separate: a critical-error burst lengthened the protocol-violation bench. + prevInvalidCooldownUntil := score.InvalidRateCooldownUntil + if !prevInvalidCooldownUntil.IsZero() && time.Since(prevInvalidCooldownUntil) < DefaultMaxCooldown { + score.InvalidRateCooldownCount++ + } else { + score.InvalidRateCooldownCount = 1 + } + invalidCooldownDuration := DefaultRateCooldown * time.Duration(score.InvalidRateCooldownCount) + if invalidCooldownDuration > DefaultMaxCooldown { + invalidCooldownDuration = DefaultMaxCooldown + } + + invalidCooldownUntil := time.Now().Add(invalidCooldownDuration) + score.InvalidRateCooldownUntil = invalidCooldownUntil + if invalidCooldownUntil.After(score.CooldownUntil) { + score.CooldownUntil = invalidCooldownUntil + } + // Reset so the endpoint starts clean and must re-accumulate a sustained violation + // rate to trip again, rather than re-flapping on its first post-cooldown request. + score.RecentInvalidRate = 0 + metrics.RecordReputationInvalidRateCooldown(string(key.ServiceID)) + if s.logger != nil { + s.logger.Warn(). + Str("endpoint", key.String()). + Float64("invalid_rate", trippedInvalidRate). + Int("invalid_rate_cooldown_count", score.InvalidRateCooldownCount). + Dur("cooldown_duration", invalidCooldownDuration). + Msg("[INVALID_RATE_COOLDOWN] Endpoint cooled down due to sustained protocol-violation rate") + } } } @@ -251,32 +314,18 @@ func (s *service) RecordSignal(ctx context.Context, key EndpointKey, signal Sign // Buffer full, write will be picked up on next flush from cache } - // Per-supplier observability: emit a supplier-labeled signal counter so - // relay-miner operators can see what's happening on their endpoints. - // Empty supplier (per-domain reputation keys) is dropped by RecordSupplierSignal. - metrics.RecordSupplierSignal(supplierFromKey(key), string(key.ServiceID), string(signal.Type)) + // A supplier-labeled signal counter (path_supplier_signal_total) was emitted + // here until 2026-08-12. RecordSignal runs ~14,400/s fleet-wide and the + // supplier set rotates every session, so the metric minted 60,674 distinct + // series over one pod's 7.7h life while only 6,523 were ever live — a cost no + // cardinality guard can bound, since the guard caps the live registry and not + // the number of distinct series Prometheus retains. The same signal is + // available per operator, with the full taxonomy rather than a 3-class + // collapse, on path_relays_total{domain, reputation_signal, request_type}. return nil } -// supplierFromKey extracts the supplier address from an EndpointKey. -// Returns "" for per-domain keys (where supplier is not recoverable). -// -// The EndpointKey.EndpointAddr varies by KeyGranularity: -// - per-endpoint → "pokt1abc-https://..." (has dash, supplier before) -// - per-supplier → "pokt1abc..." (bech32 only) -// - per-domain → "node.example.com" (host only — no supplier available) -func supplierFromKey(key EndpointKey) string { - addr := string(key.EndpointAddr) - if i := strings.IndexByte(addr, '-'); i > 0 { - return addr[:i] - } - if strings.HasPrefix(addr, "pokt1") { - return addr - } - return "" -} - // GetScore retrieves the current reputation score for an endpoint. // Always reads from local cache for minimal latency. // If the endpoint's score is below threshold and hasn't received signals diff --git a/reputation/signals.go b/reputation/signals.go index e8f00b211..1376d6a68 100644 --- a/reputation/signals.go +++ b/reputation/signals.go @@ -69,6 +69,22 @@ type Signal struct { // endpoint that serves reads at 99.8% success). Set by the health-check executor. IsHealthCheck bool + // IsProtocolViolation marks a response that is structurally invalid rather than merely + // failed — one that no configuration, capability limit, or transient fault makes correct. + // Today: a zero-length payload on a body-bearing 2xx. + // + // It exists because such a response is BOTH rare and never legitimate, so it needs a far + // lower rate threshold than a 5xx. The critical-rate detector is tuned for "unambiguously + // broken" (30% of requests); a violation sustained at well under 1% is invisible to it, + // yet is not a transient the network should absorb. Measured 2026-08-19: two domains at + // ~0.2-0.9% against a fleet noise floor of ~0.00003% — three orders of magnitude of + // separation that no per-event penalty could express, because an additive score at that + // rate is outvoted by successes (+998 vs -50 per 1000 requests). + // + // Set by the producer (the protocol-layer classifier), consumed by the invalid-rate + // detector, exactly as IsHealthCheck is. + IsProtocolViolation bool + // Metadata holds additional signal-specific data. Metadata map[string]string } diff --git a/reputation/storage/redis.go b/reputation/storage/redis.go index f3668fadd..d3227dc4c 100644 --- a/reputation/storage/redis.go +++ b/reputation/storage/redis.go @@ -47,6 +47,10 @@ const ( fieldArchivalExpires = "archival_expires_at" fieldRecentCriticalRate = "recent_critical_rate" fieldRateCooldownCount = "rate_cooldown_count" + fieldRecentInvalidRate = "recent_invalid_rate" + fieldInvalidCooldownCnt = "invalid_rate_cooldown_count" + fieldRateCooldownUntil = "rate_cooldown_until" + fieldInvalidCooldownUnt = "invalid_rate_cooldown_until" ) // perceivedBlockTTL bounds how long a perceived block-height entry lives in @@ -205,6 +209,10 @@ func (r *RedisStorage) Set(ctx context.Context, key reputation.EndpointKey, scor fieldArchivalExpires: strconv.FormatInt(score.ArchivalExpiresAt.Unix(), 10), fieldRecentCriticalRate: strconv.FormatFloat(score.RecentCriticalRate, 'f', -1, 64), fieldRateCooldownCount: strconv.Itoa(score.RateCooldownCount), + fieldRecentInvalidRate: strconv.FormatFloat(score.RecentInvalidRate, 'f', -1, 64), + fieldInvalidCooldownCnt: strconv.Itoa(score.InvalidRateCooldownCount), + fieldRateCooldownUntil: strconv.FormatInt(score.RateCooldownUntil.Unix(), 10), + fieldInvalidCooldownUnt: strconv.FormatInt(score.InvalidRateCooldownUntil.Unix(), 10), } pipe := r.client.Pipeline() @@ -250,6 +258,10 @@ func (r *RedisStorage) SetMultiple(ctx context.Context, scores map[reputation.En fieldArchivalExpires: strconv.FormatInt(score.ArchivalExpiresAt.Unix(), 10), fieldRecentCriticalRate: strconv.FormatFloat(score.RecentCriticalRate, 'f', -1, 64), fieldRateCooldownCount: strconv.Itoa(score.RateCooldownCount), + fieldRecentInvalidRate: strconv.FormatFloat(score.RecentInvalidRate, 'f', -1, 64), + fieldInvalidCooldownCnt: strconv.Itoa(score.InvalidRateCooldownCount), + fieldRateCooldownUntil: strconv.FormatInt(score.RateCooldownUntil.Unix(), 10), + fieldInvalidCooldownUnt: strconv.FormatInt(score.InvalidRateCooldownUntil.Unix(), 10), } pipe.HSet(ctx, redisKey, fields) @@ -394,6 +406,34 @@ func (r *RedisStorage) parseScore(data map[string]string) (reputation.Score, err score.RateCooldownCount = count } + // Protocol-violation rate EWMA and its escalation counter. Absent on older records, + // which correctly leaves a pre-existing endpoint at 0.0 rather than inventing history. + if v, ok := data[fieldRecentInvalidRate]; ok { + if rate, err := strconv.ParseFloat(v, 64); err == nil { + score.RecentInvalidRate = rate + } + } + if v, ok := data[fieldInvalidCooldownCnt]; ok { + if n, err := strconv.Atoi(v); err == nil { + score.InvalidRateCooldownCount = n + } + } + + // Per-detector cooldown ends. These are the reference points the two escalation + // counters compare against; they are NOT a second selection gate — CooldownUntil above + // remains the only field selection reads. Absent on records written before this field + // existed, which leaves them zero and costs at most one non-escalated trip. + if v, ok := data[fieldRateCooldownUntil]; ok { + if ts, err := strconv.ParseInt(v, 10, 64); err == nil && ts > 0 { + score.RateCooldownUntil = time.Unix(ts, 0) + } + } + if v, ok := data[fieldInvalidCooldownUnt]; ok { + if ts, err := strconv.ParseInt(v, 10, 64); err == nil && ts > 0 { + score.InvalidRateCooldownUntil = time.Unix(ts, 0) + } + } + // Parse archival fields (for multi-instance coordination) if v, ok := data[fieldIsArchival]; ok { score.IsArchival = v == "1" || v == "true" diff --git a/reputation/supplier_extract_test.go b/reputation/supplier_extract_test.go deleted file mode 100644 index cb6a5b2a8..000000000 --- a/reputation/supplier_extract_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package reputation - -import ( - "testing" - - sharedtypes "github.com/pokt-network/poktroll/x/shared/types" - "github.com/stretchr/testify/require" - - "github.com/pokt-network/path/protocol" -) - -func TestSupplierFromKey(t *testing.T) { - cases := []struct { - name string - addr protocol.EndpointAddr - want string - }{ - { - name: "per-endpoint key (supplier-url)", - addr: protocol.EndpointAddr("pokt1abc123def-https://node.example.com"), - want: "pokt1abc123def", - }, - { - name: "per-supplier key (bech32 only)", - addr: protocol.EndpointAddr("pokt1abc123def"), - want: "pokt1abc123def", - }, - { - name: "per-domain key (no dash, no pokt1 prefix)", - addr: protocol.EndpointAddr("nodefleet.net"), - want: "", - }, - { - name: "empty", - addr: protocol.EndpointAddr(""), - want: "", - }, - { - name: "leading dash (malformed) → empty", - addr: protocol.EndpointAddr("-https://x"), - want: "", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - key := NewEndpointKey("eth", c.addr, sharedtypes.RPCType_JSON_RPC) - require.Equal(t, c.want, supplierFromKey(key)) - }) - } -} diff --git a/router/operational_endpoints.go b/router/operational_endpoints.go index 9d8ab8ae7..fd8d32a40 100644 --- a/router/operational_endpoints.go +++ b/router/operational_endpoints.go @@ -571,3 +571,54 @@ func (r *router) handleWebsocketTumble(w http.ResponseWriter, req *http.Request) w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(result) } + +// handleRequestSample handles GET /admin/request-sample[/{serviceId}] +// +// Without a service ID: one summary row per observed service. With one: the full report — +// uniqueness ratio, per-method distinct counts and the most repeated fingerprints with a +// payload snippet. Query: window=previous (default: the live window), top=N (default 20). +// +// Per-pod: the sampler is in-memory, so ask several pods (or the one carrying the most +// traffic) before drawing a fleet-wide conclusion. +func (r *router) handleRequestSample(w http.ResponseWriter, req *http.Request) { + if r.requestSampleAdmin == nil { + http.Error(w, `{"error":"request sampling not enabled (PATH_REQUEST_SAMPLE_RATE=0)"}`, http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + + serviceID := strings.TrimPrefix(strings.TrimPrefix(req.URL.Path, "/admin/request-sample"), "/") + if serviceID == "" { + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "services": r.requestSampleAdmin.Summary(), + "hint": "GET /admin/request-sample/{serviceId}?window=previous&top=20 for fingerprints", + }) + return + } + + q := req.URL.Query() + previous := q.Get("window") == "previous" + top := 20 + if v := q.Get("top"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + http.Error(w, `{"error":"top must be a positive integer"}`, http.StatusBadRequest) + return + } + top = n + } + + report, found := r.requestSampleAdmin.Report(serviceID, previous, top) + if !found { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "service_id": serviceID, + "window": q.Get("window"), + "error": "no sample for this service and window yet (service unobserved on this pod, or no completed window)", + }) + return + } + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(report) +} diff --git a/router/request_sample_endpoint_test.go b/router/request_sample_endpoint_test.go new file mode 100644 index 000000000..b13daab99 --- /dev/null +++ b/router/request_sample_endpoint_test.go @@ -0,0 +1,97 @@ +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/pokt-network/path/config" + "github.com/pokt-network/path/gateway" + "github.com/pokt-network/path/health" +) + +type fakeRequestSampleAdmin struct { + report gateway.RequestSampleReport + found bool + lastSvc string + lastWin bool + lastTop int +} + +func (f *fakeRequestSampleAdmin) Report(serviceID string, previous bool, top int) (gateway.RequestSampleReport, bool) { + f.lastSvc, f.lastWin, f.lastTop = serviceID, previous, top + return f.report, f.found +} +func (f *fakeRequestSampleAdmin) Summary() []gateway.RequestSampleSummary { + return []gateway.RequestSampleSummary{{ServiceID: "solana", Sampled: 3}} +} + +func newRouterWithSampleAdmin(t *testing.T, admin RequestSampleAdmin) *httptest.Server { + t.Helper() + ctrl := gomock.NewController(t) + r := NewRouter(polyzero.NewLogger(), NewMockgatewayHandler(ctrl), NewMockdisqualifiedEndpointsReporter(ctrl), + &health.Checker{}, config.RouterConfig{}, nil, nil, nil, nil, nil, admin) + ts := httptest.NewServer(r.mux) + t.Cleanup(ts.Close) + return ts +} + +func TestRequestSampleEndpoint(t *testing.T) { + t.Run("disabled sampler reports 503", func(t *testing.T) { + ts := newRouterWithSampleAdmin(t, nil) + resp, err := http.Get(ts.URL + "/admin/request-sample/solana") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + }) + + t.Run("summary without service id", func(t *testing.T) { + ts := newRouterWithSampleAdmin(t, &fakeRequestSampleAdmin{}) + resp, err := http.Get(ts.URL + "/admin/request-sample") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var body struct { + Services []gateway.RequestSampleSummary `json:"services"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&body)) + require.Len(t, body.Services, 1) + require.Equal(t, "solana", body.Services[0].ServiceID) + }) + + t.Run("report passes window and top through", func(t *testing.T) { + admin := &fakeRequestSampleAdmin{found: true, report: gateway.RequestSampleReport{ServiceID: "solana", Sampled: 7, Uniqueness: 0.5}} + ts := newRouterWithSampleAdmin(t, admin) + resp, err := http.Get(ts.URL + "/admin/request-sample/solana?window=previous&top=5") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "solana", admin.lastSvc) + require.True(t, admin.lastWin) + require.Equal(t, 5, admin.lastTop) + var got gateway.RequestSampleReport + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + require.Equal(t, uint64(7), got.Sampled) + }) + + t.Run("unknown service is 404", func(t *testing.T) { + ts := newRouterWithSampleAdmin(t, &fakeRequestSampleAdmin{found: false}) + resp, err := http.Get(ts.URL + "/admin/request-sample/nope") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusNotFound, resp.StatusCode) + }) + + t.Run("bad top is 400", func(t *testing.T) { + ts := newRouterWithSampleAdmin(t, &fakeRequestSampleAdmin{found: true}) + resp, err := http.Get(ts.URL + "/admin/request-sample/solana?top=zero") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + }) +} diff --git a/router/router.go b/router/router.go index f38446337..dec621e7b 100644 --- a/router/router.go +++ b/router/router.go @@ -41,6 +41,7 @@ type ( websocketAdmin WebsocketAdmin reputationAdmin ReputationAdmin staticResponses StaticResponseResolver + requestSampleAdmin RequestSampleAdmin } gatewayHandler interface { HandleServiceRequest(context.Context, *http.Request, http.ResponseWriter) @@ -66,6 +67,13 @@ type ( ChainStateAdmin interface { ResetChainState(ctx context.Context, serviceID string) (found bool, err error) } + // RequestSampleAdmin reads the request-shape sampler: is a service's traffic many + // different requests or the same few repeated? found=false means the service has not + // been observed (or the requested window does not exist yet). + RequestSampleAdmin interface { + Report(serviceID string, previous bool, top int) (report gateway.RequestSampleReport, found bool) + Summary() []gateway.RequestSampleSummary + } // WebsocketAdmin allows redistributing live websocket connections via admin // endpoints. A websocket connection binds one endpoint for its whole lifetime, so // connections that landed on a concentrated operator stay there until they rebind; @@ -95,6 +103,7 @@ func NewRouter( websocketAdmin WebsocketAdmin, reputationAdmin ReputationAdmin, staticResponses StaticResponseResolver, + requestSampleAdmin RequestSampleAdmin, ) *router { r := &router{ logger: logger.With("package", "router"), @@ -110,6 +119,7 @@ func NewRouter( websocketAdmin: websocketAdmin, reputationAdmin: reputationAdmin, staticResponses: staticResponses, + requestSampleAdmin: requestSampleAdmin, } r.handleRoutes() return r @@ -155,6 +165,13 @@ func (r *router) handleRoutes() { // endpoints for a service (cooldown only; reputation itself is left untouched) r.mux.HandleFunc("POST /admin/reputation/drain/", r.handleReputationDrain) + // GET /admin/request-sample[/{serviceId}] - sampled request-shape report: uniqueness + // ratio, top repeated fingerprints, per-method distinct counts. Answers "is this + // traffic diverse or the same few requests over and over" — the condition under which a + // cached endpoint wins every race. Per-pod sample (in-memory). + r.mux.HandleFunc("GET /admin/request-sample", r.handleRequestSample) + r.mux.HandleFunc("GET /admin/request-sample/", r.handleRequestSample) + // requestHandlerFn defines the middleware chain for all service requests. // staticResponseMiddleware runs after the prefix strip (so it sees the cleaned path) // and before the relay handler, short-circuiting any configured static route. diff --git a/router/router_test.go b/router/router_test.go index 0e2647814..d76f0bf98 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -34,6 +34,7 @@ func newTestRouter(t *testing.T) (*router, *MockgatewayHandler, *httptest.Server nil, // no websocket admin in tests nil, // no reputation admin in tests nil, // no static responses in tests + nil, // no request sampler in tests ) ts := httptest.NewServer(r.mux) t.Cleanup(ts.Close) diff --git a/router/static_response_test.go b/router/static_response_test.go index 8f7284a46..aa316186f 100644 --- a/router/static_response_test.go +++ b/router/static_response_test.go @@ -41,6 +41,7 @@ func newStaticTestRouter(t *testing.T, resolver StaticResponseResolver) (*Mockga nil, nil, resolver, + nil, ) ts := httptest.NewServer(r.mux) t.Cleanup(ts.Close)