From 7fff264358b8781dac786171ef8157300874087f Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Sat, 25 Jul 2026 00:45:06 -0500 Subject: [PATCH 1/3] feat(gkapi): meter Tor exits through one shared invite ceiling ## Problem The per-IP invite limit (4/24h) is structurally defeated by Tor circuit rotation: every exit node is a fresh IP and therefore a fresh bucket. Measured 2026-07-25 from the live rate-limit store, one actor pulled 246 invites in ~20 minutes across 202 distinct IPs -- 200 of them via Tor exits -- without any single IP approaching the limit. Tightening the per-IP number cannot fix this; rotation routes around it at any value. Downstream effect: every invite mints a fresh keypair, so the Official River room got a stream of new identities that moderation could not converge on (banning is per-identity). ## Approach The Tor exit set is enumerable, so stop treating exits as independent identities and meter the whole set through ONE shared hourly bucket. Rotation then buys the attacker nothing. Sizing from the same data: organic Tor traffic peaked at 11 invites/hour (mean 6.8) across non-burst hours; the burst hit 158/hour. A ceiling of 25/hour leaves 2.3x headroom over the worst organic hour while capping a rotation burst hard. Deliberately NOT a Tor block. Tor is ~4.8% of non-burst invite traffic (69 requests, 56 distinct exits, nearly all 1-2 requests each -- ordinary usage). Freenet is a privacy project and quickstart is the main onboarding path; blocking that traffic costs real users for no extra protection a bucket does not already give. Validated against the live exit list: 159/202 burst IPs are listed, covering 200/246 burst invites. The ceiling would have admitted 25 and refused 175 -- a ~71% cut to that burst. The rest came from non-Tor IPs and is unaffected, which the ceiling does not claim to address. ## Failure policy Fails OPEN throughout. If the list cannot be fetched and no cache exists, nothing is treated as Tor and gkapi behaves exactly as before. A garbage 200 never wipes a good list; a fetch error never clears one; the cache is written via write-then-rename so a crash cannot truncate it. Ordering in the handler is deliberate: the Tor capacity check runs BEFORE the per-IP check so a refused burst does not burn the requester's per-IP allowance, and the matching record() runs only AFTER the per-IP check passes so a per-IP rejection never consumes shared Tor capacity. ## Testing 9 new tests: parsing (v4/v6/junk/whitespace), fail-open on empty, corrupt, and missing cache, atomic cache write, bucket limit/expiry, shared-not-per-identity, and a sizing regression pinning the ceiling above the observed organic peak and below the observed burst. Plus an #[ignore]d live test against the real Tor Project list (run before deploying changes to the fetch path); it fetched 1386 exits and round-tripped the cache. Refs freenet/web#81 [AI-assisted - Claude] --- rust/api/Cargo.toml | 3 + rust/api/src/errors.rs | 2 +- rust/api/src/invite.rs | 13 +- rust/api/src/main.rs | 20 ++ rust/api/src/rate_limit.rs | 214 +++++++++++++++++- rust/api/src/routes.rs | 59 ++++- rust/api/src/tor.rs | 439 +++++++++++++++++++++++++++++++++++++ 7 files changed, 733 insertions(+), 17 deletions(-) create mode 100644 rust/api/src/tor.rs diff --git a/rust/api/Cargo.toml b/rust/api/Cargo.toml index b01d093b..46f92ca1 100644 --- a/rust/api/Cargo.toml +++ b/rust/api/Cargo.toml @@ -32,6 +32,9 @@ serde = { version = "1.0", features = ["derive"] } blind-rsa-signatures = "0.15.1" ciborium = "0.2" bs58 = "0.5" +# Tor exit-list fetch. rustls (not native-tls) to match axum-server's TLS stack +# and avoid pulling in openssl. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } [dev-dependencies] tempfile = "3" diff --git a/rust/api/src/errors.rs b/rust/api/src/errors.rs index 87c99fe9..3b80c608 100644 --- a/rust/api/src/errors.rs +++ b/rust/api/src/errors.rs @@ -1,5 +1,5 @@ -use serde::de::StdError; use ghostkey_lib::errors::GhostkeyError; +use serde::de::StdError; #[derive(Debug)] pub enum CertificateError { diff --git a/rust/api/src/invite.rs b/rust/api/src/invite.rs index 0a549856..1bf76040 100644 --- a/rust/api/src/invite.rs +++ b/rust/api/src/invite.rs @@ -166,18 +166,19 @@ mod tests { let owner_vk_bytes = bs58::decode("93XNNwmRLQ6nwUwi4dDmp3kpjMb5ekMRc2e22x5TAnUY") .into_vec() .expect("Failed to decode owner VK"); - let owner_vk = VerifyingKey::from_bytes(&owner_vk_bytes.try_into().expect("Invalid VK length")) - .expect("Invalid VK"); + let owner_vk = + VerifyingKey::from_bytes(&owner_vk_bytes.try_into().expect("Invalid VK length")) + .expect("Invalid VK"); // Signing key from rooms.json for freenet-chat let signing_key_bytes: [u8; 32] = [ - 1, 3, 163, 211, 4, 113, 25, 236, 171, 57, 117, 76, 11, 233, 182, 31, - 111, 137, 94, 202, 149, 4, 41, 59, 145, 54, 18, 82, 243, 194, 71, 224 + 1, 3, 163, 211, 4, 113, 25, 236, 171, 57, 117, 76, 11, 233, 182, 31, 111, 137, 94, 202, + 149, 4, 41, 59, 145, 54, 18, 82, 243, 194, 71, 224, ]; let signing_key = SigningKey::from_bytes(&signing_key_bytes); - let invite = create_invitation(&owner_vk, &signing_key) - .expect("Failed to create invitation"); + let invite = + create_invitation(&owner_vk, &signing_key).expect("Failed to create invitation"); println!("\n=== Generated Invite for freenet-chat ==="); println!("{}", invite); diff --git a/rust/api/src/main.rs b/rust/api/src/main.rs index af5c71ff..0e24810a 100644 --- a/rust/api/src/main.rs +++ b/rust/api/src/main.rs @@ -19,6 +19,7 @@ mod handle_sign_cert; mod invite; mod rate_limit; mod routes; +mod tor; /// Canonical env var for the notary key directory. The legacy name /// `DELEGATE_DIR` is also read (in `delegates::notary_dir`) for backward @@ -78,6 +79,12 @@ fn load_invite_config(matches: &clap::ArgMatches) -> Option { .map(|s| s.as_str()) .unwrap_or("/var/lib/gkapi/invite_rate_limits.json"), ); + let tor_exit_cache = Some(PathBuf::from( + matches + .get_one::("tor-exit-cache") + .map(|s| s.as_str()) + .unwrap_or("/var/lib/gkapi/tor_exit_list.txt"), + )); // Load signing key from file (32 bytes raw) let signing_key_bytes = match fs::read(signing_key_path) { @@ -135,6 +142,7 @@ fn load_invite_config(matches: &clap::ArgMatches) -> Option { Some(InviteState::new( rate_limit_file, + tor_exit_cache, room_owner_vk, inviter_signing_key, room_name, @@ -241,6 +249,14 @@ async fn main() { .default_value("/var/lib/gkapi/invite_rate_limits.json") .help("Path to rate limit JSON file"), ) + .arg( + Arg::new("tor-exit-cache") + .long("tor-exit-cache") + .value_name("FILE") + .env("TOR_EXIT_CACHE") + .default_value("/var/lib/gkapi/tor_exit_list.txt") + .help("Path to the cached Tor exit-node list (refreshed hourly)"), + ) .get_matches(); let notary_dir = matches.get_one::("notary-dir").unwrap(); @@ -284,6 +300,10 @@ async fn main() { "River room invite endpoint enabled for room: {}", state.room_name ); + // Keep the Tor exit list current so the shared Tor ceiling can be + // applied. If this never succeeds the list stays empty and invite + // limiting silently degrades to per-IP only (fail open). + tor::spawn_refresher(Arc::clone(&state.tor_exits)); app = app.merge(routes::get_invite_routes(state)); } else { warn!("River room invite endpoint not configured. Set ROOM_SIGNING_KEY_FILE and ROOM_OWNER_VK to enable."); diff --git a/rust/api/src/rate_limit.rs b/rust/api/src/rate_limit.rs index 926235ed..75e3a6b6 100644 --- a/rust/api/src/rate_limit.rs +++ b/rust/api/src/rate_limit.rs @@ -4,8 +4,8 @@ use chrono::{DateTime, Duration, Utc}; use serde::{Deserialize, Serialize}; -use sha2::{Sha256, Digest}; -use std::collections::HashMap; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, VecDeque}; use std::fs; use std::net::IpAddr; use std::path::PathBuf; @@ -20,12 +20,28 @@ use thiserror::Error; /// account creation. Raising this without a matching anti-abuse story re-opens /// the vector this limit exists to slow. See /// `test_rate_limiter_enforces_four_per_window`. -const MAX_INVITES_PER_WINDOW: usize = 4; +/// +/// NOTE: this limit is per-IP and therefore cannot bound an actor who rotates +/// IPs. For the Tor exit set specifically, see [`AggregateBucket`] and +/// [`TOR_INVITES_PER_HOUR`]. +pub const MAX_INVITES_PER_WINDOW: usize = 4; + +/// Shared hourly invite ceiling across the ENTIRE Tor exit set. +/// +/// Sizing (measured 2026-07-24/25 from `invite_rate_limits.json`): organic Tor +/// traffic peaked at **11 invites/hour** (mean 6.8) across 6 non-burst hours, +/// while the abuse burst hit **158/hour**. 25 leaves 2.3x headroom over the +/// worst observed organic hour, so ordinary Tor users are unaffected, while +/// capping a rotation burst at ~16% of what it achieved. See +/// `tor_bucket_admits_organic_peak_but_caps_burst`. +pub const TOR_INVITES_PER_HOUR: usize = 25; + +/// Window for [`TOR_INVITES_PER_HOUR`], in minutes. +pub const TOR_WINDOW_MINUTES: i64 = 60; /// SHA256 hashes of IPs exempt from rate limiting (for testing) -const EXEMPT_IP_HASHES: &[&str] = &[ - "0cf75236cce089f9c592bb2b50925c48cbbb4d0f83094b2cd091dda4b53e1a4c", -]; +const EXEMPT_IP_HASHES: &[&str] = + &["0cf75236cce089f9c592bb2b50925c48cbbb4d0f83094b2cd091dda4b53e1a4c"]; /// Check if an IP is exempt from rate limiting fn is_exempt(ip: &IpAddr) -> bool { @@ -170,12 +186,186 @@ impl RateLimiter { } } +/// A single sliding-window counter shared by many identities. +/// +/// # Why a shared bucket +/// +/// Per-IP limiting assumes an IP approximates a person. For Tor that assumption +/// is false in the attacker's favour: exits are a public, rotatable pool, so N +/// exits multiply any per-IP limit by N. Metering the whole pool through ONE +/// bucket removes the multiplier entirely — rotating costs the attacker nothing +/// and gains them nothing. +/// +/// Deliberately in-memory (not persisted like [`RateLimiter`]): a restart +/// forgives at most [`TOR_INVITES_PER_HOUR`] requests, gkapi restarts only on +/// deploy, and this keeps the hot path free of the read-modify-write file IO +/// the per-IP limiter does. +pub struct AggregateBucket { + limit: usize, + window: Duration, + hits: Mutex>>, +} + +impl AggregateBucket { + pub fn new(limit: usize, window_minutes: i64) -> Self { + Self { + limit, + window: Duration::minutes(window_minutes), + hits: Mutex::new(VecDeque::new()), + } + } + + /// Drop hits that have aged out of the window. + fn prune(&self, hits: &mut VecDeque>, now: DateTime) { + while let Some(front) = hits.front() { + if now - *front >= self.window { + hits.pop_front(); + } else { + break; + } + } + } + + /// Is there room in the window right now? + /// + /// Checked separately from [`Self::record`] so a caller can reject a request + /// BEFORE spending the requester's per-IP allowance on it (see the ordering + /// note in `routes::create_room_invite`). The gap between the two admits a + /// small overshoot under concurrent load, bounded by the number of requests + /// in flight; for an anti-abuse ceiling that is not worth a global lock. + pub fn has_capacity(&self) -> bool { + let now = Utc::now(); + match self.hits.lock() { + Ok(mut hits) => { + self.prune(&mut hits, now); + hits.len() < self.limit + } + // Fail open: a poisoned lock must not block legitimate users. + Err(_) => true, + } + } + + /// Record one hit against the window. + pub fn record(&self) { + let now = Utc::now(); + if let Ok(mut hits) = self.hits.lock() { + self.prune(&mut hits, now); + hits.push_back(now); + } + } + + /// Seconds until the window has room again, or `None` if it has room now. + pub fn retry_after_seconds(&self) -> Option { + let now = Utc::now(); + let mut hits = self.hits.lock().ok()?; + self.prune(&mut hits, now); + if hits.len() < self.limit { + return None; + } + hits.front() + .map(|oldest| (*oldest + self.window - now).num_seconds().max(0)) + } + + /// Current occupancy (for logging / diagnostics). + pub fn current(&self) -> usize { + let now = Utc::now(); + match self.hits.lock() { + Ok(mut hits) => { + self.prune(&mut hits, now); + hits.len() + } + Err(_) => 0, + } + } +} + #[cfg(test)] mod tests { use super::*; use std::net::Ipv4Addr; use tempfile::tempdir; + #[test] + fn aggregate_bucket_admits_up_to_limit_then_refuses() { + let bucket = AggregateBucket::new(3, 60); + for i in 1..=3 { + assert!(bucket.has_capacity(), "hit {i} should have capacity"); + bucket.record(); + } + assert!(!bucket.has_capacity(), "4th hit must be refused"); + assert_eq!(bucket.current(), 3); + assert!(bucket.retry_after_seconds().is_some()); + } + + /// The whole point: many distinct identities share ONE budget, so rotating + /// between them gains nothing. + #[test] + fn aggregate_bucket_is_not_per_identity() { + let bucket = AggregateBucket::new(2, 60); + // Simulate three different "IPs" all funnelling through one bucket. + bucket.record(); // exit A + bucket.record(); // exit B + assert!( + !bucket.has_capacity(), + "a third distinct exit must NOT get its own allowance" + ); + } + + /// Sizing regression: the deployed ceiling must stay above the measured + /// organic Tor peak (11/hour) and far below the observed burst (158/hour). + #[test] + fn tor_bucket_admits_organic_peak_but_caps_burst() { + const OBSERVED_ORGANIC_PEAK: usize = 11; + const OBSERVED_BURST: usize = 158; + + assert!( + TOR_INVITES_PER_HOUR > OBSERVED_ORGANIC_PEAK, + "ceiling {TOR_INVITES_PER_HOUR} must exceed the organic peak \ + {OBSERVED_ORGANIC_PEAK}/h or real Tor users get blocked" + ); + assert!( + TOR_INVITES_PER_HOUR < OBSERVED_BURST / 2, + "ceiling {TOR_INVITES_PER_HOUR} must be well under the observed \ + burst {OBSERVED_BURST}/h or it does not actually bound abuse" + ); + + let bucket = AggregateBucket::new(TOR_INVITES_PER_HOUR, TOR_WINDOW_MINUTES); + for _ in 0..OBSERVED_ORGANIC_PEAK { + assert!( + bucket.has_capacity(), + "organic traffic must never be refused" + ); + bucket.record(); + } + // Now replay the burst; it must be cut off at the ceiling. + let mut admitted = OBSERVED_ORGANIC_PEAK; + for _ in 0..OBSERVED_BURST { + if bucket.has_capacity() { + bucket.record(); + admitted += 1; + } + } + assert_eq!( + admitted, TOR_INVITES_PER_HOUR, + "burst must be capped at the ceiling, not merely slowed" + ); + } + + #[test] + fn aggregate_bucket_expires_old_hits() { + let bucket = AggregateBucket::new(2, 60); + // Backdate both hits beyond the window. + { + let mut hits = bucket.hits.lock().unwrap(); + let old = Utc::now() - Duration::minutes(61); + hits.push_back(old); + hits.push_back(old); + } + assert_eq!(bucket.current(), 0, "expired hits must be pruned"); + assert!(bucket.has_capacity()); + assert!(bucket.retry_after_seconds().is_none()); + } + #[test] fn test_rate_limiter_allows_first_request() { let dir = tempdir().unwrap(); @@ -226,11 +416,19 @@ mod tests { // Should allow MAX_INVITES_PER_WINDOW requests for i in 0..MAX_INVITES_PER_WINDOW { - assert!(limiter.check_and_record(ip).unwrap(), "Request {} should be allowed", i + 1); + assert!( + limiter.check_and_record(ip).unwrap(), + "Request {} should be allowed", + i + 1 + ); } // Next request should be blocked - assert!(!limiter.check_and_record(ip).unwrap(), "Request {} should be blocked", MAX_INVITES_PER_WINDOW + 1); + assert!( + !limiter.check_and_record(ip).unwrap(), + "Request {} should be blocked", + MAX_INVITES_PER_WINDOW + 1 + ); } #[test] diff --git a/rust/api/src/routes.rs b/rust/api/src/routes.rs index 540c64b8..803abea0 100644 --- a/rust/api/src/routes.rs +++ b/rust/api/src/routes.rs @@ -22,12 +22,17 @@ use crate::handle_sign_cert::{ sign_certificate, CertificateError, SignCertificateRequest, SignCertificateResponse, }; use crate::invite; -use crate::rate_limit::RateLimiter; +use crate::rate_limit::{AggregateBucket, RateLimiter, TOR_INVITES_PER_HOUR, TOR_WINDOW_MINUTES}; +use crate::tor::TorExitList; /// Shared application state for invite generation #[derive(Clone)] pub struct InviteState { pub rate_limiter: Arc, + /// Shared ceiling across the whole Tor exit set (see `AggregateBucket`). + pub tor_bucket: Arc, + /// Membership test for "is this IP a Tor exit". Empty => nothing is Tor. + pub tor_exits: Arc, pub room_owner_vk: VerifyingKey, pub inviter_signing_key: SigningKey, pub room_name: String, @@ -36,12 +41,18 @@ pub struct InviteState { impl InviteState { pub fn new( rate_limit_file: PathBuf, + tor_exit_cache: Option, room_owner_vk: VerifyingKey, inviter_signing_key: SigningKey, room_name: String, ) -> Self { Self { rate_limiter: Arc::new(RateLimiter::new(rate_limit_file, 24)), + tor_bucket: Arc::new(AggregateBucket::new( + TOR_INVITES_PER_HOUR, + TOR_WINDOW_MINUTES, + )), + tor_exits: Arc::new(TorExitList::new(tor_exit_cache)), room_owner_vk, inviter_signing_key, room_name, @@ -368,7 +379,43 @@ async fn create_room_invite( ConnectInfo(addr): ConnectInfo, ) -> Result, (StatusCode, Json)> { let client_ip = get_client_ip(addr); - info!("Received create-invite request from IP: {}", client_ip); + let via_tor = state.tor_exits.is_exit(&client_ip); + info!( + "Received create-invite request from IP: {} (tor_exit={})", + client_ip, via_tor + ); + + // Tor exits share ONE hourly ceiling, because per-IP limiting cannot bound + // an actor who rotates exit nodes (2026-07-25: 152 invites via 113 exits, + // none near the per-IP limit). + // + // Ordering matters. This capacity check runs BEFORE `check_and_record` so a + // request refused here does not also burn the requester's per-IP allowance + // -- otherwise a burst would silently consume the quota of the ordinary Tor + // users it is sharing the bucket with. The matching `record()` happens only + // AFTER the per-IP check passes, so a request rejected for per-IP reasons + // never consumes shared Tor capacity either. + if via_tor && !state.tor_bucket.has_capacity() { + let retry_after = state.tor_bucket.retry_after_seconds(); + info!( + "Tor exit {} refused: shared Tor ceiling reached ({}/{} per hour), retry_after: {:?}", + client_ip, + state.tor_bucket.current(), + TOR_INVITES_PER_HOUR, + retry_after + ); + return Err(( + StatusCode::TOO_MANY_REQUESTS, + Json(InviteErrorResponse { + error: format!( + "Invite requests from Tor are limited to {} per hour in total. \ + Please try again shortly, or request an invite without Tor.", + TOR_INVITES_PER_HOUR + ), + retry_after_seconds: retry_after, + }), + )); + } // Check rate limit match state.rate_limiter.check_and_record(client_ip) { @@ -403,6 +450,14 @@ async fn create_room_invite( } } + // The per-IP check passed, so this request is being served -- charge it to + // the shared Tor ceiling. Done here rather than alongside `has_capacity()` + // so that a request rejected by the per-IP limiter never consumes shared + // capacity that ordinary Tor users are relying on. + if via_tor { + state.tor_bucket.record(); + } + // Generate invite match invite::create_invitation(&state.room_owner_vk, &state.inviter_signing_key) { Ok(invite_code) => { diff --git a/rust/api/src/tor.rs b/rust/api/src/tor.rs new file mode 100644 index 00000000..33014374 --- /dev/null +++ b/rust/api/src/tor.rs @@ -0,0 +1,439 @@ +//! Tor exit-node awareness for the invite rate limiter. +//! +//! # Why this exists +//! +//! gkapi's per-IP invite limit ([`crate::rate_limit::MAX_INVITES_PER_WINDOW`]) +//! is structurally defeated by Tor circuit rotation: every new exit node +//! presents a fresh IP, and therefore a fresh bucket. Measured on 2026-07-25, +//! a single actor pulled **152 invites in ~20 minutes across 113 distinct exit +//! IPs**, and not one of those IPs came close to the per-IP limit. Tightening +//! the per-IP number does nothing about this — rotation simply routes around +//! it, whatever the number is. +//! +//! Because the Tor exit set is *enumerable*, the fix is to stop treating exits +//! as independent identities and meter them as one shared bucket (see +//! [`crate::rate_limit::AggregateBucket`]). Rotation then buys the attacker +//! nothing at all. This module supplies the membership test that makes that +//! possible. +//! +//! # Why not simply block Tor +//! +//! Measured over the same 23-hour window, Tor accounts for ~4.8% of invite +//! traffic *outside* the attack burst — 69 requests from 56 distinct exits, +//! nearly all of them 1-2 requests each, i.e. ordinary usage. Freenet is a +//! privacy project and the quickstart is the main onboarding path, so blocking +//! that traffic outright costs real users. A shared bucket sized above the +//! observed organic peak (11/hour) leaves those users untouched while capping +//! a rotation burst hard. +//! +//! # Failure policy: FAIL OPEN +//! +//! If the exit list cannot be fetched and no cached copy exists, [`TorExitList`] +//! is simply empty, [`TorExitList::is_exit`] returns `false` for everything, and +//! gkapi behaves exactly as it did before this module existed. Losing the list +//! must never turn into blocking legitimate users. It must equally never be +//! read as "everything is Tor" — hence a plain `HashSet` membership test with +//! no sentinel/unknown state. + +use chrono::{DateTime, Utc}; +use log::{info, warn}; +use std::collections::HashSet; +use std::net::IpAddr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; +use thiserror::Error; + +/// Authoritative bulk exit list published by the Tor Project. +/// +/// Plain text, one IP per line. This is the list TorDNSEL is built from and is +/// the canonical source for "is this address a Tor exit". +pub const TOR_BULK_EXIT_LIST_URL: &str = "https://check.torproject.org/torbulkexitlist"; + +/// How often the list is re-fetched. Exits churn on the order of hours, so +/// hourly keeps us close enough without hammering the Tor Project. +pub const REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 60); + +/// Refuse a response larger than this. The real list is well under 1 MB; this +/// only exists so a corrupted or hostile response cannot exhaust memory. +const MAX_RESPONSE_BYTES: usize = 4 * 1024 * 1024; + +/// Hard cap on retained entries, for the same reason as `MAX_RESPONSE_BYTES`. +/// The real list is ~2000 exits, so this is ~50x headroom. +const MAX_EXITS: usize = 100_000; + +/// Beyond this age the cached list is still USED (stale data beats no data for +/// a rate-limit hint) but is logged as stale, because decommissioned exits can +/// be reassigned to ordinary users and would then be metered as Tor. +const STALE_AFTER_HOURS: i64 = 24; + +/// Network timeout for a single refresh attempt. +const FETCH_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Error, Debug)] +pub enum TorListError { + #[error("HTTP error: {0}")] + Http(#[from] reqwest::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("response too large: {0} bytes exceeds cap of {MAX_RESPONSE_BYTES}")] + TooLarge(usize), + #[error("response contained no parseable exit addresses")] + Empty, + #[error("lock poisoned")] + Lock, +} + +#[derive(Default)] +struct Snapshot { + exits: HashSet, + updated: Option>, +} + +/// A periodically-refreshed set of Tor exit-node addresses. +pub struct TorExitList { + inner: RwLock, + /// Where the last good copy is persisted, so a restart does not start blind. + cache_path: Option, + url: String, +} + +impl TorExitList { + /// Build a list, seeding from the on-disk cache if one is present. + /// + /// Never fails: an unreadable or corrupt cache just yields an empty set + /// (fail open), which is logged. + pub fn new(cache_path: Option) -> Self { + Self::with_url(cache_path, TOR_BULK_EXIT_LIST_URL.to_string()) + } + + /// As [`Self::new`], with an overridable source URL (used by tests). + pub fn with_url(cache_path: Option, url: String) -> Self { + let mut snapshot = Snapshot::default(); + + if let Some(path) = cache_path.as_deref() { + match Self::read_cache(path) { + Ok(Some((exits, updated))) => { + info!( + "Loaded {} Tor exit addresses from cache at {}", + exits.len(), + path.display() + ); + snapshot = Snapshot { + exits, + updated: Some(updated), + }; + } + Ok(None) => { + info!( + "No Tor exit cache at {} yet; starting empty (fail open)", + path.display() + ); + } + Err(e) => { + warn!( + "Could not read Tor exit cache at {}: {e}; starting empty (fail open)", + path.display() + ); + } + } + } + + Self { + inner: RwLock::new(snapshot), + cache_path, + url, + } + } + + /// Is `ip` a known Tor exit node? + /// + /// Returns `false` when the list is empty or unavailable — see the + /// fail-open policy in the module docs. + pub fn is_exit(&self, ip: &IpAddr) -> bool { + match self.inner.read() { + Ok(snap) => snap.exits.contains(ip), + Err(_) => { + warn!("Tor exit list lock poisoned; treating as non-Tor (fail open)"); + false + } + } + } + + /// Number of known exits (0 when unavailable). + pub fn len(&self) -> usize { + self.inner.read().map(|s| s.exits.len()).unwrap_or(0) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// When the list was last successfully refreshed or loaded. + pub fn last_updated(&self) -> Option> { + self.inner.read().ok().and_then(|s| s.updated) + } + + /// True when the list is older than [`STALE_AFTER_HOURS`]. Stale lists are + /// still used; this is for logging and operator visibility. + pub fn is_stale(&self) -> bool { + match self.last_updated() { + Some(t) => Utc::now() - t > chrono::Duration::hours(STALE_AFTER_HOURS), + None => true, + } + } + + /// Fetch the list and atomically replace the in-memory set. + /// + /// On any error the previous set is left untouched, so a transient outage + /// degrades to "keep using the last good list" rather than to an empty one. + pub async fn refresh(&self) -> Result { + let client = reqwest::Client::builder().timeout(FETCH_TIMEOUT).build()?; + + let response = client.get(&self.url).send().await?.error_for_status()?; + + // Reject an oversized body up front when the server declares a length. + if let Some(len) = response.content_length() { + if len as usize > MAX_RESPONSE_BYTES { + return Err(TorListError::TooLarge(len as usize)); + } + } + + let body = response.text().await?; + // ...and again after the fact, since content-length is advisory. + if body.len() > MAX_RESPONSE_BYTES { + return Err(TorListError::TooLarge(body.len())); + } + + let exits = Self::parse(&body); + if exits.is_empty() { + // Never let a garbage 200 wipe a good list. + return Err(TorListError::Empty); + } + + let now = Utc::now(); + let count = exits.len(); + + if let Some(path) = self.cache_path.as_deref() { + if let Err(e) = Self::write_cache(path, &body) { + // A cache we cannot persist is survivable; the in-memory set is + // what actually gates requests. + warn!( + "Could not persist Tor exit cache to {}: {e}", + path.display() + ); + } + } + + { + let mut snap = self.inner.write().map_err(|_| TorListError::Lock)?; + snap.exits = exits; + snap.updated = Some(now); + } + + Ok(count) + } + + /// Parse the bulk exit list: one address per line, `#` comments and blank + /// lines ignored, unparseable lines skipped. Handles IPv4 and IPv6. + fn parse(body: &str) -> HashSet { + let mut out = HashSet::new(); + for line in body.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Ok(ip) = line.parse::() { + out.insert(ip); + if out.len() >= MAX_EXITS { + warn!("Tor exit list hit the {MAX_EXITS} entry cap; truncating"); + break; + } + } + } + out + } + + fn read_cache(path: &Path) -> Result, DateTime)>, TorListError> { + if !path.exists() { + return Ok(None); + } + let content = std::fs::read_to_string(path)?; + let exits = Self::parse(&content); + if exits.is_empty() { + return Ok(None); + } + let updated: DateTime = std::fs::metadata(path)? + .modified() + .map(DateTime::::from) + .unwrap_or_else(|_| Utc::now()); + Ok(Some((exits, updated))) + } + + fn write_cache(path: &Path, body: &str) -> Result<(), TorListError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + // Write-then-rename so a crash mid-write cannot leave a truncated list + // that would silently shrink the exit set on next start. + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, body)?; + std::fs::rename(&tmp, path)?; + Ok(()) + } +} + +/// Refresh `list` immediately, then every [`REFRESH_INTERVAL`], forever. +/// +/// Errors are logged and retried on the next tick; they never abort the loop +/// and never clear the current set. +pub fn spawn_refresher(list: Arc) { + if list.is_empty() { + info!("Tor exit list starting empty; shared Tor ceiling inactive until first refresh"); + } else { + info!( + "Tor exit list seeded with {} entries from cache (last updated: {})", + list.len(), + list.last_updated() + .map(|t| t.to_rfc3339()) + .unwrap_or_else(|| "unknown".to_string()) + ); + } + + tokio::spawn(async move { + loop { + match list.refresh().await { + Ok(n) => info!("Refreshed Tor exit list: {n} exit addresses"), + Err(e) => warn!( + "Tor exit list refresh failed: {e} (using {} cached entries, last updated {}{}; \ + Tor traffic is metered per-IP only until this succeeds)", + list.len(), + list.last_updated() + .map(|t| t.to_rfc3339()) + .unwrap_or_else(|| "never".to_string()), + if list.is_stale() { ", STALE" } else { "" } + ), + } + tokio::time::sleep(REFRESH_INTERVAL).await; + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr}; + use tempfile::tempdir; + + #[test] + fn parses_ipv4_ipv6_and_ignores_junk() { + let body = "\ +# comment line +185.220.101.30 + +192.42.116.15 +2001:db8::1 +not-an-ip + 171.25.193.78 +"; + let set = TorExitList::parse(body); + assert_eq!(set.len(), 4, "expected 4 parseable addresses, got {set:?}"); + assert!(set.contains(&IpAddr::V4(Ipv4Addr::new(185, 220, 101, 30)))); + assert!(set.contains(&IpAddr::V4(Ipv4Addr::new(192, 42, 116, 15)))); + assert!(set.contains(&IpAddr::V6("2001:db8::1".parse::().unwrap()))); + // Surrounding whitespace must not defeat the match. + assert!(set.contains(&IpAddr::V4(Ipv4Addr::new(171, 25, 193, 78)))); + } + + /// Live end-to-end check against the real Tor Project list. + /// + /// `#[ignore]`d so CI stays hermetic; run explicitly before deploying a + /// change to the fetch/parse path: + /// `cargo test --bins -- --ignored refresh_fetches_real_tor_exit_list --nocapture` + /// + /// Asserts on shape rather than an exact count (the list churns): a + /// plausible number of exits, and that a few long-lived exit ranges observed + /// in the 2026-07-25 burst are recognised. + #[tokio::test] + #[ignore] + async fn refresh_fetches_real_tor_exit_list() { + let dir = tempdir().unwrap(); + let cache = dir.path().join("tor_exits.txt"); + let list = TorExitList::new(Some(cache.clone())); + + let n = list.refresh().await.expect("refresh should succeed"); + println!("fetched {n} Tor exit addresses"); + + assert!( + (500..MAX_EXITS).contains(&n), + "expected a plausible exit count, got {n}" + ); + assert_eq!(list.len(), n); + assert!(!list.is_stale(), "freshly refreshed list must not be stale"); + assert!(cache.exists(), "refresh must persist the cache"); + + // A refreshed list must round-trip through the cache unchanged. + let reloaded = TorExitList::new(Some(cache)); + assert_eq!(reloaded.len(), n, "cache reload must preserve the set"); + + // Sanity: a non-Tor address must not be flagged. + assert!(!list.is_exit(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)))); + } + + /// The fail-open contract: with no list, nothing is Tor. + #[test] + fn empty_list_treats_everything_as_non_tor() { + let list = TorExitList::new(None); + assert!(list.is_empty()); + assert!(!list.is_exit(&IpAddr::V4(Ipv4Addr::new(185, 220, 101, 30)))); + assert!(!list.is_exit(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)))); + assert!(list.is_stale(), "a never-populated list counts as stale"); + } + + #[test] + fn seeds_from_cache_on_construction() { + let dir = tempdir().unwrap(); + let path = dir.path().join("tor_exits.txt"); + std::fs::write(&path, "185.220.101.30\n192.42.116.15\n").unwrap(); + + let list = TorExitList::new(Some(path)); + assert_eq!(list.len(), 2); + assert!(list.is_exit(&IpAddr::V4(Ipv4Addr::new(185, 220, 101, 30)))); + assert!(!list.is_exit(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)))); + assert!(list.last_updated().is_some()); + } + + /// A corrupt cache must not panic or wedge startup -- it degrades to empty. + #[test] + fn corrupt_cache_fails_open() { + let dir = tempdir().unwrap(); + let path = dir.path().join("tor_exits.txt"); + std::fs::write(&path, "this is not\nan ip list at all\n").unwrap(); + + let list = TorExitList::new(Some(path)); + assert!(list.is_empty()); + assert!(!list.is_exit(&IpAddr::V4(Ipv4Addr::new(185, 220, 101, 30)))); + } + + #[test] + fn missing_cache_path_fails_open() { + let dir = tempdir().unwrap(); + let list = TorExitList::new(Some(dir.path().join("does-not-exist.txt"))); + assert!(list.is_empty()); + } + + #[test] + fn cache_write_is_atomic_and_reloadable() { + let dir = tempdir().unwrap(); + let path = dir.path().join("nested").join("tor_exits.txt"); + TorExitList::write_cache(&path, "185.220.101.30\n").unwrap(); + assert!(path.exists()); + assert!( + !path.with_extension("tmp").exists(), + "temp file must be renamed away, not left behind" + ); + + let reloaded = TorExitList::new(Some(path)); + assert_eq!(reloaded.len(), 1); + } +} From 65234de7913e79c17a92feaaddc5df9a5ee3c1e5 Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Sat, 25 Jul 2026 00:45:27 -0500 Subject: [PATCH 2/3] chore(gkapi): lockfile for reqwest (Tor exit-list fetch) [AI-assisted - Claude] --- rust/Cargo.lock | 73 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 85e5c94d..8b9e7e91 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -323,7 +323,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.71", "which", @@ -1083,6 +1083,7 @@ dependencies = [ "log", "rand 0.8.5", "rand_core 0.6.4", + "reqwest", "rmp-serde", "serde", "serde_json", @@ -1414,6 +1415,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -1997,6 +1999,54 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.3", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring", + "rustc-hash 2.1.3", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -2143,7 +2193,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2151,11 +2204,13 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] @@ -2229,6 +2284,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2259,6 +2320,7 @@ checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3074,6 +3136,15 @@ dependencies = [ "url", ] +[[package]] +name = "webpki-roots" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" From 6c7e49646fb857f9d1120a921a43e3dd4782af68 Mon Sep 17 00:00:00 2001 From: Ian Clarke Date: Sat, 25 Jul 2026 01:05:51 -0500 Subject: [PATCH 3/3] fix(gkapi): address review findings; install rustls provider, atomic ceiling Five reviewers (4 Claude perspectives + Codex) found 20+ issues. The critical one would have taken gkapi down on deploy. ## CRITICAL: HTTPS startup panic (would have caused an outage) `reqwest`'s `rustls-tls` enables `rustls/ring`; axum-server's `tls-rustls` enables `rustls/aws-lc-rs`. With BOTH provider features on, rustls 0.23's `CryptoProvider::from_crate_features()` returns None by design and TLS construction panics: no process-level CryptoProvider available Confirmed empirically: the binary exits 101 before binding :443, taking down donations and cert-signing too, not just invites. No unit test could catch it (none construct a RustlsConfig) and `cargo run` without --tls-cert takes the plain-HTTP branch. Fix: install the aws-lc-rs provider explicitly in main(), preserving the provider axum-server used before reqwest existed. Pinned by a new tests/https_startup.rs that spawns the real binary with real TLS material and makes a real HTTPS request -- verified to FAIL without the fix and pass with it. ## Sizing was wrong: 25/hour would have refused real users The original ceiling came from a smaller sample filtered by MAGNITUDE (hours above a threshold excluded as "burst", then the remaining max called the organic peak) -- circular, and it understated the peak 3x. Re-derived from one dataset excluding the burst window BY TIME: organic Tor peaks at 33/hour (mean 11.9), not 11. Default raised to 60 and now overridable at runtime. ## Honest reframing The claim "ordinary Tor users are unaffected" was false and is removed. Sharing a budget across an anonymity set means the loudest member takes it: an attacker sustaining the ceiling denies invites to every Tor user, and since refused requests cost nothing, a poller beats a human to each freed slot. Inherent to any aggregate cap over an unlinkable pool; closing it needs proof-of-work (#81). This is rate-shaping that buys time, not a fix. ## Other fixes - AggregateBucket: `try_acquire()` is now the atomic admission authority (check+record under one lock), so the ceiling holds under any concurrency and stays correct if an `.await` is later introduced. Pinned by a 16-thread contention test. - Monotonic `Instant` instead of wall clock: an NTP step backwards would have stalled pruning and pinned the bucket full, denying all Tor users. - Runtime off-switch: `--tor-invites-per-hour` / `TOR_INVITES_PER_HOUR`, `0` disables. Previously required a rebuild to disable or resize. - Tor slot released when invite generation fails, so a generation-failure burst no longer locks out Tor users on top of the outage. - IPv4-mapped IPv6 canonicalised: if the bind ever changes to `::`, every exit would otherwise silently escape metering. - Fetch streams with a running cap; `response.text()` buffered the whole body BEFORE the size check, so a chunked response could OOM a small VM. - List validation: "non-empty" was insufficient -- a truncated 200 or an HTML error page containing one IP could replace a good ~1400-entry list with a handful and silently disable metering. Now requires a plausible minimum and rejects implausible shrinkage. - Refresh backoff (30s/1m/5m/15m) while the list is empty; previously one transient failure at startup disabled the ceiling for a full hour. - Refresher panic is now logged at error!; previously it would freeze the exit list forever, silently. - Exhaustion logged at warn! not info! -- it means real users are being refused. - 429 copy no longer advises privacy users to stop using Tor, and no longer states the exact ceiling (which told an attacker what budget to drain). - Per-IP 429 message interpolates the constant instead of hardcoding a stale 4. - river-invite-button.html rendered every sub-hour retry as "approximately 1 hour(s)". The Tor window is 60 min, so this PR made that routinely wrong; now renders seconds/minutes/hours, and treats 0 as a value rather than absent. - Cargo.toml comment corrected: it claimed to avoid openssl, which was false (integration_test pulls native-tls in via feature unification). ## Testing 35 unit tests + 1 HTTPS startup integration test. New handler tests close the gap every reviewer flagged: nothing previously exercised the wiring, so dropping try_acquire or reordering it past the per-IP check left the suite green and the limiter defeated. Mutation-verified -- both regressions are caught by 4 tests each. New CI job (rust-api-tests.yml): CI never built rust/api at all, so none of these tests ran and a compile error would merge green. Refs #81 [AI-assisted - Claude] --- .github/workflows/rust-api-tests.yml | 65 ++++ .../shortcodes/river-invite-button.html | 22 +- rust/Cargo.lock | 2 + rust/api/Cargo.toml | 13 +- rust/api/src/main.rs | 57 ++++ rust/api/src/rate_limit.rs | 306 +++++++++++++----- rust/api/src/routes.rs | 272 ++++++++++++++-- rust/api/src/tor.rs | 180 ++++++++--- rust/api/tests/https_startup.rs | 148 +++++++++ 9 files changed, 916 insertions(+), 149 deletions(-) create mode 100644 .github/workflows/rust-api-tests.yml create mode 100644 rust/api/tests/https_startup.rs diff --git a/.github/workflows/rust-api-tests.yml b/.github/workflows/rust-api-tests.yml new file mode 100644 index 00000000..0fb48fea --- /dev/null +++ b/.github/workflows/rust-api-tests.yml @@ -0,0 +1,65 @@ +# Compile and test the Rust API crate. +# +# Why this exists: before it, CI never built `rust/api` at all. `deploy.yml` +# runs `cargo make build-site` (wasm + hugo only), and `integration-test.yml` +# is `workflow_dispatch:` only. So a compile error in `rust/api` -- or any +# failing unit test in it -- would merge green and only surface when someone +# ran the deploy script on vega. +# +# That gap let a real bug through: adding `reqwest` with `rustls-tls` enabled a +# second rustls provider feature alongside axum-server's, which made gkapi panic +# at startup in HTTPS mode ("no process-level CryptoProvider available"). Unit +# tests could not catch it because none of them construct a TLS config, hence +# the https_startup integration test, which this job runs. +name: Rust API Tests + +on: + pull_request: + paths: + - "rust/**" + - ".github/workflows/rust-api-tests.yml" + push: + branches: [main] + paths: + - "rust/**" + - ".github/workflows/rust-api-tests.yml" + +concurrency: + group: rust-api-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + + - name: Formatting + working-directory: rust + run: cargo fmt -p ghostkey-api -- --check + + - name: Build + working-directory: rust + run: cargo build -p ghostkey-api + + # Unit tests plus the HTTPS startup smoke test. The latter spawns the real + # binary with a self-signed cert and makes a real HTTPS request, because + # the TLS provider failure mode is invisible to any in-process test. + # + # Network-dependent tests are #[ignore]d and deliberately NOT run here -- + # the live Tor Project fetch would make this job flaky. Run them manually: + # cargo test --bins -- --ignored + - name: Test + working-directory: rust + run: cargo test -p ghostkey-api diff --git a/hugo-site/themes/freenet/layouts/shortcodes/river-invite-button.html b/hugo-site/themes/freenet/layouts/shortcodes/river-invite-button.html index bc0ff6e3..2b5c36e1 100644 --- a/hugo-site/themes/freenet/layouts/shortcodes/river-invite-button.html +++ b/hugo-site/themes/freenet/layouts/shortcodes/river-invite-button.html @@ -209,9 +209,25 @@ inviteLoading.style.display = 'none'; inviteError.style.display = 'block'; errorMessage.textContent = data.error; - if (data.retry_after_seconds) { - const hours = Math.ceil(data.retry_after_seconds / 3600); - retryMessage.textContent = `You can try again in approximately ${hours} hour(s).`; + // Retry hints can now be sub-hour: the shared Tor ceiling uses a + // 60-minute window, so values arrive anywhere in 0..3600s. + // Rounding everything up to hours told a user waiting 90 seconds + // to come back in "approximately 1 hour(s)". + // Note `!= null` rather than truthiness, so a legitimate 0 is + // rendered instead of leaving a stale message from a prior try. + if (data.retry_after_seconds != null) { + const secs = Math.max(0, Math.round(data.retry_after_seconds)); + let when; + if (secs < 60) { + when = `${Math.max(1, secs)} second(s)`; + } else if (secs < 3600) { + when = `${Math.ceil(secs / 60)} minute(s)`; + } else { + when = `${Math.ceil(secs / 3600)} hour(s)`; + } + retryMessage.textContent = `You can try again in approximately ${when}.`; + } else { + retryMessage.textContent = ''; } return; } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8b9e7e91..25f1fee8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1085,6 +1085,7 @@ dependencies = [ "rand_core 0.6.4", "reqwest", "rmp-serde", + "rustls", "serde", "serde_json", "sha2 0.10.8", @@ -2319,6 +2320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", diff --git a/rust/api/Cargo.toml b/rust/api/Cargo.toml index 46f92ca1..03b06b43 100644 --- a/rust/api/Cargo.toml +++ b/rust/api/Cargo.toml @@ -32,9 +32,18 @@ serde = { version = "1.0", features = ["derive"] } blind-rsa-signatures = "0.15.1" ciborium = "0.2" bs58 = "0.5" -# Tor exit-list fetch. rustls (not native-tls) to match axum-server's TLS stack -# and avoid pulling in openssl. +# Tor exit-list fetch. +# +# NOTE: `rustls-tls` enables `rustls/ring`, while axum-server's `tls-rustls` +# enables `rustls/aws-lc-rs`. With BOTH provider features on, rustls 0.23's +# `CryptoProvider::from_crate_features()` returns None and TLS startup panics +# with "no process-level CryptoProvider available". main() therefore installs +# the aws-lc-rs provider explicitly before any TLS is constructed -- see +# `install_crypto_provider()` and `https_mode_starts_without_crypto_provider_panic`. +# Do NOT remove that call, and do NOT assume unit tests cover it: only an +# actual HTTPS-mode start exercises this path. reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +rustls = { version = "0.23", features = ["aws-lc-rs"] } [dev-dependencies] tempfile = "3" diff --git a/rust/api/src/main.rs b/rust/api/src/main.rs index 0e24810a..90153495 100644 --- a/rust/api/src/main.rs +++ b/rust/api/src/main.rs @@ -85,6 +85,18 @@ fn load_invite_config(matches: &clap::ArgMatches) -> Option { .map(|s| s.as_str()) .unwrap_or("/var/lib/gkapi/tor_exit_list.txt"), )); + // Runtime off-switch / tuning knob for the shared Tor ceiling. `0` disables + // it entirely, so an operator can turn this off (or resize it) without a + // rebuild if it turns out to refuse legitimate users. + let tor_invites_per_hour = matches + .get_one::("tor-invites-per-hour") + .and_then(|s| match s.parse::() { + Ok(v) => Some(v), + Err(e) => { + error!("Invalid --tor-invites-per-hour {s:?}: {e}; using default"); + None + } + }); // Load signing key from file (32 bytes raw) let signing_key_bytes = match fs::read(signing_key_path) { @@ -143,14 +155,49 @@ fn load_invite_config(matches: &clap::ArgMatches) -> Option { Some(InviteState::new( rate_limit_file, tor_exit_cache, + tor_invites_per_hour, room_owner_vk, inviter_signing_key, room_name, )) } +/// Install the process-wide rustls crypto provider. +/// +/// REQUIRED, and load-bearing: this crate ends up with BOTH `rustls/aws-lc-rs` +/// (via axum-server's `tls-rustls`) and `rustls/ring` (via reqwest's +/// `rustls-tls`) enabled. When both provider features are on, rustls 0.23's +/// `CryptoProvider::from_crate_features()` returns `None` on purpose, and the +/// first TLS construction panics with: +/// +/// no process-level CryptoProvider available -- call +/// CryptoProvider::install_default() before this point +/// +/// That panic is on the main task, so gkapi dies at startup in HTTPS mode and +/// takes the donation and cert-signing endpoints down with it -- not just +/// invites. It is invisible to `cargo test`: nothing in the test suite builds +/// an `axum-server` `RustlsConfig`, and `cargo run` without `--tls-cert` +/// takes the plain-HTTP branch. Only an actual HTTPS start reaches it. +/// +/// aws-lc-rs is chosen to preserve the provider axum-server used before +/// reqwest was introduced. Pinned by +/// `https_mode_starts_without_crypto_provider_panic` in tests/https_startup.rs. +fn install_crypto_provider() { + if rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .is_err() + { + // Already installed (e.g. a second call in a test process). Not fatal: + // the invariant we need is "a provider exists", not "we installed it". + warn!("rustls crypto provider was already installed"); + } +} + #[tokio::main] async fn main() { + // MUST run before anything constructs a TLS config. See the function docs. + install_crypto_provider(); + // Pre-scan argv for the legacy --delegate-dir spelling so we can emit a // deprecation warning before clap normalizes it to the canonical name. // (See the same pattern in rust/cli/src/bin/ghostkey.rs.) @@ -257,6 +304,16 @@ async fn main() { .default_value("/var/lib/gkapi/tor_exit_list.txt") .help("Path to the cached Tor exit-node list (refreshed hourly)"), ) + .arg( + Arg::new("tor-invites-per-hour") + .long("tor-invites-per-hour") + .value_name("N") + .env("TOR_INVITES_PER_HOUR") + .help( + "Shared hourly invite ceiling across ALL Tor exits (0 disables). \ + Defaults to DEFAULT_TOR_INVITES_PER_HOUR.", + ), + ) .get_matches(); let notary_dir = matches.get_one::("notary-dir").unwrap(); diff --git a/rust/api/src/rate_limit.rs b/rust/api/src/rate_limit.rs index 75e3a6b6..c211a6bf 100644 --- a/rust/api/src/rate_limit.rs +++ b/rust/api/src/rate_limit.rs @@ -10,6 +10,7 @@ use std::fs; use std::net::IpAddr; use std::path::PathBuf; use std::sync::Mutex; +use std::time::{Duration as StdDuration, Instant}; use thiserror::Error; /// Maximum number of invites allowed per IP within the time window. @@ -26,17 +27,34 @@ use thiserror::Error; /// [`TOR_INVITES_PER_HOUR`]. pub const MAX_INVITES_PER_WINDOW: usize = 4; -/// Shared hourly invite ceiling across the ENTIRE Tor exit set. +/// Default shared hourly invite ceiling across the ENTIRE Tor exit set. /// -/// Sizing (measured 2026-07-24/25 from `invite_rate_limits.json`): organic Tor -/// traffic peaked at **11 invites/hour** (mean 6.8) across 6 non-burst hours, -/// while the abuse burst hit **158/hour**. 25 leaves 2.3x headroom over the -/// worst observed organic hour, so ordinary Tor users are unaffected, while -/// capping a rotation burst at ~16% of what it achieved. See -/// `tor_bucket_admits_organic_peak_but_caps_burst`. -pub const TOR_INVITES_PER_HOUR: usize = 25; - -/// Window for [`TOR_INVITES_PER_HOUR`], in minutes. +/// # Sizing, and why this number is uncomfortable +/// +/// From `invite_rate_limits.json` (1728 invites, 2026-07-24T06:23Z.. +/// 2026-07-25T05:54Z), classified against the official bulk exit list: +/// +/// | Tor invites/hour | | +/// |---|---| +/// | organic hours (burst window excluded BY TIME, n=7) | mean 11.9, **max 33** | +/// | burst hour 02:00Z | **208** | +/// +/// An earlier revision used 25, derived from a smaller sample filtered by +/// MAGNITUDE (hours above a threshold were excluded as "burst", then the +/// remaining max was called the organic peak). That is circular, and it +/// understated the peak by 3x. 25 would have refused real users. +/// +/// 60 sits above the observed organic max (33) with ~1.8x headroom and still +/// cuts the observed burst hour by ~71%. It is a judgement call on ambiguous +/// data, not a derived constant: hours 04Z (28 invites / 25 exits) and 05Z +/// (33 / 33) are ~1.0 invites per exit, which looks like many distinct real +/// users -- but a one-request-per-exit attacker is indistinguishable from that +/// by volume alone. Treat this as a starting value to tune from telemetry, +/// which is why it is overridable at runtime (`--tor-invites-per-hour` / +/// `TOR_INVITES_PER_HOUR`, `0` disables the ceiling entirely). +pub const DEFAULT_TOR_INVITES_PER_HOUR: usize = 60; + +/// Window for the Tor ceiling, in minutes. pub const TOR_WINDOW_MINUTES: i64 = 60; /// SHA256 hashes of IPs exempt from rate limiting (for testing) @@ -193,32 +211,60 @@ impl RateLimiter { /// Per-IP limiting assumes an IP approximates a person. For Tor that assumption /// is false in the attacker's favour: exits are a public, rotatable pool, so N /// exits multiply any per-IP limit by N. Metering the whole pool through ONE -/// bucket removes the multiplier entirely — rotating costs the attacker nothing -/// and gains them nothing. +/// bucket removes the multiplier. +/// +/// # The cost, stated plainly +/// +/// Sharing a budget across an anonymity set means the loudest member can +/// consume the whole allowance. An attacker who sustains the ceiling denies +/// invites to EVERY Tor user for as long as they keep it up, and refused +/// requests cost them nothing, so an automated poller wins each freed slot +/// against a human. This is inherent to any aggregate cap over an unlinkable +/// pool, not an implementation defect -- fixing it needs a per-request cost +/// signal (proof-of-work / CAPTCHA), tracked in freenet/web#81. Do not describe +/// this bucket as leaving ordinary Tor users unaffected: that is only true +/// while nobody is attacking. /// /// Deliberately in-memory (not persisted like [`RateLimiter`]): a restart -/// forgives at most [`TOR_INVITES_PER_HOUR`] requests, gkapi restarts only on -/// deploy, and this keeps the hot path free of the read-modify-write file IO -/// the per-IP limiter does. +/// forgives at most one window's worth, gkapi restarts only on deploy, and this +/// keeps the hot path free of the read-modify-write file IO the per-IP limiter +/// does. +/// +/// Uses a MONOTONIC clock ([`Instant`]) rather than wall time: an NTP step +/// backwards would otherwise stall pruning and pin the bucket at full, denying +/// every Tor user until the clock caught up. [`RateLimiter`] genuinely needs +/// wall time because it persists; this does not. pub struct AggregateBucket { limit: usize, - window: Duration, - hits: Mutex>>, + window: StdDuration, + hits: Mutex>, } impl AggregateBucket { pub fn new(limit: usize, window_minutes: i64) -> Self { Self { limit, - window: Duration::minutes(window_minutes), + window: StdDuration::from_secs((window_minutes.max(0) as u64) * 60), hits: Mutex::new(VecDeque::new()), } } + /// A limit of 0 disables the ceiling entirely (runtime off-switch). + pub fn is_disabled(&self) -> bool { + self.limit == 0 + } + + pub fn limit(&self) -> usize { + self.limit + } + /// Drop hits that have aged out of the window. - fn prune(&self, hits: &mut VecDeque>, now: DateTime) { + /// + /// `hits` is append-only with the clock sampled UNDER the lock, so it is + /// sorted ascending and the front is always the oldest. + fn prune(&self, hits: &mut VecDeque, now: Instant) { while let Some(front) = hits.front() { - if now - *front >= self.window { + if now.duration_since(*front) >= self.window { hits.pop_front(); } else { break; @@ -226,49 +272,87 @@ impl AggregateBucket { } } - /// Is there room in the window right now? + /// Atomically take a slot if one is free. /// - /// Checked separately from [`Self::record`] so a caller can reject a request - /// BEFORE spending the requester's per-IP allowance on it (see the ordering - /// note in `routes::create_room_invite`). The gap between the two admits a - /// small overshoot under concurrent load, bounded by the number of requests - /// in flight; for an anti-abuse ceiling that is not worth a global lock. - pub fn has_capacity(&self) -> bool { - let now = Utc::now(); + /// This is the ADMISSION AUTHORITY -- check and record happen under a + /// single lock acquisition, so the ceiling holds no matter how many + /// requests race, and stays correct if an `.await` is ever introduced + /// between the pre-check and here. Returns false when the window is full. + pub fn try_acquire(&self) -> bool { + if self.is_disabled() { + return true; + } + let now = Instant::now(); match self.hits.lock() { Ok(mut hits) => { self.prune(&mut hits, now); - hits.len() < self.limit + if hits.len() < self.limit { + hits.push_back(now); + true + } else { + false + } } // Fail open: a poisoned lock must not block legitimate users. Err(_) => true, } } - /// Record one hit against the window. - pub fn record(&self) { - let now = Utc::now(); + /// Give back a slot taken by [`Self::try_acquire`] for a request that was + /// not ultimately served. + /// + /// Removes the newest entry rather than the specific one acquired. Since + /// every entry in flight is within microseconds of the others and only the + /// COUNT is meaningful, that is equivalent, and it avoids handing out + /// tokens just to identify which entry to drop. + pub fn release(&self) { + if self.is_disabled() { + return; + } if let Ok(mut hits) = self.hits.lock() { - self.prune(&mut hits, now); - hits.push_back(now); + hits.pop_back(); + } + } + + /// Cheap non-consuming pre-check. + /// + /// Used only to reject early WITHOUT spending the requester's per-IP + /// allowance. It is advisory: [`Self::try_acquire`] is what actually + /// enforces the ceiling. + pub fn has_capacity(&self) -> bool { + if self.is_disabled() { + return true; + } + let now = Instant::now(); + match self.hits.lock() { + Ok(mut hits) => { + self.prune(&mut hits, now); + hits.len() < self.limit + } + Err(_) => true, } } /// Seconds until the window has room again, or `None` if it has room now. pub fn retry_after_seconds(&self) -> Option { - let now = Utc::now(); + if self.is_disabled() { + return None; + } + let now = Instant::now(); let mut hits = self.hits.lock().ok()?; self.prune(&mut hits, now); if hits.len() < self.limit { return None; } - hits.front() - .map(|oldest| (*oldest + self.window - now).num_seconds().max(0)) + hits.front().map(|oldest| { + let elapsed = now.duration_since(*oldest); + self.window.saturating_sub(elapsed).as_secs() as i64 + }) } /// Current occupancy (for logging / diagnostics). pub fn current(&self) -> usize { - let now = Utc::now(); + let now = Instant::now(); match self.hits.lock() { Ok(mut hits) => { self.prune(&mut hits, now); @@ -289,81 +373,135 @@ mod tests { fn aggregate_bucket_admits_up_to_limit_then_refuses() { let bucket = AggregateBucket::new(3, 60); for i in 1..=3 { - assert!(bucket.has_capacity(), "hit {i} should have capacity"); - bucket.record(); + assert!(bucket.try_acquire(), "hit {i} should be admitted"); } - assert!(!bucket.has_capacity(), "4th hit must be refused"); + assert!(!bucket.try_acquire(), "4th hit must be refused"); assert_eq!(bucket.current(), 3); assert!(bucket.retry_after_seconds().is_some()); } /// The whole point: many distinct identities share ONE budget, so rotating - /// between them gains nothing. + /// between them gains nothing. (Identity-blindness is what makes this true; + /// the end-to-end proof across real rotating IPs lives in the handler tests + /// in `routes.rs`, since the bucket itself has no concept of an IP.) #[test] fn aggregate_bucket_is_not_per_identity() { let bucket = AggregateBucket::new(2, 60); - // Simulate three different "IPs" all funnelling through one bucket. - bucket.record(); // exit A - bucket.record(); // exit B + assert!(bucket.try_acquire()); // exit A + assert!(bucket.try_acquire()); // exit B assert!( - !bucket.has_capacity(), + !bucket.try_acquire(), "a third distinct exit must NOT get its own allowance" ); } - /// Sizing regression: the deployed ceiling must stay above the measured - /// organic Tor peak (11/hour) and far below the observed burst (158/hour). + /// `try_acquire` must be the atomic admission authority: N threads racing + /// must never admit more than `limit` in total. + #[test] + fn aggregate_bucket_try_acquire_is_atomic_under_contention() { + use std::sync::Arc; + const LIMIT: usize = 25; + const THREADS: usize = 16; + const PER_THREAD: usize = 20; + + let bucket = Arc::new(AggregateBucket::new(LIMIT, 60)); + let mut handles = Vec::new(); + for _ in 0..THREADS { + let b = Arc::clone(&bucket); + handles.push(std::thread::spawn(move || { + (0..PER_THREAD).filter(|_| b.try_acquire()).count() + })); + } + let admitted: usize = handles.into_iter().map(|h| h.join().unwrap()).sum(); + assert_eq!( + admitted, + LIMIT, + "exactly {LIMIT} of {} racing attempts may be admitted", + THREADS * PER_THREAD + ); + assert_eq!(bucket.current(), LIMIT); + } + + #[test] + fn aggregate_bucket_release_returns_a_slot() { + let bucket = AggregateBucket::new(1, 60); + assert!(bucket.try_acquire()); + assert!(!bucket.try_acquire(), "bucket is full"); + bucket.release(); + assert_eq!(bucket.current(), 0); + assert!(bucket.try_acquire(), "released slot must be reusable"); + } + + /// A limit of 0 is the runtime off-switch: nothing is ever refused. + #[test] + fn aggregate_bucket_zero_limit_disables_the_ceiling() { + let bucket = AggregateBucket::new(0, 60); + assert!(bucket.is_disabled()); + for _ in 0..1000 { + assert!(bucket.try_acquire(), "a disabled ceiling never refuses"); + } + assert!(bucket.has_capacity()); + assert!(bucket.retry_after_seconds().is_none()); + } + + /// Sizing pin. Bounds are tied to the MEASURED organic peak, so raising the + /// ceiling far above real traffic (or dropping it below it) fails here. + /// + /// Accepted collateral, deliberately NOT claimed away by this test: while + /// an attacker holds the ceiling, ordinary Tor users are refused too. See + /// `AggregateBucket` docs and freenet/web#81. #[test] - fn tor_bucket_admits_organic_peak_but_caps_burst() { - const OBSERVED_ORGANIC_PEAK: usize = 11; - const OBSERVED_BURST: usize = 158; + #[allow(clippy::assertions_on_constants)] // deliberate: this pins the constants + fn tor_ceiling_is_sized_between_organic_peak_and_burst() { + // Measured 2026-07-24/25 against the official bulk exit list. + const OBSERVED_ORGANIC_PEAK: usize = 33; + const OBSERVED_BURST_HOUR: usize = 208; assert!( - TOR_INVITES_PER_HOUR > OBSERVED_ORGANIC_PEAK, - "ceiling {TOR_INVITES_PER_HOUR} must exceed the organic peak \ - {OBSERVED_ORGANIC_PEAK}/h or real Tor users get blocked" + DEFAULT_TOR_INVITES_PER_HOUR > OBSERVED_ORGANIC_PEAK, + "ceiling {DEFAULT_TOR_INVITES_PER_HOUR} must exceed the measured organic \ + peak {OBSERVED_ORGANIC_PEAK}/h or it refuses real users" ); assert!( - TOR_INVITES_PER_HOUR < OBSERVED_BURST / 2, - "ceiling {TOR_INVITES_PER_HOUR} must be well under the observed \ - burst {OBSERVED_BURST}/h or it does not actually bound abuse" + DEFAULT_TOR_INVITES_PER_HOUR <= 2 * OBSERVED_ORGANIC_PEAK, + "ceiling {DEFAULT_TOR_INVITES_PER_HOUR} must stay within 2x the organic \ + peak {OBSERVED_ORGANIC_PEAK}/h or it barely constrains a burst" ); + assert!( + DEFAULT_TOR_INVITES_PER_HOUR < OBSERVED_BURST_HOUR / 2, + "ceiling must be well under the observed burst {OBSERVED_BURST_HOUR}/h" + ); + } - let bucket = AggregateBucket::new(TOR_INVITES_PER_HOUR, TOR_WINDOW_MINUTES); - for _ in 0..OBSERVED_ORGANIC_PEAK { - assert!( - bucket.has_capacity(), - "organic traffic must never be refused" - ); - bucket.record(); - } - // Now replay the burst; it must be cut off at the ceiling. - let mut admitted = OBSERVED_ORGANIC_PEAK; - for _ in 0..OBSERVED_BURST { - if bucket.has_capacity() { - bucket.record(); - admitted += 1; - } + #[test] + fn aggregate_bucket_expires_old_hits_individually() { + let bucket = AggregateBucket::new(2, 60); + { + let mut hits = bucket.hits.lock().unwrap(); + // One aged out, one still inside the window. + hits.push_back(Instant::now() - StdDuration::from_secs(61 * 60)); + hits.push_back(Instant::now() - StdDuration::from_secs(30 * 60)); } - assert_eq!( - admitted, TOR_INVITES_PER_HOUR, - "burst must be capped at the ceiling, not merely slowed" - ); + assert_eq!(bucket.current(), 1, "only the expired hit is pruned"); + assert!(bucket.try_acquire(), "the freed slot is reusable"); + assert!(!bucket.try_acquire(), "and only one slot freed"); } + /// `retry_after_seconds` must reflect the OLDEST hit's expiry, not the + /// newest. Reading `back()` instead of `front()` passes a naive test. #[test] - fn aggregate_bucket_expires_old_hits() { + fn retry_after_reflects_oldest_hit() { let bucket = AggregateBucket::new(2, 60); - // Backdate both hits beyond the window. { let mut hits = bucket.hits.lock().unwrap(); - let old = Utc::now() - Duration::minutes(61); - hits.push_back(old); - hits.push_back(old); + hits.push_back(Instant::now() - StdDuration::from_secs(59 * 60)); // frees in ~60s + hits.push_back(Instant::now() - StdDuration::from_secs(60)); // frees in ~59min } - assert_eq!(bucket.current(), 0, "expired hits must be pruned"); - assert!(bucket.has_capacity()); - assert!(bucket.retry_after_seconds().is_none()); + let retry = bucket.retry_after_seconds().expect("bucket is full"); + assert!( + (30..=120).contains(&retry), + "expected ~60s from the OLDEST hit, got {retry}s (reading back() not front()?)" + ); } #[test] diff --git a/rust/api/src/routes.rs b/rust/api/src/routes.rs index 803abea0..daa6b894 100644 --- a/rust/api/src/routes.rs +++ b/rust/api/src/routes.rs @@ -13,7 +13,7 @@ use axum::{ }; use ed25519_dalek::{SigningKey, VerifyingKey}; use ghostkey_lib::armorable::Armorable; -use log::{error, info}; +use log::{error, info, warn}; use serde::{Deserialize, Serialize}; use stripe::{Client, Currency, PaymentIntent, PaymentIntentId}; @@ -22,7 +22,10 @@ use crate::handle_sign_cert::{ sign_certificate, CertificateError, SignCertificateRequest, SignCertificateResponse, }; use crate::invite; -use crate::rate_limit::{AggregateBucket, RateLimiter, TOR_INVITES_PER_HOUR, TOR_WINDOW_MINUTES}; +use crate::rate_limit::{ + AggregateBucket, RateLimiter, DEFAULT_TOR_INVITES_PER_HOUR, MAX_INVITES_PER_WINDOW, + TOR_WINDOW_MINUTES, +}; use crate::tor::TorExitList; /// Shared application state for invite generation @@ -42,6 +45,7 @@ impl InviteState { pub fn new( rate_limit_file: PathBuf, tor_exit_cache: Option, + tor_invites_per_hour: Option, room_owner_vk: VerifyingKey, inviter_signing_key: SigningKey, room_name: String, @@ -49,7 +53,7 @@ impl InviteState { Self { rate_limiter: Arc::new(RateLimiter::new(rate_limit_file, 24)), tor_bucket: Arc::new(AggregateBucket::new( - TOR_INVITES_PER_HOUR, + tor_invites_per_hour.unwrap_or(DEFAULT_TOR_INVITES_PER_HOUR), TOR_WINDOW_MINUTES, )), tor_exits: Arc::new(TorExitList::new(tor_exit_cache)), @@ -397,21 +401,22 @@ async fn create_room_invite( // never consumes shared Tor capacity either. if via_tor && !state.tor_bucket.has_capacity() { let retry_after = state.tor_bucket.retry_after_seconds(); - info!( - "Tor exit {} refused: shared Tor ceiling reached ({}/{} per hour), retry_after: {:?}", + // warn!, not info!: sustained exhaustion means legitimate Tor users are + // being refused and an operator should see it. + warn!( + "Tor exit {} refused: shared Tor ceiling reached ({}/{}), retry_after: {:?}", client_ip, state.tor_bucket.current(), - TOR_INVITES_PER_HOUR, + state.tor_bucket.limit(), retry_after ); return Err(( StatusCode::TOO_MANY_REQUESTS, Json(InviteErrorResponse { - error: format!( - "Invite requests from Tor are limited to {} per hour in total. \ - Please try again shortly, or request an invite without Tor.", - TOR_INVITES_PER_HOUR - ), + // Deliberately does NOT tell a privacy-tool user to stop using + // it, and does NOT state the exact ceiling (which would tell an + // attacker precisely what budget to drain). + error: "Too many invite requests right now. Please try again shortly.".to_string(), retry_after_seconds: retry_after, }), )); @@ -432,8 +437,9 @@ async fn create_room_invite( return Err(( StatusCode::TOO_MANY_REQUESTS, Json(InviteErrorResponse { - error: "Rate limited. You can request up to 4 invites per 24 hours." - .to_string(), + error: format!( + "Rate limited. You can request up to {MAX_INVITES_PER_WINDOW} invites per 24 hours." + ), retry_after_seconds: retry_after, }), )); @@ -450,12 +456,30 @@ async fn create_room_invite( } } - // The per-IP check passed, so this request is being served -- charge it to - // the shared Tor ceiling. Done here rather than alongside `has_capacity()` - // so that a request rejected by the per-IP limiter never consumes shared - // capacity that ordinary Tor users are relying on. - if via_tor { - state.tor_bucket.record(); + // The per-IP check passed, so charge the shared Tor ceiling. `try_acquire` + // (not the earlier `has_capacity`) is the admission AUTHORITY: it prunes, + // checks and records under one lock, so the ceiling holds however many + // requests race, and stays correct if an `.await` is ever introduced + // between the pre-check and here. + // + // Placed after the per-IP check so a per-IP rejection never consumes shared + // capacity that ordinary Tor users depend on. + if via_tor && !state.tor_bucket.try_acquire() { + // Lost the race against a concurrent request since the pre-check. + let retry_after = state.tor_bucket.retry_after_seconds(); + warn!( + "Tor exit {} refused at acquire: shared ceiling reached ({}/{})", + client_ip, + state.tor_bucket.current(), + state.tor_bucket.limit() + ); + return Err(( + StatusCode::TOO_MANY_REQUESTS, + Json(InviteErrorResponse { + error: "Too many invite requests right now. Please try again shortly.".to_string(), + retry_after_seconds: retry_after, + }), + )); } // Generate invite @@ -469,6 +493,12 @@ async fn create_room_invite( } Err(e) => { error!("Failed to generate invite: {:?}", e); + // No invite was issued, so give the Tor slot back. Otherwise a + // burst of generation failures would lock out every Tor user for a + // full window on top of the outage itself. + if via_tor { + state.tor_bucket.release(); + } Err(( StatusCode::INTERNAL_SERVER_ERROR, Json(InviteErrorResponse { @@ -499,3 +529,207 @@ pub fn get_invite_routes(state: InviteState) -> Router { .route("/create-invite", post(create_room_invite)) .with_state(state) } + +#[cfg(test)] +mod invite_handler_tests { + //! End-to-end tests for the invite handler's rate-limit wiring. + //! + //! Every unit below this is correct in isolation; the VALUE of the change + //! lives in how they compose in `create_room_invite`. Three one-line + //! regressions -- dropping `try_acquire`, inverting `via_tor`, or moving the + //! acquire above the per-IP check -- each leave the unit tests fully green + //! and the limiter fully defeated. These tests are what fail instead. + + use super::*; + use crate::rate_limit::AggregateBucket; + use crate::tor::TorExitList; + use std::net::SocketAddr; + use tempfile::TempDir; + + /// Build state with an injected exit list and a small ceiling. + fn state_with(dir: &TempDir, exits: &[String], ceiling: usize) -> InviteState { + let cache = dir.path().join("exits.txt"); + std::fs::write(&cache, exits.join("\n")).unwrap(); + let mut seed = [0u8; 32]; + seed[0] = 7; + let signing_key = SigningKey::from_bytes(&seed); + let owner = signing_key.verifying_key(); + InviteState { + rate_limiter: Arc::new(RateLimiter::new(dir.path().join("rl.json"), 24)), + tor_bucket: Arc::new(AggregateBucket::new(ceiling, 60)), + tor_exits: Arc::new(TorExitList::new(Some(cache))), + room_owner_vk: owner, + inviter_signing_key: signing_key, + room_name: "Test Room".to_string(), + } + } + + fn addr(ip: &str) -> SocketAddr { + SocketAddr::new(ip.parse::().unwrap(), 12345) + } + + async fn request(state: &InviteState, ip: &str) -> StatusCode { + match create_room_invite(State(state.clone()), ConnectInfo(addr(ip))).await { + Ok(_) => StatusCode::OK, + Err((code, _)) => code, + } + } + + /// THE regression test for the incident this change exists to fix: an actor + /// rotating many distinct Tor exits must be capped in TOTAL, not per-IP. + /// + /// Fails if `try_acquire` is dropped, or if `via_tor` is inverted. + #[tokio::test] + async fn tor_exits_share_one_ceiling_across_rotating_ips() { + const CEILING: usize = 5; + let dir = tempfile::tempdir().unwrap(); + // 40 distinct exits, each used once -- exactly the burst's shape. + let exits: Vec = (1..=40).map(|i| format!("185.220.101.{i}")).collect(); + let state = state_with(&dir, &exits, CEILING); + + let mut ok = 0; + let mut refused = 0; + for i in 1..=40 { + match request(&state, &format!("185.220.101.{i}")).await { + StatusCode::OK => ok += 1, + StatusCode::TOO_MANY_REQUESTS => refused += 1, + other => panic!("unexpected status {other}"), + } + } + assert_eq!( + ok, CEILING, + "rotation across 40 exits must yield exactly {CEILING} invites, got {ok}" + ); + assert_eq!(refused, 40 - CEILING); + } + + /// A non-Tor IP must be unaffected by a saturated Tor ceiling. + /// + /// Fails if `via_tor` is inverted or ignored. + #[tokio::test] + async fn non_tor_ip_unaffected_by_full_tor_bucket() { + let dir = tempfile::tempdir().unwrap(); + let exits: Vec = (1..=10).map(|i| format!("185.220.101.{i}")).collect(); + let state = state_with(&dir, &exits, 2); + + // Saturate via Tor. + assert_eq!(request(&state, "185.220.101.1").await, StatusCode::OK); + assert_eq!(request(&state, "185.220.101.2").await, StatusCode::OK); + assert_eq!( + request(&state, "185.220.101.3").await, + StatusCode::TOO_MANY_REQUESTS + ); + + let before = state.tor_bucket.current(); + assert_eq!( + request(&state, "203.0.113.9").await, + StatusCode::OK, + "a non-Tor IP must still be served" + ); + assert_eq!( + state.tor_bucket.current(), + before, + "a non-Tor request must not consume shared Tor capacity" + ); + } + + /// Pins the second half of the ordering rationale: a request refused by the + /// PER-IP limiter must not consume shared Tor capacity. + /// + /// Fails if `try_acquire` is moved above `check_and_record`. + #[tokio::test] + async fn per_ip_rejection_does_not_consume_tor_capacity() { + let dir = tempfile::tempdir().unwrap(); + let state = state_with(&dir, &["185.220.101.1".to_string()], 100); + + // Exhaust this single exit's per-IP allowance. + for _ in 0..MAX_INVITES_PER_WINDOW { + assert_eq!(request(&state, "185.220.101.1").await, StatusCode::OK); + } + let consumed = state.tor_bucket.current(); + assert_eq!(consumed, MAX_INVITES_PER_WINDOW); + + // Further requests are per-IP rejections; they must NOT take Tor slots. + for _ in 0..5 { + assert_eq!( + request(&state, "185.220.101.1").await, + StatusCode::TOO_MANY_REQUESTS + ); + } + assert_eq!( + state.tor_bucket.current(), + consumed, + "per-IP rejections must not consume shared Tor capacity" + ); + } + + /// Pins the first half of the ordering rationale: a request refused by the + /// TOR ceiling must not burn the requester's per-IP allowance, so an + /// ordinary Tor user still has their full quota once capacity frees. + #[tokio::test] + async fn tor_ceiling_refusal_does_not_burn_per_ip_allowance() { + let dir = tempfile::tempdir().unwrap(); + let exits: Vec = (1..=10).map(|i| format!("185.220.101.{i}")).collect(); + let state = state_with(&dir, &exits, 1); + + assert_eq!(request(&state, "185.220.101.1").await, StatusCode::OK); + // Victim is refused purely by the ceiling. + for _ in 0..3 { + assert_eq!( + request(&state, "185.220.101.5").await, + StatusCode::TOO_MANY_REQUESTS + ); + } + // Free the ceiling; the victim must still have all 4 per-IP invites. + state.tor_bucket.release(); + let mut served = 0; + for _ in 0..MAX_INVITES_PER_WINDOW { + if request(&state, "185.220.101.5").await == StatusCode::OK { + served += 1; + state.tor_bucket.release(); // keep ceiling free for this probe + } + } + assert_eq!( + served, MAX_INVITES_PER_WINDOW, + "ceiling refusals must not have consumed the victim's per-IP quota" + ); + } + + /// The fail-open claim that actually matters: with no exit list, the + /// endpoint behaves exactly as it did before this feature existed. + #[tokio::test] + async fn empty_exit_list_is_a_handler_no_op() { + let dir = tempfile::tempdir().unwrap(); + let state = state_with(&dir, &[], 1); // ceiling of 1, but nothing is Tor + + for _ in 0..MAX_INVITES_PER_WINDOW { + assert_eq!(request(&state, "185.220.101.1").await, StatusCode::OK); + } + assert_eq!( + request(&state, "185.220.101.1").await, + StatusCode::TOO_MANY_REQUESTS, + "per-IP limit still applies" + ); + assert_eq!( + state.tor_bucket.current(), + 0, + "an empty exit list must never consume Tor capacity" + ); + } + + /// The runtime off-switch must fully disable the ceiling. + #[tokio::test] + async fn zero_ceiling_disables_tor_metering() { + let dir = tempfile::tempdir().unwrap(); + let exits: Vec = (1..=20).map(|i| format!("185.220.101.{i}")).collect(); + let state = state_with(&dir, &exits, 0); + + for i in 1..=20 { + assert_eq!( + request(&state, &format!("185.220.101.{i}")).await, + StatusCode::OK, + "ceiling 0 must never refuse" + ); + } + } +} diff --git a/rust/api/src/tor.rs b/rust/api/src/tor.rs index 33014374..6d26bcd8 100644 --- a/rust/api/src/tor.rs +++ b/rust/api/src/tor.rs @@ -4,27 +4,36 @@ //! //! gkapi's per-IP invite limit ([`crate::rate_limit::MAX_INVITES_PER_WINDOW`]) //! is structurally defeated by Tor circuit rotation: every new exit node -//! presents a fresh IP, and therefore a fresh bucket. Measured on 2026-07-25, -//! a single actor pulled **152 invites in ~20 minutes across 113 distinct exit -//! IPs**, and not one of those IPs came close to the per-IP limit. Tightening -//! the per-IP number does nothing about this — rotation simply routes around -//! it, whatever the number is. +//! presents a fresh IP, and therefore a fresh bucket. +//! +//! Measured from `invite_rate_limits.json` (1728 invites, +//! 2026-07-24T06:23Z..2026-07-25T05:54Z), classified against the official bulk +//! exit list: the 02:35-03:05Z burst was **246 invites from 202 distinct IPs, +//! 200 of them via 159 Tor exits**, with no single IP near the per-IP limit. +//! Tightening the per-IP number does nothing about this -- rotation routes +//! around it at any value. //! //! Because the Tor exit set is *enumerable*, the fix is to stop treating exits //! as independent identities and meter them as one shared bucket (see -//! [`crate::rate_limit::AggregateBucket`]). Rotation then buys the attacker -//! nothing at all. This module supplies the membership test that makes that -//! possible. +//! [`crate::rate_limit::AggregateBucket`]). +//! +//! # Why not simply block Tor, and what the bucket costs instead //! -//! # Why not simply block Tor +//! Tor is ~7% of non-burst invite traffic (106 requests from 90 distinct exits +//! over the same window), nearly all 1-2 requests each -- ordinary usage. +//! Freenet is a privacy project and the quickstart is the main onboarding path, +//! so blocking that traffic outright denies real users in ALL states. //! -//! Measured over the same 23-hour window, Tor accounts for ~4.8% of invite -//! traffic *outside* the attack burst — 69 requests from 56 distinct exits, -//! nearly all of them 1-2 requests each, i.e. ordinary usage. Freenet is a -//! privacy project and the quickstart is the main onboarding path, so blocking -//! that traffic outright costs real users. A shared bucket sized above the -//! observed organic peak (11/hour) leaves those users untouched while capping -//! a rotation burst hard. +//! The bucket is better than a block, but it is NOT free, and the earlier +//! version of this comment was wrong to claim ordinary Tor users are +//! unaffected. Sharing one budget across an anonymity set means the loudest +//! member can take all of it: an attacker who sustains the ceiling denies +//! invites to every Tor user for as long as they keep it up, and because a +//! refused request costs them nothing, an automated poller wins each freed slot +//! against a human. That is inherent to any aggregate cap over an unlinkable +//! pool. Closing it needs a per-request cost signal (proof-of-work / CAPTCHA), +//! tracked in freenet/web#81. Treat this module as rate-shaping that buys time, +//! not as a fix. //! //! # Failure policy: FAIL OPEN //! @@ -62,6 +71,10 @@ const MAX_RESPONSE_BYTES: usize = 4 * 1024 * 1024; /// The real list is ~2000 exits, so this is ~50x headroom. const MAX_EXITS: usize = 100_000; +/// A real bulk exit list holds ~1000-2000 entries. Anything far below this is a +/// truncated or wrong response, not a genuinely tiny Tor network. +const MIN_PLAUSIBLE_EXITS: usize = 200; + /// Beyond this age the cached list is still USED (stale data beats no data for /// a rate-limit hint) but is logged as stale, because decommissioned exits can /// be reassigned to ordinary users and would then be metered as Tor. @@ -78,12 +91,28 @@ pub enum TorListError { Io(#[from] std::io::Error), #[error("response too large: {0} bytes exceeds cap of {MAX_RESPONSE_BYTES}")] TooLarge(usize), - #[error("response contained no parseable exit addresses")] - Empty, + #[error("implausibly small exit list: {got} entries (minimum {min}); refusing to replace a good list")] + Implausible { got: usize, min: usize }, + #[error( + "exit list shrank implausibly: {got} entries vs {previous} previously; refusing to replace" + )] + Shrank { got: usize, previous: usize }, #[error("lock poisoned")] Lock, } +/// Map an IPv4-mapped IPv6 address (`::ffff:a.b.c.d`) to its IPv4 form so both +/// spellings compare equal. Other addresses pass through unchanged. +fn canonicalize(ip: &IpAddr) -> IpAddr { + match ip { + IpAddr::V6(v6) => match v6.to_ipv4_mapped() { + Some(v4) => IpAddr::V4(v4), + None => *ip, + }, + _ => *ip, + } +} + #[derive(Default)] struct Snapshot { exits: HashSet, @@ -151,8 +180,14 @@ impl TorExitList { /// Returns `false` when the list is empty or unavailable — see the /// fail-open policy in the module docs. pub fn is_exit(&self, ip: &IpAddr) -> bool { + // Canonicalize first. gkapi binds 0.0.0.0 today so peers are always + // IpAddr::V4, but changing the bind to `::` for dual-stack (a one-line, + // plausible change) makes Linux deliver IPv4 peers as ::ffff:a.b.c.d, + // which would never match the V4 entries parsed from the list -- every + // exit would silently escape metering with nothing in the logs. + let ip = canonicalize(ip); match self.inner.read() { - Ok(snap) => snap.exits.contains(ip), + Ok(snap) => snap.exits.contains(&ip), Err(_) => { warn!("Tor exit list lock poisoned; treating as non-Tor (fail open)"); false @@ -194,21 +229,43 @@ impl TorExitList { // Reject an oversized body up front when the server declares a length. if let Some(len) = response.content_length() { - if len as usize > MAX_RESPONSE_BYTES { - return Err(TorListError::TooLarge(len as usize)); + if len > MAX_RESPONSE_BYTES as u64 { + return Err(TorListError::TooLarge(len.min(usize::MAX as u64) as usize)); } } - let body = response.text().await?; - // ...and again after the fact, since content-length is advisory. - if body.len() > MAX_RESPONSE_BYTES { - return Err(TorListError::TooLarge(body.len())); + // Stream with a running total. `response.text()` would buffer the whole + // body BEFORE any size check, so a chunked response (no content-length) + // could allocate without bound until FETCH_TIMEOUT -- an OOM on a small + // VM. The cap has to be enforced while reading, not after. + let mut response = response; + let mut buf: Vec = Vec::new(); + while let Some(chunk) = response.chunk().await? { + if buf.len() + chunk.len() > MAX_RESPONSE_BYTES { + return Err(TorListError::TooLarge(buf.len() + chunk.len())); + } + buf.extend_from_slice(&chunk); } + let body = String::from_utf8_lossy(&buf).into_owned(); let exits = Self::parse(&body); - if exits.is_empty() { - // Never let a garbage 200 wipe a good list. - return Err(TorListError::Empty); + // "Non-empty" is NOT enough validation. A truncated 200, or an HTML + // error page that happens to contain one IP-shaped string, would + // otherwise replace a good ~1400-entry list with a handful of entries + // and silently switch off nearly all Tor metering -- the worst kind of + // failure, because everything keeps "working". + if exits.len() < MIN_PLAUSIBLE_EXITS { + return Err(TorListError::Implausible { + got: exits.len(), + min: MIN_PLAUSIBLE_EXITS, + }); + } + let previous = self.len(); + if previous > 0 && exits.len() * 2 < previous { + return Err(TorListError::Shrank { + got: exits.len(), + previous, + }); } let now = Utc::now(); @@ -244,7 +301,7 @@ impl TorExitList { continue; } if let Ok(ip) = line.parse::() { - out.insert(ip); + out.insert(canonicalize(&ip)); if out.len() >= MAX_EXITS { warn!("Tor exit list hit the {MAX_EXITS} entry cap; truncating"); break; @@ -300,21 +357,62 @@ pub fn spawn_refresher(list: Arc) { ); } - tokio::spawn(async move { + let handle = tokio::spawn(async move { + // Backoff schedule used ONLY while we have no usable list at all. With + // an empty list the ceiling does not exist, so waiting a full hour after + // one transient failure would leave it off for an hour; with a good + // cached list an hourly retry is fine. + const EMPTY_RETRY_SECS: &[u64] = &[30, 60, 300, 900]; + let mut empty_attempt = 0usize; + loop { match list.refresh().await { - Ok(n) => info!("Refreshed Tor exit list: {n} exit addresses"), - Err(e) => warn!( - "Tor exit list refresh failed: {e} (using {} cached entries, last updated {}{}; \ - Tor traffic is metered per-IP only until this succeeds)", - list.len(), - list.last_updated() - .map(|t| t.to_rfc3339()) - .unwrap_or_else(|| "never".to_string()), - if list.is_stale() { ", STALE" } else { "" } - ), + Ok(n) => { + info!("Refreshed Tor exit list: {n} exit addresses"); + empty_attempt = 0; + } + Err(e) => { + let cached = list.len(); + if cached == 0 { + warn!( + "Tor exit list refresh failed with NO cached list: {e}. \ + Tor traffic is metered per-IP only (ceiling inactive)." + ); + } else { + warn!( + "Tor exit list refresh failed: {e} (still enforcing {cached} cached \ + entries, last updated {}{})", + list.last_updated() + .map(|t| t.to_rfc3339()) + .unwrap_or_else(|| "never".to_string()), + if list.is_stale() { ", STALE" } else { "" } + ); + } + } + } + + let delay = if list.is_empty() { + let d = EMPTY_RETRY_SECS[empty_attempt.min(EMPTY_RETRY_SECS.len() - 1)]; + empty_attempt = empty_attempt.saturating_add(1); + Duration::from_secs(d) + } else { + REFRESH_INTERVAL + }; + tokio::time::sleep(delay).await; + } + }); + + // The loop above is infinite, so the ONLY way this task ends is a panic -- + // which would silently freeze the exit list forever with nothing in the + // logs. Watch the handle so that failure is at least loud. + tokio::spawn(async move { + match handle.await { + Ok(()) => log::error!( + "Tor exit list refresher exited unexpectedly; the exit list is now frozen" + ), + Err(e) => { + log::error!("Tor exit list refresher panicked: {e}; the exit list is now frozen") } - tokio::time::sleep(REFRESH_INTERVAL).await; } }); } diff --git a/rust/api/tests/https_startup.rs b/rust/api/tests/https_startup.rs new file mode 100644 index 00000000..4616011f --- /dev/null +++ b/rust/api/tests/https_startup.rs @@ -0,0 +1,148 @@ +//! Startup smoke test for HTTPS mode. +//! +//! # Why this test exists +//! +//! gkapi links `rustls` with BOTH provider features enabled: `aws-lc-rs` (via +//! `axum-server`'s `tls-rustls`) and `ring` (via `reqwest`'s `rustls-tls`). +//! When both are on, rustls 0.23 deliberately refuses to guess and +//! `CryptoProvider::from_crate_features()` returns `None`, so the first TLS +//! construction panics: +//! +//! ```text +//! no process-level CryptoProvider available -- call +//! CryptoProvider::install_default() before this point +//! ``` +//! +//! That killed the process at startup in HTTPS mode, taking down every gkapi +//! endpoint (donations and cert-signing, not just invites). It was caught in +//! review, not by tests, because **no unit test constructs a `RustlsConfig`** +//! and `cargo run` without `--tls-cert` takes the plain-HTTP branch. The whole +//! failure lives in the gap between "the binary compiles and its unit tests +//! pass" and "the binary actually serves TLS". +//! +//! So this test spawns the real binary with real TLS material and makes a real +//! HTTPS request. It is deliberately end-to-end: a narrower test would not have +//! caught the bug it exists to prevent. +//! +//! Adding any dependency that enables another `rustls` provider feature will +//! fail here rather than in production. + +use std::io::Write; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Minimal self-signed cert/key so the server has something to serve. +/// Generated via `openssl` at test time to avoid committing key material. +fn generate_self_signed(dir: &std::path::Path) -> (std::path::PathBuf, std::path::PathBuf) { + let cert = dir.join("cert.pem"); + let key = dir.join("key.pem"); + let status = Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-keyout", + key.to_str().unwrap(), + "-out", + cert.to_str().unwrap(), + "-days", + "1", + "-nodes", + "-subj", + "/CN=localhost", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("openssl must be available to run this test"); + assert!(status.success(), "openssl failed to generate a test cert"); + (cert, key) +} + +/// The binary must come up in HTTPS mode and serve a request. +/// +/// Pins `install_crypto_provider()` in `main.rs`. Removing that call makes this +/// test fail with the process dying on the CryptoProvider panic. +#[test] +fn https_mode_starts_without_crypto_provider_panic() { + let dir = tempfile::tempdir().unwrap(); + let (cert, key) = generate_self_signed(dir.path()); + let notary = dir.path().join("notary"); + std::fs::create_dir_all(¬ary).unwrap(); + + // A high port so the test needs no privileges. Deliberately NOT passing + // --challenge-dir: that would bind :80 and fail without root, masking what + // we are actually testing. + let port = 18443; + let bin = env!("CARGO_BIN_EXE_ghostkey-api"); + + let mut child = Command::new(bin) + .args([ + "--tls-cert", + cert.to_str().unwrap(), + "--tls-key", + key.to_str().unwrap(), + "--notary-dir", + notary.to_str().unwrap(), + "--port", + &port.to_string(), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to spawn ghostkey-api"); + + // Poll until it serves, or it died. + let deadline = Instant::now() + Duration::from_secs(30); + let mut served = false; + let mut early_exit = None; + while Instant::now() < deadline { + if let Some(status) = child.try_wait().expect("try_wait failed") { + early_exit = Some(status); + break; + } + let out = Command::new("curl") + .args([ + "-sk", + "--max-time", + "2", + &format!("https://127.0.0.1:{port}/health"), + ]) + .output(); + if let Ok(out) = out { + if out.status.success() && !out.stdout.is_empty() { + served = true; + break; + } + } + std::thread::sleep(Duration::from_millis(250)); + } + + let _ = child.kill(); + let output = child.wait_with_output().expect("wait failed"); + let stderr = String::from_utf8_lossy(&output.stderr); + + if let Some(status) = early_exit { + // Surface the actual panic text -- this is the whole point of the test. + panic!( + "ghostkey-api exited during HTTPS startup ({status}).\n\ + If this mentions 'no process-level CryptoProvider available', a \ + dependency has enabled a second rustls provider feature and \ + install_crypto_provider() is no longer sufficient.\n\ + stderr:\n{stderr}" + ); + } + + assert!( + served, + "ghostkey-api never served an HTTPS request within 30s.\nstderr:\n{stderr}" + ); + + // Belt and braces: the panic must not appear even if we somehow served. + assert!( + !stderr.contains("no process-level CryptoProvider"), + "rustls CryptoProvider panic present in stderr:\n{stderr}" + ); + std::io::stdout().flush().ok(); +}