From 7d3ad278394e5959d14f661a27ec1779742bbedf Mon Sep 17 00:00:00 2001 From: williamspatrivk-rgb Date: Tue, 25 Aug 2026 03:02:16 +0000 Subject: [PATCH] fix: back cache.rs with real Redis, add degraded-mode fallback and health check api-server/src/cache.rs documented a Redis-backed cache with graceful fallback, but was a pure in-process DashMap with no Redis client at all. Behind multiple instances, invalidate_on_contract_event only cleared the calling instance's local entries, so other instances could keep serving stale data indefinitely after a contract event. - Back get/set/exists/ttl_remaining/invalidate/invalidate_prefix/ invalidate_pattern/clear/stats with a real Redis connection pool (redis + r2d2) when REDIS_URL is configured and reachable, while keeping every public function signature unchanged. - invalidate/invalidate_prefix/invalidate_pattern issue real DEL / SCAN+DEL against Redis, so an invalidation from any instance is immediately visible to every instance sharing that Redis. - Track reachability in an AtomicBool degraded flag: falls back to the existing in-process DashMap cache when Redis is unset/unreachable, logs and increments a cache_backend_degraded_transitions_total counter on each transition (not silently), and auto-recovers via a background health-check thread that pings Redis every 10s. - Add Docker-backed integration tests (testcontainers, gated behind the redis-integration-tests feature) proving cross-instance invalidation against a real Redis, and the documented fallback path when Redis is unreachable. - Update the cache.rs module doc comment to describe the fallback behavior as it now actually works, including the degraded-mode signal. Closes #786 --- api-server/Cargo.lock | 835 +++++++++++++++++- api-server/Cargo.toml | 9 + api-server/src/cache.rs | 290 +++++- .../tests/cache_redis_cross_instance.rs | 53 ++ api-server/tests/cache_redis_fallback.rs | 33 + 5 files changed, 1177 insertions(+), 43 deletions(-) create mode 100644 api-server/tests/cache_redis_cross_instance.rs create mode 100644 api-server/tests/cache_redis_fallback.rs diff --git a/api-server/Cargo.lock b/api-server/Cargo.lock index b7588e7..f1667eb 100644 --- a/api-server/Cargo.lock +++ b/api-server/Cargo.lock @@ -73,7 +73,7 @@ dependencies = [ "async-graphql-axum", "async-trait", "axum 0.8.9", - "base64", + "base64 0.22.1", "brotli", "chrono", "dashmap", @@ -81,6 +81,7 @@ dependencies = [ "flate2", "futures", "hex", + "hmac", "http-body-util", "jsonwebtoken", "metrics", @@ -90,17 +91,21 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "r2d2", "rand 0.8.6", + "redis", "regex", "reqwest", "serde", "serde_json", "sha2", "stellar-strkey", + "testcontainers", + "testcontainers-modules", "tokio", "tokio-stream", "tokio-tungstenite 0.21.0", - "tonic", + "tonic 0.12.3", "tower 0.5.3", "tower-http", "tracing", @@ -110,12 +115,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + [[package]] name = "ascii_utils" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" +[[package]] +name = "astral-tokio-tar" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" +dependencies = [ + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "rustix", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "async-graphql" version = "7.2.1" @@ -128,7 +155,7 @@ dependencies = [ "async-io", "async-trait", "asynk-strim", - "base64", + "base64 0.22.1", "bytes", "fast_chemail", "fnv", @@ -336,7 +363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core 0.5.6", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -416,12 +443,24 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.0" @@ -437,6 +476,80 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" +dependencies = [ + "async-stream", + "base64 0.22.1", + "bitflags 2.13.0", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.9.4", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic 0.14.6", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" +dependencies = [ + "prost 0.14.4", + "prost-types", + "tonic 0.14.6", + "tonic-prost", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.52.1-rc.29.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" +dependencies = [ + "base64 0.22.1", + "bollard-buildkit-proto", + "bytes", + "prost 0.14.4", + "serde", + "serde_json", + "serde_repr", + "time", +] + [[package]] name = "brotli" version = "7.0.0" @@ -458,6 +571,15 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -503,6 +625,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -526,6 +659,16 @@ dependencies = [ "cc", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -566,6 +709,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crate-git-revision" version = "0.0.6" @@ -618,7 +770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -726,6 +878,37 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der" version = "0.7.10" @@ -741,6 +924,9 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] [[package]] name = "derive_builder" @@ -781,6 +967,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -794,12 +981,29 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", +] + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ed25519" version = "2.2.3" @@ -855,6 +1059,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "fast_chemail" version = "0.9.6" @@ -870,6 +1084,17 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "portable-atomic", + "rand 0.10.2", + "web-time", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -1057,6 +1282,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -1139,6 +1365,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.4.2" @@ -1206,6 +1450,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-rustls" version = "0.27.9" @@ -1243,7 +1501,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1260,6 +1518,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1401,6 +1674,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1436,6 +1710,59 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -1463,7 +1790,7 @@ version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64", + "base64 0.22.1", "js-sys", "pem", "ring", @@ -1560,7 +1887,7 @@ version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ - "base64", + "base64 0.22.1", "http-body-util", "hyper", "hyper-rustls", @@ -1643,6 +1970,20 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.6", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1653,6 +1994,25 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -1668,6 +2028,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-modular" version = "0.6.4" @@ -1683,6 +2053,17 @@ dependencies = [ "num-modular", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.6", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1740,10 +2121,10 @@ dependencies = [ "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", - "prost", + "prost 0.13.5", "thiserror 1.0.69", "tokio", - "tonic", + "tonic 0.12.3", ] [[package]] @@ -1754,8 +2135,8 @@ checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost", - "tonic", + "prost 0.13.5", + "tonic 0.12.3", ] [[package]] @@ -1791,6 +2172,16 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + [[package]] name = "parking_lot_core" version = "0.9.12" @@ -1804,13 +2195,38 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.118", +] + [[package]] name = "pem" version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -1919,6 +2335,15 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1992,7 +2417,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive 0.14.4", ] [[package]] @@ -2008,6 +2443,28 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost 0.14.4", +] + [[package]] name = "quanta" version = "0.12.6" @@ -2099,6 +2556,17 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + [[package]] name = "rand" version = "0.8.6" @@ -2120,6 +2588,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -2158,13 +2637,38 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "raw-cpuid" version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags", + "bitflags 2.13.0", +] + +[[package]] +name = "redis" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f" +dependencies = [ + "arcstr", + "combine", + "itoa", + "num-bigint 0.5.1", + "percent-encoding", + "r2d2", + "ryu", + "sha1_smol", + "socket2 0.6.4", + "url", + "xxhash-rust", ] [[package]] @@ -2173,7 +2677,27 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", ] [[package]] @@ -2211,7 +2735,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "http", @@ -2278,7 +2802,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys", @@ -2356,6 +2880,39 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2368,7 +2925,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation", "core-foundation-sys", "libc", @@ -2445,6 +3002,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2457,6 +3025,39 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "jiff", + "schemars 0.9.0", + "schemars 1.2.2", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2464,10 +3065,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -2475,7 +3082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2494,6 +3101,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "signature" version = "2.2.0" @@ -2515,7 +3132,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ - "num-bigint", + "num-bigint 0.4.6", "num-traits", "thiserror 2.0.18", "time", @@ -2604,6 +3221,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.118", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "strum" version = "0.27.2" @@ -2652,6 +3292,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2685,6 +3336,46 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "testcontainers" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" +dependencies = [ + "astral-tokio-tar", + "async-trait", + "bollard", + "bytes", + "docker_credential", + "either", + "etcetera", + "ferroid", + "futures", + "http", + "itertools", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5985fde5befe4ffa77a052e035e16c2da86e8bae301baa9f9904ad3c494d357" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2799,6 +3490,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2 0.6.4", "tokio-macros", "windows-sys 0.61.2", @@ -2914,7 +3606,7 @@ dependencies = [ "async-stream", "async-trait", "axum 0.7.9", - "base64", + "base64 0.22.1", "bytes", "h2", "http", @@ -2925,7 +3617,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "socket2 0.5.10", "tokio", "tokio-stream", @@ -2935,6 +3627,46 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum 0.8.9", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2 0.6.4", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost 0.14.4", + "tonic 0.14.6", +] + [[package]] name = "tower" version = "0.4.13" @@ -2963,9 +3695,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2977,7 +3712,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.0", "bytes", "futures-util", "http", @@ -3160,6 +3895,33 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64 0.23.1", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -3170,6 +3932,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3178,6 +3941,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3599,6 +4368,22 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "yoke" version = "0.8.3" diff --git a/api-server/Cargo.toml b/api-server/Cargo.toml index 979ac18..5c989f9 100644 --- a/api-server/Cargo.toml +++ b/api-server/Cargo.toml @@ -53,3 +53,12 @@ regex = "1" base64 = "0.22" flate2 = "1.0" brotli = "7.0" +redis = { version = "1", features = ["r2d2"] } +r2d2 = "0.8" + +[dev-dependencies] +testcontainers = { version = "0.27", features = ["blocking"] } +testcontainers-modules = { version = "0.15", features = ["redis"] } + +[features] +redis-integration-tests = [] diff --git a/api-server/src/cache.rs b/api-server/src/cache.rs index 05969fc..697afb4 100644 --- a/api-server/src/cache.rs +++ b/api-server/src/cache.rs @@ -1,12 +1,26 @@ /// #316: Redis-based caching layer for IP and Swap queries. /// -/// Uses an in-process DashMap as a TTL cache when Redis is unavailable, -/// falling back gracefully so the server always starts without Redis. -use std::sync::Arc; +/// Backed by Redis when `REDIS_URL` is configured and reachable, so that +/// invalidations (`invalidate`, `invalidate_prefix`, `invalidate_pattern`) +/// are visible to every `api-server` instance sharing that Redis, not just +/// the instance that performed the write. +/// +/// Falls back to an in-process `DashMap` TTL cache — gracefully, so the +/// server always starts and serves correct (if not shared) data — whenever +/// Redis is not configured, not reachable at startup, or becomes +/// unreachable while running. That degraded state is *not* silent: it is +/// tracked in [`is_degraded`], logged on each transition, and exposed via +/// the `cache_backend_degraded_transitions_total` counter. A background +/// thread pings Redis every [`HEALTH_CHECK_INTERVAL`] and flips the cache +/// back to shared mode automatically once Redis is reachable again. +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use dashmap::DashMap; +use metrics::{counter, describe_counter}; use once_cell::sync::Lazy; +use r2d2::Pool; +use redis::Commands; use serde::{de::DeserializeOwned, Serialize}; const DEFAULT_TTL_SECS: u64 = 30; @@ -14,6 +28,12 @@ const IP_TTL_SECS: u64 = 60; const SWAP_TTL_SECS: u64 = 30; const REPUTATION_TTL_SECS: u64 = 300; +/// How often the background thread pings Redis to detect recovery from a +/// degraded state. Cache operations do not retry Redis on every call while +/// degraded — they defer to this thread — so this interval is also the +/// worst-case time to resume shared caching after Redis comes back. +const HEALTH_CHECK_INTERVAL: Duration = Duration::from_secs(10); + struct Entry { value: String, expires_at: Instant, @@ -21,6 +41,137 @@ struct Entry { static STORE: Lazy> = Lazy::new(DashMap::new); +/// `true` when the cache is serving from the in-process `DashMap` fallback +/// instead of the shared Redis store — either because `REDIS_URL` isn't +/// configured, or because Redis is currently unreachable. Starts `true` and +/// flips to `false` once a real connection is confirmed. +static DEGRADED: AtomicBool = AtomicBool::new(true); + +/// Connection pool to the shared Redis store, built once from `REDIS_URL`. +/// `None` when `REDIS_URL` is unset or invalid, in which case the cache +/// runs in-process only and no health-check thread is started. +static REDIS_POOL: Lazy>> = Lazy::new(|| { + describe_counter!( + "cache_backend_degraded_transitions_total", + "Transitions of the cache between shared Redis mode and degraded in-process mode" + ); + + let url = match std::env::var("REDIS_URL") { + Ok(url) => url, + Err(_) => { + tracing::warn!("cache: REDIS_URL not set, running in-process memory cache only"); + return None; + } + }; + + let client = match redis::Client::open(url) { + Ok(client) => client, + Err(err) => { + tracing::warn!(error = %err, "cache: invalid REDIS_URL, running in-process memory cache only"); + return None; + } + }; + + // `build_unchecked` never fails and never blocks on a live connection — + // the server must always start even if Redis is down at boot. + let pool = Pool::builder() + .max_size(16) + .connection_timeout(Duration::from_millis(300)) + .build_unchecked(client); + + // Establish real reachability synchronously (bounded by the 300ms + // connection timeout above) so that the very first cache operation + // after startup sees an accurate `DEGRADED` state instead of racing the + // background health-check thread's first tick. + match ping(&pool) { + Ok(()) => mark_healthy(), + Err(reason) => mark_degraded(&reason), + } + + spawn_health_check(pool.clone()); + Some(pool) +}); + +fn ping(pool: &Pool) -> Result<(), String> { + pool.get() + .map_err(|err| err.to_string()) + .and_then(|mut conn| { + redis::cmd("PING") + .query::(&mut *conn) + .map(|_| ()) + .map_err(|err| err.to_string()) + }) +} + +fn spawn_health_check(pool: Pool) { + let spawned = std::thread::Builder::new() + .name("cache-redis-health".to_string()) + .spawn(move || loop { + std::thread::sleep(HEALTH_CHECK_INTERVAL); + match ping(&pool) { + Ok(()) => mark_healthy(), + Err(reason) => mark_degraded(&reason), + } + }); + + if let Err(err) = spawned { + tracing::error!(error = %err, "cache: failed to spawn Redis health-check thread; degraded state will only clear on the next successful cache operation"); + } +} + +/// Record a Redis failure. Logs and increments the transition counter only +/// on the edge (healthy -> degraded), not on every failed operation. +fn mark_degraded(reason: &str) { + let was_degraded = DEGRADED.swap(true, Ordering::SeqCst); + if !was_degraded { + tracing::warn!( + reason, + "cache: Redis unreachable, falling back to in-process memory cache (degraded mode)" + ); + counter!( + "cache_backend_degraded_transitions_total", + "direction" => "to_degraded", + ) + .increment(1); + } +} + +/// Record a successful Redis health check. Logs and increments the +/// transition counter only on the edge (degraded -> healthy). +fn mark_healthy() { + let was_degraded = DEGRADED.swap(false, Ordering::SeqCst); + if was_degraded { + tracing::info!("cache: Redis connection restored, resuming shared cache mode"); + counter!( + "cache_backend_degraded_transitions_total", + "direction" => "to_healthy", + ) + .increment(1); + } +} + +/// Whether the cache is currently serving from the in-process fallback +/// rather than shared Redis. +pub fn is_degraded() -> bool { + DEGRADED.load(Ordering::SeqCst) +} + +/// Get a pooled Redis connection, marking the cache degraded on failure. +/// Returns `None` when Redis isn't configured or is currently unreachable. +fn redis_conn() -> Option> { + let pool = REDIS_POOL.as_ref()?; + if is_degraded() { + return None; + } + match pool.get() { + Ok(conn) => Some(conn), + Err(err) => { + mark_degraded(&err.to_string()); + None + } + } +} + // ── Cache Configuration ─────────────────────────────────────────────────────── /// Configure cache TTL for different data types. @@ -44,7 +195,8 @@ impl Default for CacheConfig { } /// Global cache configuration. -static CONFIG: Lazy> = Lazy::new(|| Arc::new(CacheConfig::default())); +static CONFIG: Lazy> = + Lazy::new(|| std::sync::Arc::new(CacheConfig::default())); /// Initialize cache with custom configuration. /// Note: has no effect after the cache has been first accessed (Lazy is already initialized). @@ -62,19 +214,37 @@ pub fn set(key: &str, value: &T) { /// Write a value into the cache under `key` with custom TTL. pub fn set_with_ttl(key: &str, value: &T, ttl_secs: u64) { - if let Ok(json) = serde_json::to_string(value) { - STORE.insert( - key.to_string(), - Entry { - value: json, - expires_at: Instant::now() + Duration::from_secs(ttl_secs), - }, - ); + let Ok(json) = serde_json::to_string(value) else { + return; + }; + + if let Some(mut conn) = redis_conn() { + let result: redis::RedisResult<()> = conn.set_ex(key, &json, ttl_secs.max(1)); + match result { + Ok(()) => return, + Err(err) => mark_degraded(&err.to_string()), + } } + + STORE.insert( + key.to_string(), + Entry { + value: json, + expires_at: Instant::now() + Duration::from_secs(ttl_secs), + }, + ); } /// Read a cached value. Returns `None` on miss or expiry. pub fn get(key: &str) -> Option { + if let Some(mut conn) = redis_conn() { + match conn.get::<_, Option>(key) { + Ok(Some(json)) => return serde_json::from_str(&json).ok(), + Ok(None) => return None, + Err(err) => mark_degraded(&err.to_string()), + } + } + let entry = STORE.get(key)?; if entry.expires_at < Instant::now() { drop(entry); @@ -86,6 +256,13 @@ pub fn get(key: &str) -> Option { /// Check if a key exists and is not expired. pub fn exists(key: &str) -> bool { + if let Some(mut conn) = redis_conn() { + match conn.exists::<_, bool>(key) { + Ok(exists) => return exists, + Err(err) => mark_degraded(&err.to_string()), + } + } + match STORE.get(key) { Some(entry) => entry.expires_at >= Instant::now(), None => false, @@ -94,6 +271,14 @@ pub fn exists(key: &str) -> bool { /// Get TTL remaining for a key in seconds. Returns None if key doesn't exist or is expired. pub fn ttl_remaining(key: &str) -> Option { + if let Some(mut conn) = redis_conn() { + match conn.ttl::<_, i64>(key) { + Ok(ttl) if ttl >= 0 => return Some(ttl as u64), + Ok(_) => return None, + Err(err) => mark_degraded(&err.to_string()), + } + } + let entry = STORE.get(key)?; if entry.expires_at < Instant::now() { drop(entry); @@ -105,34 +290,86 @@ pub fn ttl_remaining(key: &str) -> Option { } /// Invalidate a single cache key. +/// +/// Removes the key from Redis (visible to every instance sharing it) and +/// from the local fallback store, so a flip between backends around the +/// time of the call can never leave a stale copy behind. pub fn invalidate(key: &str) { + if let Some(mut conn) = redis_conn() { + let result: redis::RedisResult<()> = conn.del(key); + if let Err(err) = result { + mark_degraded(&err.to_string()); + } + } STORE.remove(key); } /// Invalidate all keys that start with `prefix`. pub fn invalidate_prefix(prefix: &str) { + if let Some(mut conn) = redis_conn() { + match redis_delete_matching(&mut conn, &format!("{prefix}*")) { + Ok(()) => {} + Err(err) => mark_degraded(&err.to_string()), + } + } STORE.retain(|k, _| !k.starts_with(prefix)); } /// Invalidate all keys matching a pattern (supports * wildcards). pub fn invalidate_pattern(pattern: &str) { - if pattern.contains('*') { - let regex_pattern = pattern.replace('*', ".*"); - if let Ok(regex) = regex::Regex::new(®ex_pattern) { - STORE.retain(|k, _| !regex.is_match(k)); - } - } else { + if !pattern.contains('*') { invalidate_prefix(pattern); + return; } + + if let Some(mut conn) = redis_conn() { + // Redis glob patterns already use `*`/`?`/`[...]`, so the pattern is + // passed through to SCAN MATCH as-is — no regex translation needed. + match redis_delete_matching(&mut conn, pattern) { + Ok(()) => {} + Err(err) => mark_degraded(&err.to_string()), + } + } + + let regex_pattern = pattern.replace('*', ".*"); + if let Ok(regex) = regex::Regex::new(®ex_pattern) { + STORE.retain(|k, _| !regex.is_match(k)); + } +} + +/// Scan for keys matching a Redis glob `pattern` and delete them. +fn redis_delete_matching(conn: &mut redis::Connection, pattern: &str) -> redis::RedisResult<()> { + let keys: Vec = conn + .scan_match::<_, String>(pattern)? + .collect::, redis::RedisError>>()?; + if !keys.is_empty() { + let _: () = conn.del(keys)?; + } + Ok(()) } /// Clear all cache entries. +/// +/// When Redis-backed, this flushes the connected Redis database — deployments +/// that need `clear()` scoped strictly to this cache's keys should point +/// `REDIS_URL` at a Redis instance/logical DB dedicated to it. pub fn clear() { + if let Some(mut conn) = redis_conn() { + let result: redis::RedisResult<()> = redis::cmd("FLUSHDB").query(&mut *conn); + if let Err(err) = result { + mark_degraded(&err.to_string()); + } + } STORE.clear(); } /// Get cache statistics. pub fn stats() -> CacheStats { + if let Some(mut conn) = redis_conn() { + if let Ok(total_entries) = redis::cmd("DBSIZE").query::(&mut *conn) { + return CacheStats { total_entries }; + } + } CacheStats { total_entries: STORE.len(), } @@ -373,4 +610,21 @@ mod tests { assert!(!exists("ip:1")); assert!(!exists("ip:list:owner:10:0")); } + + #[test] + fn test_degraded_by_default_without_redis_url() { + // In the test binary REDIS_URL is unset, so the cache runs + // in-process only and reports itself as degraded — not silently. + assert!(is_degraded()); + } } + +// Redis integration tests (real Redis via testcontainers, gated behind the +// `redis-integration-tests` feature) live in `api-server/tests/`, not here. +// +// They must run as separate integration-test binaries — each `tests/*.rs` +// file gets its own process — because `REDIS_POOL` above is a `Lazy` that +// reads `REDIS_URL` exactly once per process, on first cache access. A +// `#[cfg(test)]` module in this file would share a process (and therefore +// the same already-initialized `REDIS_POOL`) with the in-process unit tests +// above, which rely on `REDIS_URL` staying unset. diff --git a/api-server/tests/cache_redis_cross_instance.rs b/api-server/tests/cache_redis_cross_instance.rs new file mode 100644 index 0000000..f2d3247 --- /dev/null +++ b/api-server/tests/cache_redis_cross_instance.rs @@ -0,0 +1,53 @@ +#![cfg(feature = "redis-integration-tests")] +//! Integration test for #786: cross-instance cache invalidation via Redis. +//! +//! Requires Docker and is gated behind the `redis-integration-tests` feature +//! so a plain `cargo test` never needs a Docker daemon: +//! +//! ```sh +//! cargo test --features redis-integration-tests --test cache_redis_cross_instance +//! ``` + +use api_server::cache; +use serde::{Deserialize, Serialize}; +use testcontainers::runners::SyncRunner; +use testcontainers_modules::redis::Redis; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct Dummy { + val: u64, +} + +/// `cache`'s Redis pool is a `Lazy` read from `REDIS_URL` on first access, so +/// the env var must be set before any `cache::` call in this process — this +/// integration binary contains only Redis-backed tests, so that's safe here. +#[test] +fn cross_instance_invalidation_is_visible_immediately() { + let container = Redis::default() + .start() + .expect("failed to start Redis container"); + let host = container.get_host().expect("failed to get container host"); + let port = container + .get_host_port_ipv4(6379) + .expect("failed to get mapped Redis port"); + std::env::set_var("REDIS_URL", format!("redis://{host}:{port}")); + + let key = "test:cross_instance:1"; + let value = Dummy { val: 7 }; + + // "Instance A" writes the value. + cache::set(key, &value); + assert!( + !cache::is_degraded(), + "expected the cache to be Redis-backed against a live container" + ); + assert_eq!(cache::get::(key), Some(Dummy { val: 7 })); + + // "Instance B" invalidates it. Because both "instances" share the same + // Redis, this issues a real DEL against the shared store rather than a + // local-only removal, so the invalidation is visible everywhere. + cache::invalidate(key); + + // "Instance A" must no longer observe the stale value. + assert_eq!(cache::get::(key), None); +} diff --git a/api-server/tests/cache_redis_fallback.rs b/api-server/tests/cache_redis_fallback.rs new file mode 100644 index 0000000..aa251d5 --- /dev/null +++ b/api-server/tests/cache_redis_fallback.rs @@ -0,0 +1,33 @@ +#![cfg(feature = "redis-integration-tests")] +//! Integration test for #786: the documented fallback path when Redis is +//! configured but unreachable. +//! +//! Gated behind the `redis-integration-tests` feature; run with: +//! ```sh +//! cargo test --features redis-integration-tests --test cache_redis_fallback +//! ``` + +use api_server::cache; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct Dummy { + val: u64, +} + +/// Kept in its own binary (separate process from +/// `cache_redis_cross_instance`) because `cache`'s Redis pool reads +/// `REDIS_URL` once per process on first access. +#[test] +fn fallback_serves_correct_data_when_redis_unreachable() { + // Nothing listens on this port, so every Redis attempt fails fast and + // the cache falls back to in-process memory rather than erroring. + std::env::set_var("REDIS_URL", "redis://127.0.0.1:1"); + + let key = "test:fallback:1"; + let value = Dummy { val: 3 }; + + cache::set(key, &value); + assert_eq!(cache::get::(key), Some(Dummy { val: 3 })); + assert!(cache::is_degraded()); +}