diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0a80516 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: lint (${{ matrix.module }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Every module has its own go.mod and is linted independently. + module: [".", "lfu", "policies", "policies/arc", "policies/tinylfu", "metrics", "bench", "examples/basic", "examples/migration"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache-dependency-path: ${{ matrix.module }}/go.sum + - name: golangci-lint + # v7 of the action is required for golangci-lint v2. + uses: golangci/golangci-lint-action@v7 + with: + version: v2.8.0 + working-directory: ${{ matrix.module }} + + test: + name: test (${{ matrix.module }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: [".", "lfu", "policies", "policies/arc", "policies/tinylfu", "metrics", "bench", "examples/basic", "examples/migration"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache-dependency-path: ${{ matrix.module }}/go.sum + - name: go vet + working-directory: ${{ matrix.module }} + run: go vet ./... + - name: go test + working-directory: ${{ matrix.module }} + run: go test -race -short -count=1 ./... diff --git a/.gitignore b/.gitignore index fa92975..1a1a798 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,12 @@ -*.out \ No newline at end of file +*.out + +# Compiled example binaries (built via `go build` / `go run` in each example dir) +/examples/basic/basic +/examples/migration/migration + +# cache replay traces: downloaded at test time, never committed +traces/ +*.trace +*.trace.gz +*.spc.bz2 +cluster0* diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..efcb954 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,82 @@ +# golangci-lint configuration (schema v2). +# A single root config is discovered by golangci-lint when run from any of the +# repository's modules (root, lfu, examples/*), so all modules share these rules. +version: "2" + +run: + # Lint test files as well as production code. + tests: true + +linters: + # Start from the curated "standard" set (errcheck, govet, ineffassign, + # staticcheck, unused) and layer additional checks on top. + default: standard + enable: + - revive # replacement for golint: style and exported-doc checks + - gosec # security analysis + - misspell # spelling mistakes in comments/strings + - unconvert # unnecessary type conversions + - unparam # unused function parameters/results + - gocritic # opinionated diagnostics, performance and style checks + - bodyclose # HTTP response bodies must be closed + - errorlint # correct error wrapping / comparison + - copyloopvar # obsolete loop-variable copies (Go 1.22+) + - prealloc # slices that could be preallocated + - nolintlint # keep //nolint directives well-formed and justified + + settings: + revive: + rules: + - name: exported + - name: package-comments + - name: blank-imports + - name: context-as-argument + - name: error-return + - name: error-strings + - name: error-naming + - name: if-return + - name: indent-error-flow + - name: receiver-naming + - name: time-naming + - name: unreachable-code + - name: var-declaration + gocritic: + enabled-tags: + - diagnostic + - performance + - style + disabled-checks: + - hugeParam + - unnamedResult + - rangeValCopy + + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - std-error-handling + # Files excluded from every linter. + paths: + - policytype_string\.go # generated stringer output + rules: + # Test files: relax security/allocation/param linters that add noise there. + - path: _test\.go + linters: + - gosec + - unparam + - prealloc + # Examples are teaching material: keep them readable, don't require full + # godoc on every exported identifier. + - path: examples/ + linters: + - revive + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/sshaplygin/as-cache diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..082b194 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "makefile.configureOnOpen": false +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 2455dcb..cdeb613 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ Instead of forcing users to choose a fixed eviction algorithm upfront, as-cache runs multiple policies in parallel (shadow caching), measures their hit/miss rates per epoch, and uses Thompson Sampling to select the best-performing policy dynamically. **Module:** `github.com/sshaplygin/as-cache` -**Go version:** 1.21+ +**Go version:** 1.25+ **Status:** Experimental --- @@ -58,10 +58,48 @@ as-cache/ │ ├── interfaces.go # Core interface definitions ├── models.go # PolicyType, PolicyStats, ShadowStats, GlobalStats -├── cache.go # AdaptiveCache implementation +├── errors.go # Sentinel errors returned by NewAdaptiveCache +├── settings.go # Settings + NewAdaptiveCache validation +├── cache.go # AdaptiveCache struct + public cache API +├── epoch.go # Epoch loop, bandit reporting, policy selection +├── migration.go # Migration strategies (cold/warm/gradual) +├── advice.go # ObserveOnly reporting: Advice, PolicyReport +├── shadow.go # Promotion/demotion, shadow duty, value dropping +├── sampling.go # keySampler + miniature capacity maths +├── stability.go # Switch gates (cool-down, min improvement) ├── wrapper.go # CacheWrapper (wraps any Cacher, adds stats) ├── policytype_string.go # Generated: PolicyType.String() via stringer │ +├── policies/ # Separate module: ready-made policy adapters +│ ├── go.mod / go.sum # depends on root + hashicorp/golang-lru v2.0.6 +│ ├── adapters.go # NewLRU / NewTwoQueue / NewTTL / NewRandomPolicy +│ ├── adapt.go # PartialCacher + AdaptedCache (Resize-by-rebuild) +│ ├── random.go # RandomCache, implemented from scratch +│ ├── ttl.go # TTLCache: own expiry over plain LRU (see below) +│ └── conformance_test.go # shared Cacher/Policy contract suite +│ +├── policies/arc/ # Separate module, SOLELY for patent isolation +│ ├── go.mod / go.sum # depends on hashicorp/golang-lru/arc/v2 +│ └── arc.go # ARC adapter via policies.Adapt +│ +├── policies/tinylfu/ # Separate module: keeps otter's deps isolated +│ ├── go.mod / go.sum # depends on maypok86/otter/v2 +│ └── tinylfu.go # W-TinyLFU adapter (natively resizable) +│ +├── metrics/ # Separate module: expvar export (stdlib only) +│ ├── go.mod / go.sum +│ └── metrics.go # Advisor, Snapshot, Take, Publish +│ +├── bench/ # Separate module: workloads + evidence harness +│ ├── workload.go # deterministic zipf/uniform/loop/scan/phase-shift +│ ├── bandit.go # Thompson + greedy bandits (root ships none) +│ ├── harness.go # replay, Result, tables +│ ├── evidence_test.go # policy comparison + sampling-fidelity check +│ ├── timeline_test.go # ActivePolicy() plot over a phase-shift run +│ ├── trace.go # real-trace loaders (Twitter/LIRS/ARC formats) +│ ├── memory_test.go # memory multiplier + allocations +│ └── tuning_test.go # epoch/migration configuration sweep +│ ├── lfu/ # Separate module: LFU cache │ ├── go.mod / go.sum │ ├── lfu.go # Thread-safe LFU wrapper with eviction callbacks @@ -121,8 +159,8 @@ SelectPolicy() PolicyType |---|---|---| | `AdaptiveCache[K,V]` | cache.go | Main adaptive cache orchestrator | | `CacheWrapper[K,V]` | wrapper.go | Wraps any Cacher, adds hit/miss tracking | -| `PolicyType` | models.go | Enum: Undefined, LRU, LFU | -| `MigrationStrategy` | models.go | Enum: MigrationCold, MigrationWarm | +| `PolicyType` | models.go | Enum: Undefined, LRU, LFU, TwoQueue, ARC, Random, TTL, TinyLFU | +| `MigrationStrategy` | models.go | Enum: MigrationCold, MigrationWarm, MigrationGradual | | `PolicyStats` | models.go | Hits + Misses counters | | `ShadowStats` | models.go | Per-epoch policy performance | | `GlobalStats` | models.go | Aggregate statistics | @@ -133,7 +171,7 @@ SelectPolicy() PolicyType | Package | Version | Role | |---|---|---| -| `hashicorp/golang-lru/v2` | v2.0.7 | LRU reference implementation | +| `hashicorp/golang-lru/v2` | v2.0.6 | LRU/2Q/expirable (policies module). NOT v2.0.7: that release does not build | | `stitchfix/mab` | v0.1.1 | Multi-Armed Bandit (Thompson Sampling) | | `gonum.org/v1/gonum` | v0.8.2 | Numerical computing (used by mab) | | `golang.org/x/exp` | indirect | Used by gonum | @@ -179,6 +217,11 @@ cd lfu && go test ./... # Run example cd examples/basic && go run main.go +# Lint all modules (golangci-lint v2, config in .golangci.yml) +make lint # or: golangci-lint run ./... (per module) +make lint-fix # apply --fix and formatters +make install-tools # install the pinned golangci-lint version + # Regenerate stringer (after modifying PolicyType in models.go) go generate ./... @@ -199,21 +242,201 @@ cd examples/basic && go mod tidy - [x] `AdaptiveCache.Resize()` - resizes all policies, returns total eviction count - [x] `AdaptiveCache.Contains()` - delegates to active policy - [x] `AdaptiveCache.Keys()` / `Values()` / `Len()` / `Peek()` - delegate to active policy -- [x] `AdaptiveCache.Stats()` - returns cumulative hit/miss from active policy +- [x] `AdaptiveCache.Stats()` - cumulative: completed epochs (`globalStats`) plus the active policy's in-progress epoch - [x] Background epoch goroutine with bandit-based policy selection - [x] `CacheWrapper` with hit/miss statistics - [x] LFU implementation (simplelfu + thread-safe wrapper) — all methods implemented - [x] `lfu.Cache`: `Resize`, `ContainsOrAdd`, `PeekOrAdd`, `RemoveOldest`, `GetOldest` - [x] `simplelfu.LFU`: `Resize`, `GetOldest`, `RemoveOldest` - [x] Basic example with HTTP server +- [x] Roadmap Milestone 1 (correctness): locked epoch switch in `runEpoch`; atomic + `CacheWrapper` counters; active policy's per-epoch stats reported to the bandit + alongside shadows with all counters reset each epoch (active counts folded into + `AdaptiveCache.globalStats` so `Stats()` stays cumulative and demotion never + leaks active-tenure stats into a shadow epoch); constructor validation with + sentinel errors in `errors.go` (`ErrNilSettings`, `ErrInvalidEpochDuration`, + `ErrNilBandit`, `ErrNilPolicy`, `ErrDuplicatePolicy`); idempotent `Close()` + (`sync.Once`) that waits for the epoch goroutine (`sync.WaitGroup`); explicit + `go vet` CI step and `TestAdaptiveCache_StressAcrossEpochBoundaries` (1ms + epochs, all three migration strategies, full public API under `-race`). + Migration strategies moved verbatim from `cache.go` to `migration.go` to keep + files under the 400-line rule. + +- [x] Roadmap Milestone 2 (overhead reduction), three of four items: + - **Sampled shadow caching** (`sampling.go`, `shadow.go`): `Settings.ShadowSampleRate` + gates the shadow fan-out on `maphash.Comparable(seed, key) < threshold`. One + sampler is shared by every policy so all arms measure the same substream -- + per-policy seeds would make their hit rates incomparable. Shadows resize to + `ceil(rate*Cap)` so each stays a faithful miniature; `MinShadowCapacity` + (default 256) floors that, raising the *effective rate* rather than only the + capacity, and disabling sampling when a cache is too small to host a useful + miniature. + - **Stats deviate from the roadmap wording deliberately**: sampled counts are + NOT scaled up by `1/rate`. That would restore magnitude while inventing + confidence, handing a Beta posterior 20x the evidence collected. Instead the + *active* arm is measured over the same sampled substream + (`activeSampledHits`/`activeSampledMisses`, reported by `selectPolicyLocked`), + so every arm carries equal, honest evidence. `Stats()` still reports real + unsampled traffic; only the bandit sees the sample. + - **Value dropping on demotion** (`demoteLocked`): a demoted policy keeps its + keys (that is its eviction bookkeeping) but its entries are rewritten to the + zero value, unsampled keys removed, then it shrinks to miniature capacity. + Rewriting in `Keys()` order preserves LRU recency and leaves LFU relative + order untouched. Deferred while a gradual window is open, since that window + promotes out of the source's real values. + - **Switch stability** (`stability.go`): `MinHitRateImprovement`, + `SwitchCooldownEpochs`, `MinEpochRequests`, all off at their zero values. + - **Ordering invariant** (documented on `switchLocked`): *every mutation of a + policy must happen while that policy is not the active one.* Mutate the + incoming policy before promoting it, the outgoing one after demoting it. + This is what keeps a caller from ever reading a policy mid-rewrite. + - **Test policies**: `mockPolicy` does not enforce capacity (its `Resize` only + records the new size). That blind spot hid three capacity bugs found in + review. `evictingPolicy` in `evicting_test.go` does enforce it -- use it for + anything whose subject is capacity, resizing, or eviction. + - Measured on an M1 Max with mutex-backed stub policies: `Get` 100 -> 36 ns/op + with one shadow, 145 -> 38 ns/op with three; `Add` 109 -> 52 and 189 -> 57; + mixed parallel 184 -> 85. The structural win is that cost becomes flat in the + number of shadows, so adding policies is nearly free. + - **Deferred: lock-free reads** (`atomic.Pointer` to the active policy). An + adversarial safety analysis found the usual seqlock-retry framing unsound and + surfaced five further hazards -- retry must wrap all six read delegations + (`Values()` on a stale post-drop state returns N zeros at the right length), + retries are not side-effect-free (double-counted stats, double-bumped + recency), the `GetStats`/`ResetStats` split silently discards hits once reads + are unlocked (fixing it changes the public `CacheStats` interface), and + `MigrationGradual` cannot go lock-free since `promoteLocked` mutates from + inside `Get`. Worth its own scoped change. + +- [x] Roadmap Milestone 4 (evidence), synthetic half. `make evidence` replays + the suite; see the README for the tables. The headline finding is negative and + should not be smoothed over: **adaptive selection never beats the best fixed + policy on these workloads**, including phase-shift, where W-TinyLFU wins by + 3.8 points. The `ActivePolicy()` timeline shows the bandit working correctly — + it explores, picks W-TinyLFU, and holds it 90% of the run — but there is no + crossover to exploit because W-TinyLFU is best in both regimes. What the + library does deliver is a bound on the cost of guessing wrong (77.5% vs LRU's + 0.0% on `loop`). That argues for Milestone 5 advisor mode as the primary + product rather than a stepping stone. + - Milestone 2's sampling was validated here: sampled shadows preserve the + policy ranking, running 1-3 points pessimistic uniformly across arms. + - Evidence tests are guarded by `testing.Short()` and excluded from + `make test`, which now passes `-short`. Under `-race` the epoch pacing + changes ~15x and the measurements become meaningless. Run `make evidence`. + - The root module ships no `Bandit`; `bench/bandit.go` has a Thompson + sampler (Beta posteriors via Marsaglia-Tsang gamma draws, with discounting + so it can change its mind) and a greedy control. Worth promoting if + advisor mode lands. + +- [x] Roadmap Milestone 4 (evidence), real traces. `./scripts/fetch-traces.sh` + downloads five published traces; none are committed (see `.gitignore`). + Loaders self-test against published record/distinct counts, and the ARC + layout's range expansion is asserted -- each record stands for `blockCount` + accesses, and reading it as one-key-per-line would silently produce a + workload incomparable with the literature. + - **The real traces overturned the synthetic conclusion.** The best fixed + policy varies by trace: 2Q wins on Twitter Twemcache and ARC OLTP, + W-TinyLFU on ARC P3 and the LIRS traces. On OLTP, W-TinyLFU is + second-*worst*. Tuned sensibly (50ms epoch, warm migration), adaptive lands + within ~1 point of the best fixed policy and beats it on P3 by 0.76. + - **Epoch duration is the setting that matters.** A 2ms epoch on a 20k cache + means copying the cache hundreds of times per replay: 13,476 ns/op and a + 7-point hit-rate loss on P3. `MigrationCold` costs 28 points on OLTP. The + stability gates cost 37 points on `loop`, which must re-adapt constantly. + - Memory: six policies cost 2.65x a single LRU, not 6x (shadows hold keys, + never values); 1.32x with sampling. The old README claim was wrong. +- [x] Roadmap Milestone 5 (advisor mode). `Settings.ObserveOnly` measures every + arm while guaranteeing the cache behaves exactly like the policy it was built + with; `Advice()` reports which policy wins and by how much. The bandit may be + nil in this mode (implementing one is the fiddliest part of using the + library, and nothing is ever selected), and the `EvictPartialCapacityFilling` + capacity gate is bypassed, since it exists to avoid switching on thin + evidence and would otherwise suppress the very measurement being asked for. + The `metrics` module publishes a `Snapshot` through expvar, evaluated on + scrape, with a duplicate-name check so a registration mistake returns an + error instead of panicking the process. + - **`tenureStats`, not lifetime stats.** A policy's measurements are cleared + when it changes role. Accumulating across a role change pooled its active + tenure (full capacity, all traffic) with its shadow tenure (miniature + capacity, a sample), and left the outgoing policy's long history + outweighing the incoming one's short history -- so right after a correct + switch, `Advice` named the policy the cache had just moved away from as + best, for a number of epochs linear in the history length. Do not + "improve" this by accumulating for longer. + - **`Advice.Epochs` counts reporting epochs, not ticks.** The capacity gate + can skip measurement indefinitely, and `epochID` would report thousands of + epochs of evidence behind nothing. + - Ties in `Advice` are broken by `PolicyType`: ranging a map and sorting + stably on hit rate alone made `Best` flap between equally-performing arms + on an unchanged cache. + - `metrics.Publish` needs both the mutex and the `recover`: `expvar.Get` + followed by `expvar.Publish` is check-then-act, and expvar panics on a + duplicate name. + +- [x] LFU added to `policies` (`NewLFU`) and to the evidence suite. Putting it + through the shared conformance suite for the first time found a real bug: + it accepted entries at zero capacity, where every other policy holds nothing. + A pre-existing test asserted the buggy behaviour (that the entry stayed + retrievable) and was corrected -- its purpose was guarding a panic, and the + retrievability assertion was incidental. + - Evidence: LFU is the **best** policy on synthetic `zipf` (73.5%) and the + **worst** on both large real traces (41.4% Twitter, 45.4% OLTP). Synthetic + Zipf holds popularity stationary, which is exactly LFU's assumption; real + traffic shifts and stale frequency counts pin dead entries. This is the + clearest evidence in the repo that synthetic workloads mislead. ### Incomplete / TODO - [x] Data migration between policies on switch — `MigrationStrategy` in `Settings` (`MigrationCold` default, `MigrationWarm` copies all keys from old active to new active) -- [x] Unit tests for LFU packages (simplelfu: 100% coverage, lfu wrapper: 93.2% coverage) -- [x] Unit tests for root package (cache_test.go: 96.5% coverage -- CacheWrapper, AdaptiveCache delegated methods, tryChangePolicy, epoch-based switching, constructor edge cases, concurrent access) -- [ ] Additional policies: Random, 2Q, ARC (mentioned in README but not implemented) -- [ ] README Usage and Idea sections +- [x] Unit tests for LFU packages (simplelfu: 98.3% coverage, lfu wrapper: 100% coverage) +- [x] Unit tests for root package (cache_test.go: 93.6% coverage -- CacheWrapper, AdaptiveCache delegated methods, tryChangePolicy, epoch-based switching, constructor edge cases, concurrent access) +- [x] Roadmap Milestone 3 (policy coverage), two of three items — LRU, 2Q, + Random, TTL and ARC adapters. Four findings shaped the design: + - **ARC is patented by IBM** (US 6,996,676). Upstream `hashicorp/golang-lru` + split it into its own module in v2 for that reason; `policies/arc` keeps + that split so importing `policies` never pulls a patented implementation + into a build. Do not merge it into `policies` for convenience. + - **Neither 2Q nor ARC satisfies `Cacher`**: their `Add`/`Remove` return + nothing and neither has `Resize`, which the miniature-shadow mechanism + depends on. One shared `policies.Adapt` covers both. Its `Resize` rebuilds + the cache, discarding the algorithm's learned adaptation state — so an + adapted policy resized every epoch would stay permanently unadapted and + under-report its own hit rate. + - **`TTLCache` deliberately does NOT use `expirable.LRU`.** Three defects + made that untenable, all found in review: its `Get`/`Peek` return + `(zeroValue, true)` for an expired-but-unreaped entry (a bare `return` over + an already-true named result) -- a zero-value leak, the one invariant this + library is built on; its `Values` returns a full-length slice padded with + trailing zeros that no longer line up with `Keys`; and `NewLRU` starts a + reaper goroutine per cache with no way to stop it, leaking the goroutine + and the whole cache forever. `TTLCache` instead stores a `ttlEntry{value, + expiresAt}` in a plain `lru.Cache` and expires lazily on read. It also + makes size 0 mean *empty*, where `expirable` documents 0 as *unlimited* -- + which, since shadows are resized automatically, would have turned a + bounded shadow into an unbounded one. + - **`Keys()` order is not portable.** 2Q returns frequent-then-recent, ARC + returns recent-then-frequent; neither is a global recency order. So + `AdaptedCache.Resize` replays every entry and lets the rebuilt cache pick + its own victims -- which entries survive a shrink is explicitly not + meaningful. Any "keep the tail" rule is right for one cache and exactly + backwards for the other. + - **`RandomCache.Add` must evict before inserting.** Inserting first puts the + caller's own write into the victim draw, losing it with probability + 1/(size+1) -- a write accepted and gone before the next read. No other + policy here does that, and no conformance test caught it until one was + added that fills a cache and then reads the fresh key back. + - **golang-lru v2.0.7 does not build**: its published `simplelru` imports + `.../simplelru/internal`, which the module does not contain (verified + against the checksum database, so it is upstream, not a local cache + problem). Everything is pinned to v2.0.6. Do not bump without checking. +- [x] W-TinyLFU arm (`policies/tinylfu`, over `maypok86/otter` v2) — the + baseline that actually needs beating. otter is natively resizable, so unlike + 2Q and ARC this arm never rebuilds and keeps its frequency sketch. Caveat: + otter reports an *approximate* size, so `Len()` is approximate and the + `EvictPartialCapacityFilling=false` capacity gate (which compares `Len()` to + `Cap()` for exact equality) may not fire — set it to true when this arm is in + play. +- [ ] README Idea section --- @@ -222,20 +445,33 @@ cd examples/basic && go mod tidy ### Phase 1: Test Coverage Priority: fill empty test stubs before adding new features. -**`lfu/simplelfu/lfu_test.go`** -- DONE (100% coverage) +**`lfu/simplelfu/lfu_test.go`** -- DONE (98.3% coverage) - Test Add/Get/Contains/Peek/Remove/Purge/Keys/Values/Len - Test eviction behavior (least-frequently-used item removed) - Test frequency increment on repeated access - Edge cases: empty cache, single item, duplicate keys - Bug fixes applied: removed double Freq increment in Add, fixed Keys/Values slice init - -**`lfu/lfu_test.go`** -- DONE (93.2% coverage; uncovered methods are panic stubs) +- Bucket-index invariant enforced: every bucket in `evictList` holds at least one + entry, and `minFreq` always addresses a live bucket while the cache is non-empty. + `Add`'s eviction path and `removeElement` previously left an emptied bucket in + the map, so a later `minFreq` recompute selected it and panicked on a nil + dereference (`GetOldest`/`RemoveOldest`/`Resize`). Entry removal now funnels + through `detach`/`recomputeMinFreq`, `Add` only evicts when the cache is + non-empty (so `Resize(0)` + `Add` cannot panic), and the lookup helpers degrade + to a miss instead of panicking if the index is ever corrupted. + +**`lfu/lfu_test.go`** -- DONE (100% coverage) + +- `Resize`, `ContainsOrAdd`, `PeekOrAdd`, `RemoveOldest` and `GetOldest` are + covered directly; they are fully implemented (there are no panic stubs) but + were previously untested, which is why the simplelfu bucket-index panics went + unnoticed -- those methods are their public entry points. - Test thread-safe wrapper around simplelfu - Test eviction callbacks (buffered channel, DefaultEvictedBufferSize=16) - Test concurrent Add/Get under race detector - Concurrent tests for mixed operations, purge-while-reading, keys/values -**Root package tests (`cache_test.go`)** -- DONE (96.5% coverage) +**Root package tests (`cache_test.go`)** -- DONE (93.6% coverage) - `CacheWrapper`: hit/miss stats tracking, GetStats/ResetStats, Cap, Name, GetType, delegated methods - `AdaptiveCache`: Stats, Resize, Contains, Keys, Values, Len, Peek, ActivePolicy - `AdaptiveCache`: Add/Get with epoch-based switching, tryChangePolicy (switch, no-switch, skip-when-not-full) @@ -264,7 +500,7 @@ Implement the missing methods that currently return zero values: - `MigrationCold` (default, 0): start fresh — simple, causes temporary miss spike - `MigrationWarm`: on switch, purge zero-value shadow entries from new active policy, then copy all key/value pairs from old active via `Keys()`+`Peek()` -- `MigrationGradual`: Get-time promotion (miss in new active → peek old policy → add to new active) + Add-time drain (one key migrated per Add call). Migration window closes when all keys are drained, on `Purge()`, or at the next epoch boundary. +- `MigrationGradual`: Get-time promotion (during the window, `Get` takes the write lock and promotes the eligible key from the old policy into the new active BEFORE the counted lookup, so promoted requests register as active-policy hits — not misses) + Add-time drain (one key migrated per Add call). Migration window closes when no eligible keys remain (drained, promoted, or removed), on `Purge()`, on the next policy switch, and unconditionally at the next epoch boundary (`runEpoch` calls `closeMigrationLocked` first, so a workload that stops touching pending keys cannot leave the source pinned at full capacity holding real values). Switching away from an empty policy never opens a window. Bug fix applied during implementation: all three strategies now purge shadow zero-value entries from the new active policy at switch time, so callers never observe a shadow zero as a real cached value. @@ -292,6 +528,32 @@ Each new policy only needs to implement the `Cacher` interface and be wrapped by --- +## Releasing + +This is a multi-module repository, which has one trap that will bite anyone who +tags without knowing about it. + +Every sibling module depends on the others through a `replace` directive +pointing at a local path. That is what makes local development work -- and +**`replace` directives are ignored when a module is consumed as a dependency**. +So `policies/go.mod` requiring `github.com/sshaplygin/as-cache v0.0.0` builds +and tests perfectly here while being impossible for anyone else to use: + + reading github.com/sshaplygin/as-cache/go.mod at revision v0.0.0: + unknown revision v0.0.0 + +`make release-check` catches this; it is part of `make all`. Do not tag until it +passes. + +Releasing therefore goes bottom-up through the dependency graph -- root, then +`lfu`, then `policies`, then `policies/arc`, `policies/tinylfu` and `metrics` -- +updating each module's `require` to the version its dependency was just tagged +at. `bench` and `examples/*` are internal and are never tagged. + +Go module versions are immutable once the module proxy has fetched them. A +broken `v0.1.0` cannot be replaced, only superseded, so verify the whole chain +resolves before pushing any tag. + ## Rules 1. No emojis in code, comments, or documentation diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d25d45b --- /dev/null +++ b/Makefile @@ -0,0 +1,63 @@ +# Each of these directories is a separate Go module (own go.mod), so tooling is +# run once per module. The root .golangci.yml is shared by all of them. +MODULES := . lfu policies policies/arc policies/tinylfu metrics bench examples/basic examples/migration + +GOLANGCI_LINT_VERSION := v2.8.0 + +.PHONY: all +all: fmt vet lint test ## Format, vet, lint and test + +.PHONY: lint +lint: ## Run golangci-lint across all modules + @set -e; for m in $(MODULES); do \ + echo "==> lint $$m"; \ + ( cd $$m && golangci-lint run ./... ); \ + done + +.PHONY: lint-fix +lint-fix: ## Run golangci-lint with --fix and apply formatters across all modules + @set -e; for m in $(MODULES); do \ + echo "==> lint-fix $$m"; \ + ( cd $$m && golangci-lint run --fix ./... && golangci-lint fmt ./... ); \ + done + +.PHONY: fmt +fmt: ## Apply gofmt/goimports via the golangci-lint formatters + @set -e; for m in $(MODULES); do \ + echo "==> fmt $$m"; \ + ( cd $$m && golangci-lint fmt ./... ); \ + done + +.PHONY: vet +vet: ## Run go vet across all modules + @set -e; for m in $(MODULES); do \ + echo "==> vet $$m"; \ + ( cd $$m && go vet ./... ); \ + done + +.PHONY: test +test: ## Run tests with the race detector across all modules + @set -e; for m in $(MODULES); do \ + echo "==> test $$m"; \ + ( cd $$m && go test -race -short -count=1 ./... ); \ + done + +.PHONY: evidence +evidence: ## Replay the workload suite and print the policy comparison tables + ( cd bench && go test -count=1 -timeout 20m -v ./... ) + +.PHONY: tidy +tidy: ## Run go mod tidy across all modules + @set -e; for m in $(MODULES); do \ + echo "==> tidy $$m"; \ + ( cd $$m && go mod tidy ); \ + done + +.PHONY: install-tools +install-tools: ## Install golangci-lint at the pinned version + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + +.PHONY: help +help: ## Show this help + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 62d1512..5799234 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,66 @@ # as-cache — Adaptive Selection Cache -An experimental Go library that uses a **Multi-Armed Bandit (MAB)** algorithm to automatically select the optimal cache replacement policy at runtime. +[![CI](https://github.com/sshaplygin/as-cache/actions/workflows/ci.yml/badge.svg)](https://github.com/sshaplygin/as-cache/actions/workflows/ci.yml) +[![Go Reference](https://pkg.go.dev/badge/github.com/sshaplygin/as-cache.svg)](https://pkg.go.dev/github.com/sshaplygin/as-cache) +[![Go Report Card](https://goreportcard.com/badge/github.com/sshaplygin/as-cache)](https://goreportcard.com/report/github.com/sshaplygin/as-cache) -## Disclaimer +A Go library that uses a **Multi-Armed Bandit (MAB)** algorithm to select the +cache replacement policy at runtime, measuring candidate policies against your +real traffic instead of asking you to guess. -Experimental. Running multiple policies in parallel multiplies memory consumption proportionally to the number of candidate policies. +## Status + +Experimental, but the claims here are measured rather than asserted -- see +[Evidence](#evidence). Two things are worth knowing before adopting it. + +**No single policy wins everywhere, and that is the point.** On real traces the +best fixed policy changes: 2Q wins on the Twitter and OLTP traces, W-TinyLFU on +the ARC P3 and LIRS traces -- and on OLTP, W-TinyLFU is second-*worst*. Tuned +sensibly, adaptive selection lands within about a point of the best fixed +policy on most traces and beats it on one, without being told which to pick. + +**It is sensitive to configuration.** The same traces with a too-short epoch +lose up to 7 points and cost 30x the per-operation time, because the cache +spends its life migrating rather than serving. See +[Configuring it](#configuring-it) before drawing conclusions from your own +numbers. + +**Memory costs less than the obvious guess.** Running N policies in parallel +does not multiply memory by N, because shadow policies hold keys and eviction +bookkeeping but never real values. Measured with six policies over 50k entries +of 256-byte values: + +| Configuration | Memory | Multiplier | +| --- | --- | --- | +| single LRU | 18.5 MiB | 1.00x | +| adaptive, 6 policies | 48.9 MiB | 2.65x | +| adaptive, 6 policies, `ShadowSampleRate: 0.05` | 24.4 MiB | 1.32x | + +Per-operation cost on a warm cache, same configurations (`Get`, 0 allocs/op +throughout): + +| Configuration | ns/op | +| --- | --- | +| single LRU | 32 | +| adaptive, 6 policies | 596 | +| adaptive, 6 policies, sampled | 85 | + +### When to use it + +- You do not know which policy suits your traffic, and cannot easily find out. +- Your traffic changes shape and you would rather not re-tune. +- You want the measurement more than the switching -- see advisor mode on the + roadmap. + +### When not to use it + +- You have already measured your traffic and know which policy wins. Use that + policy directly; this library's best case is roughly to match it. +- The hot path is latency-critical at single-digit nanoseconds. Even sampled, + the adaptive layer costs several times a bare LRU per operation. +- You need a hard memory ceiling. The multiplier is modest but real. +- You cannot give it enough traffic per epoch to measure anything. The bandit + needs many requests per epoch to tell arms apart. ## Problem @@ -14,7 +70,7 @@ Choosing the right cache replacement algorithm for a workload is a separate rese Every epoch the background goroutine: -1. Collects hit/miss statistics from each shadow policy. +1. Collects hit/miss statistics from every policy — shadows and the active one alike. 2. Feeds them as Beta-distribution parameters into the MAB bandit. 3. Samples from the distributions and switches the active policy to the winner. 4. Shadow caches continue tracking access patterns with zero-value dummy entries so no real data leaks. @@ -30,10 +86,12 @@ See [examples/basic/main.go](examples/basic/main.go) for a complete runnable exa | Policy | Status | Notes | | --- | --- | --- | | LRU | implemented | via `hashicorp/golang-lru/v2` | -| LFU | implemented | native O(1) implementation in `lfu/` | -| 2Q | planned | — | -| ARC | planned | — | -| Random | planned | — | +| LFU | implemented | native O(1) implementation in `lfu/`; `policies.NewLFU` | +| 2Q | implemented | `policies.NewTwoQueue` | +| Random | implemented | `policies.NewRandomPolicy` | +| TTL | implemented | `policies.NewTTL` | +| ARC | implemented | `policies/arc` — separate module, patented | +| W-TinyLFU | implemented | `policies/tinylfu` — separate module | ## AdaptiveCache API @@ -69,6 +127,17 @@ type Settings struct { // MigrationStrategy controls data transfer on policy switch. // Default: MigrationCold. MigrationStrategy MigrationStrategy + + // ShadowSampleRate has shadows track a fraction of the keyspace. + // Zero means 1 (no sampling). See "Reducing shadow overhead". + ShadowSampleRate float64 + MinShadowCapacity int + + // Switch stability gates; all inactive at zero. + // See "Keeping switches stable". + MinHitRateImprovement float64 + SwitchCooldownEpochs int64 + MinEpochRequests int64 } ``` @@ -78,14 +147,15 @@ type Settings struct { | --- | --- | --- | | `MigrationCold` (default) | New active policy starts empty | Simple; causes a temporary miss spike | | `MigrationWarm` | All key/value pairs copied at switch time | No miss spike; O(n) work at switch | -| `MigrationGradual` | Keys lazily promoted on Get-miss; one key drained per Add | Spreads migration cost; window closes at next epoch | +| `MigrationGradual` | Keys promoted on Get; one key drained per Add | Spreads migration cost; window closes at the next epoch at the latest | ## Architecture ```text AdaptiveCache |-- active policy (CacheWrapper -> real Cacher impl) - |-- shadow policy (CacheWrapper -> real Cacher impl, zero-value adds only) + |-- shadow policy (CacheWrapper -> real Cacher impl, zero-value adds only, + | optionally a sampled miniature -- see ShadowSampleRate) |-- Bandit (Thompson Sampling via stitchfix/mab) |-- background goroutine (epoch ticker -> tryChangePolicy -> migrateData) ``` @@ -94,7 +164,8 @@ AdaptiveCache ```go type Bandit interface { - // RecordStats delivers shadow-cache hit/miss stats for one epoch. + // RecordStats delivers one policy's hit/miss stats since its last + // report; every policy reports, the active one included. RecordStats(stats ShadowStats) // SelectPolicy returns the policy that should become active next epoch. @@ -104,13 +175,352 @@ type Bandit interface { A full Thompson Sampling adapter using `stitchfix/mab` is provided in [examples/basic/main.go](examples/basic/main.go). +## Reducing shadow overhead + +Running policies in parallel costs something on every operation: each shadow is +another lookup and another lock. Since a shadow exists only to estimate a hit +rate, and a hit rate can be estimated from a sample, `ShadowSampleRate` lets +shadows track a deterministic fraction of the keyspace instead of mirroring +everything. + +```go +&ascache.Settings{ + EpochDuration: time.Minute, + ShadowSampleRate: 0.05, // shadows track 5% of keys +} +``` + +Shadows shrink along with the rate, so each remains a faithful miniature of a +full-size cache rather than an undersized one, and every shadow samples the same +keys so their hit rates stay comparable. The active policy still serves every +key -- only the measurement is sampled, and it is sampled for the active policy +too, so no arm is judged on more evidence than another. `Stats()` continues to +report real, unsampled traffic. + +The effect is that per-operation cost stops scaling with the number of policies +(measured with mutex-backed stub policies on an M1 Max, `-benchtime=200ms`): + +| Benchmark | shadows | sampling off | rate 0.05 | +| --- | --- | --- | --- | +| `Get` | 1 | 100 ns/op | 36 ns/op | +| `Get` | 3 | 145 ns/op | 38 ns/op | +| `Add` | 1 | 109 ns/op | 52 ns/op | +| `Add` | 3 | 189 ns/op | 57 ns/op | +| `MixedParallel` | 1 | 184 ns/op | 85 ns/op | + +Sampling is off by default. Very small caches disable it automatically, since a +miniature of a handful of entries measures noise rather than a policy. + +## Keeping switches stable + +By default every bandit selection is applied. On noisy traffic two policies that +perform almost identically can trade places every epoch, and each switch costs a +migration. Three settings damp that, all inactive at their zero value: + +```go +&ascache.Settings{ + MinHitRateImprovement: 0.02, // require a 2-point hit-rate win to switch + SwitchCooldownEpochs: 3, // and at most one switch every 3 epochs + MinEpochRequests: 500, // and ignore epochs with thin evidence +} +``` + +## Ready-made policies + +The core module has no dependencies. Ready-made arms live in a companion +module, so you pull in a cache library only if you use one: + +```bash +go get github.com/sshaplygin/as-cache/policies +``` + +```go +lru, _ := policies.NewLRU[string, int](10000) +twoQ, _ := policies.NewTwoQueue[string, int](10000) + +cache, err := ascache.NewAdaptiveCache( + []ascache.Policy[string, int]{ + lru, + twoQ, + policies.NewRandomPolicy[string, int](10000), + policies.NewTTL[string, int](10000, 5*time.Minute), + }, + myBandit, + &ascache.Settings{EpochDuration: time.Minute, ShadowSampleRate: 0.05}, +) +``` + +| Policy | Constructor | Notes | +| --- | --- | --- | +| LRU | `policies.NewLRU` | `hashicorp/golang-lru/v2` | +| LFU | `policies.NewLFU` | this repository's O(1) LFU; strong on stationary popularity, weak when it shifts | +| 2Q | `policies.NewTwoQueue` | scan-resistant; a scan cannot flush the working set | +| Random | `policies.NewRandomPolicy` | no bookkeeping; the control arm worth beating | +| TTL | `policies.NewTTL` | expiry as well as recency | +| ARC | `policies/arc.NewPolicy` | separate module — see below | +| W-TinyLFU | `policies/tinylfu.NewPolicy` | separate module; the strongest baseline | + +`Random` is worth keeping in the mix precisely because it assumes nothing: a +policy that cannot beat random on your traffic is not earning its bookkeeping. + +### ARC is a separate module + +```bash +go get github.com/sshaplygin/as-cache/policies/arc +``` + +ARC is patented by IBM (US 6,996,676), which is why upstream `hashicorp/golang-lru` +moved it to its own module in v2. This repository keeps that split, so importing +`policies` never pulls a patented implementation into your build and the choice +to use ARC is always explicit. Whether the patent still restricts anything is a +question for you and your counsel. + +### Adapting your own cache + +Any type satisfying `Cacher[K, V]` can be an arm. If your cache does not report +evictions or cannot be resized — as `2Q` and `ARC` do not — wrap it: + +```go +cache, err := policies.Adapt[string, int](size, func(size int) (policies.PartialCacher[string, int], error) { + return mylib.New[string, int](size) +}) +``` + +Note that `Resize` on an adapted cache rebuilds it, discarding whatever +adaptation the algorithm had learned. `AdaptiveCache` resizes shadow policies +when its own capacity changes, so adapted policies are heavier arms to carry +than natively resizable ones. + +### W-TinyLFU + +```bash +go get github.com/sshaplygin/as-cache/policies/tinylfu +``` + +Carried in its own module so otter and its dependencies stay out of builds that +do not use it. This is the arm worth including if the question is whether an +adaptive cache beats the state of the art rather than whether it beats LRU. + +Note that otter reports an approximate size, so this policy's `Len()` is +approximate. Set `EvictPartialCapacityFilling: true` when using it, since the +capacity gate compares `Len()` against `Cap()` for exact equality. + +## Advisor mode + +The safest way to adopt this library is not to let it switch anything. In +`ObserveOnly` mode the cache behaves exactly like the first policy you give it +-- nothing ever migrates, nothing ever switches -- while every other policy is +measured in the background against your real traffic. + +```go +cache, err := ascache.NewAdaptiveCache( + []ascache.Policy[string, int]{lru, twoQ, tinyLFU}, + nil, // observing needs no bandit + &ascache.Settings{ + EpochDuration: time.Minute, + ObserveOnly: true, + ShadowSampleRate: 0.05, + }, +) + +// ... later, after real traffic ... +fmt.Println(cache.Advice()) +``` + +```text +On this traffic TwoQueue beats LRU by 3.28 points of hit rate, over 240 epochs. +Rates are estimated from 5.0% of the keyspace. + +policy hit rate hits misses + TwoQueue 59.62% 596200 403800 +*LRU 56.34% 563400 436600 + Random 54.80% 548000 452000 + +* currently active +``` + +That answers a question that is otherwise expensive to ask, at no risk: you +learn whether a different eviction policy would serve your traffic better, and +by how much, without changing what your cache does. Acting on the answer is +then your choice -- switch to that policy directly, or turn `ObserveOnly` off +and let the bandit do it. + +`Advice()` is safe to call at any time. Check `Epochs` before believing it: a +handful of epochs is not evidence. + +## Observability + +A cache that changes its own eviction policy needs to be visible in staging. +The `metrics` module turns the cache's accounting into a scrapeable snapshot +and publishes it via `expvar` (standard library only): + +```bash +go get github.com/sshaplygin/as-cache/metrics +``` + +```go +if err := metrics.Publish("cache", myCache); err != nil { + log.Panic(err) +} +// snapshot now appears in /debug/vars under "cache" +``` + +`metrics.Take(cache)` returns the same data as a struct if you would rather +feed it somewhere else. The series worth graphing is `active_policy` over +time; the one worth alerting on is `improvement`, which measures how much hit +rate the cache is currently leaving on the table. + +For Prometheus, wrap `metrics.Take` in a collector -- how metrics are named and +labelled belongs to your application, not to a cache library, so this package +does not impose a dependency on it. + +## Evidence + +`make evidence` replays a suite of deterministic workloads against every policy +and against the adaptive cache. The numbers below are from an M1 Max, cache +capacity 500, 200k requests per workload. Reproduce with `make evidence`; the +generators are in [bench/workload.go](bench/workload.go). + +Hit rate by policy and workload: + +| Workload | LRU | LFU | 2Q | ARC | Random | W-TinyLFU | +| --- | --- | --- | --- | --- | --- | --- | +| zipf (skewed popularity) | 66.9% | **73.5%** | 72.0% | 73.2% | 62.6% | 73.3% | +| uniform (no structure) | 10.0% | 10.0% | 10.0% | 10.0% | 10.1% | **12.3%** | +| loop (cycle just over capacity) | 0.0% | 0.0% | 68.6% | 0.1% | 82.1% | **94.0%** | +| scan (hot set + sweeps) | 30.0% | **40.0%** | **40.0%** | **40.0%** | 32.0% | 39.7% | +| phase-shift (alternating regimes) | 34.5% | 69.7% | 61.5% | 39.9% | 68.2% | **82.1%** | + +Two things stand out. LRU and LFU both score **exactly zero** on `loop`, where a +cyclic scan just over capacity evicts every key immediately before it is needed +again -- that is the textbook pathology, and it is worth knowing your workload +is not that shape. And W-TinyLFU wins or ties nearly everywhere here. + +### Does adaptive selection beat picking one policy? + +On these workloads: **no, and this is the honest result.** + +| Workload | Adaptive | Best fixed | Worst fixed | Adaptive vs best | +| --- | --- | --- | --- | --- | +| zipf | 73.3% | LFU 73.5% | 62.6% | -0.2 pts | +| uniform | 10.0% | W-TinyLFU 12.3% | 10.0% | -2.3 pts | +| loop | 77.5% | W-TinyLFU 94.0% | 0.0% | -16.5 pts | +| scan | 38.9% | LFU/2Q/ARC 40.0% | 30.0% | -1.1 pts | +| phase-shift | 78.8% | W-TinyLFU 82.1% | 34.5% | -3.3 pts | + +Adaptive selection reliably beats the *worst* fixed choice, sometimes hugely +(77.5% against LRU's 0.0% on `loop`). It never meaningfully beats the *best* +one. Even on `phase-shift` -- the workload built specifically to need adaptation +-- a fixed W-TinyLFU wins by 3.8 points. + +The timeline explains why. Replaying `phase-shift` and sampling `ActivePolicy()` +throughout: + +```text +phase Z------L------Z------L------Z------L------Z------L------ (Z = zipf, L = loop) +LRU ### +TwoQueue ####### +ARC ## +TinyLFU ######################################################### + +share of time active: LRU 2%, TwoQueue 6%, ARC 1%, TinyLFU 90% +``` + +The bandit works exactly as designed: it explores, identifies W-TinyLFU, and +holds it for 90% of the run. It does not oscillate at phase boundaries, because +there is no crossover to exploit -- W-TinyLFU is the best arm in *both* regimes. +The remaining gap is the price of exploring and of migrating between arms. + +So the case for this library is not "it beats the best policy." It is: + +- **You do not know which policy is best for your traffic**, and the cost of + guessing wrong is large (0.0% vs 92.3% on `loop`). Adaptive selection bounds + that downside without requiring you to know. +- **It tells you what to use.** The most valuable output may be the measurement + rather than the switching -- see the roadmap's advisor mode. + +For a workload that genuinely crosses over, the picture could differ. These are +synthetic; real traces are the next thing to run. + +### Real traces + +`./scripts/fetch-traces.sh` downloads five published traces (nothing is +committed -- see [docs on traces](#real-traces)), then +`AS_CACHE_TRACES=... make evidence` replays them. Adaptive here runs a 50ms +epoch with warm migration and `ShadowSampleRate: 0.05`: + +| Trace | Requests | Best fixed | Worst fixed | Adaptive | Delta | +| --- | --- | --- | --- | --- | --- | +| Twitter Twemcache cluster052 | 1.0M | 2Q 59.6% | LFU 41.4% | 59.4% | -0.25 pts | +| ARC OLTP (FAST '03) | 0.9M | 2Q 68.3% | LFU 45.4% | 67.1% | -1.19 pts | +| ARC P3 (FAST '03) | 2.0M | W-TinyLFU 11.7% | LRU 1.9% | **12.7%** | **+0.92 pts** | +| LIRS 2_pools | 100k | W-TinyLFU 54.8% | Random 50.1% | 54.4% | -0.36 pts | +| LIRS loop | 505k | W-TinyLFU 45.9% | LRU/LFU 0.0% | 42.5%* | -3.43 pts | + +\* `loop` needs a 2ms epoch: it is short and changes character quickly, so a +50ms epoch gives the bandit too few epochs to react and it drops to 33.3%. + +Note that the best fixed policy is **not the same policy across traces**. On +OLTP, W-TinyLFU -- the strongest general-purpose baseline -- comes second to +last at 63.2% while 2Q wins at 68.3%. That is the case for not committing to a +policy in advance, and it does not show up on synthetic workloads, where +W-TinyLFU wins nearly everything. + +LFU is the sharpest illustration of why synthetic workloads mislead. It is the +**best** policy on synthetic `zipf` (73.5%) and the **worst** on both large real +traces (41.4% on Twitter, 45.4% on OLTP). Synthetic Zipf holds popularity +*stationary*, which is exactly the assumption classic LFU makes; real traffic +shifts, and an entry that was hot once keeps a frequency count that holds it +resident long after it stops being useful. That is the failure W-TinyLFU's aged +frequency sketch exists to avoid, and it is invisible until you replay real +traffic. + +### Configuring it + +The epoch duration is the setting that matters most, and the failure mode is +not subtle. Measured on the ARC P3 trace with a 20k-entry cache: + +| Configuration | Hit rate | ns/op | +| --- | --- | --- | +| 50ms epoch, warm migration | 12.2% | 540 | +| 2ms epoch, warm migration | 4.8% | 13,476 | +| 2ms epoch, cold migration | 0.9% | 580 | + +An epoch short enough to trigger frequent switches makes the cache copy its +entire contents on every switch, so it spends its time migrating rather than +serving. Cold migration is worse: it discards the cache at each switch, which +on the OLTP trace costs 28 points. + +Rules of thumb: + +- Make the epoch long enough that migrating the cache is a small fraction of + the work done in it, and short enough that the workload sees many epochs. +- Prefer `MigrationWarm`. `MigrationCold` is only reasonable if switches are + rare. +- The stability gates help on steady traffic and hurt on fast-changing traffic + -- they cost 37 points on `loop`, which needs to re-adapt constantly. + +### Does sampling distort the comparison? + +Milestone 2's sampled shadows are only sound if a 5% miniature ranks policies +the way full-size shadows would. Measured directly: + +```text +zipf full-size: ARC=81.4% 2Q=81.2% TinyLFU=79.8% LRU=79.2% TTL=79.2% Random=28.9% + sampled: ARC=78.7% 2Q=78.3% TinyLFU=77.1% LRU=76.7% TTL=76.5% Random=27.9% + +scan full-size: 2Q=28.3% ARC=28.3% TinyLFU=27.0% LRU=21.4% TTL=21.4% Random=17.1% + sampled: 2Q=27.1% ARC=27.1% TinyLFU=25.1% TTL=20.5% LRU=20.5% Random=16.6% +``` + +The order is preserved. Sampled rates run uniformly 1-3 points pessimistic, but +the bias applies to every arm alike, so comparisons hold and the bandit picks +the same arm. + ## TODO -- [ ] 2Q policy wrapper (`hashicorp/golang-lru/v2` 2Q variant) -- [ ] ARC policy wrapper (`hashicorp/golang-lru/v2` ARC variant) -- [ ] Random eviction policy -- [ ] TTL-based policy (`hashicorp/golang-lru/v2/expirable`) -- [ ] README: detailed benchmarks comparing policies per workload type +- [ ] Trace-driven benchmarks (ARC paper traces, `twitter/cache-trace`) +- [ ] Advisor mode: measure without switching, and report ## References diff --git a/advice.go b/advice.go new file mode 100644 index 0000000..f51fdc3 --- /dev/null +++ b/advice.go @@ -0,0 +1,168 @@ +package ascache + +import ( + "fmt" + "sort" + "strings" +) + +// PolicyReport is one policy's measured performance over the cache's lifetime. +type PolicyReport struct { + // Policy is the arm this report describes. + Policy PolicyType + // Hits and Misses are the requests measured for this policy in its current + // role - as the active policy, or as a shadow - since it last changed + // role. Under ShadowSampleRate these are counts over the sampled + // substream, not over all traffic: the rate is meaningful, the magnitude + // is a sample. + Hits int64 + Misses int64 + // Active reports whether this policy was serving requests when the report + // was taken. + Active bool +} + +// HitRate returns the fraction of measured requests this policy served, or 0 +// when it has measured nothing. +func (r PolicyReport) HitRate() float64 { + total := r.Hits + r.Misses + if total == 0 { + return 0 + } + + return float64(r.Hits) / float64(total) +} + +// Advice is what the cache has learned about which policy suits the traffic it +// has seen. +// +// It is the answer to a question that is otherwise expensive to ask: not "is +// my cache fast" but "would a different eviction policy serve my traffic +// better, and by how much". Running in ObserveOnly mode makes that answerable +// without ever changing what the cache does. +type Advice struct { + // Epochs is how many epochs actually measured something and fed this + // advice. Advice from a handful of epochs is not worth acting on. It + // counts reporting epochs rather than elapsed ticks, so an epoch the + // capacity gate skipped is not counted as evidence. + // + // It resets to nothing for a policy that changes role, so shortly after a + // switch the advice is deliberately thin rather than confidently stale. + Epochs int64 + // Active is the policy serving requests. + Active PolicyType + // Best is the policy with the highest measured hit rate. + Best PolicyType + // Improvement is how many percentage points Best beats Active by. It is + // zero when they are the same policy. + Improvement float64 + // Sampled reports whether the measurements come from a sampled substream, + // in which case the rates are estimates. + Sampled bool + // SampleRate is the fraction of the keyspace measured, 1 when sampling is + // off. + SampleRate float64 + // Reports holds every policy, best hit rate first. + Reports []PolicyReport +} + +// String renders the advice as a short human-readable summary. +func (a Advice) String() string { + if len(a.Reports) == 0 { + return "no measurements yet" + } + + var b strings.Builder + + if a.Best == a.Active || a.Improvement <= 0 { + fmt.Fprintf(&b, "%s is the best of the %d policies measured over %d epochs.\n", + a.Active, len(a.Reports), a.Epochs) + } else { + fmt.Fprintf(&b, "On this traffic %s beats %s by %.2f points of hit rate, over %d epochs.\n", + a.Best, a.Active, a.Improvement*100, a.Epochs) + } + + if a.Sampled { + fmt.Fprintf(&b, "Rates are estimated from %.1f%% of the keyspace.\n", a.SampleRate*100) + } + + fmt.Fprintf(&b, "\n%-10s %9s %12s %12s\n", "policy", "hit rate", "hits", "misses") + for _, r := range a.Reports { + marker := " " + if r.Active { + marker = "*" + } + fmt.Fprintf(&b, "%s%-9s %8.2f%% %12d %12d\n", + marker, r.Policy, r.HitRate()*100, r.Hits, r.Misses) + } + b.WriteString("\n* currently active\n") + + return b.String() +} + +// observerBandit stands in when a cache is built for observation alone. It +// records nothing and selects nothing, because in ObserveOnly mode its +// selection would be discarded anyway. +type observerBandit struct{} + +func (observerBandit) RecordStats(_ ShadowStats) {} +func (observerBandit) SelectPolicy() PolicyType { return Undefined } + +// Advice reports which policy has served this cache's traffic best. +// +// It is safe to call at any time and does not disturb measurement. The advice +// is only as good as the traffic behind it: a cache that has run for a few +// epochs, or one whose policies are within noise of each other, has nothing +// useful to say, and Epochs is included so a caller can tell. +func (c *AdaptiveCache[K, V]) Advice() Advice { + c.mu.RLock() + defer c.mu.RUnlock() + + advice := Advice{ + Epochs: c.reportingEpochs, + Active: c.activePolicy, + Best: c.activePolicy, + Sampled: c.sampler.sampling, + SampleRate: c.sampler.rate, + Reports: make([]PolicyReport, 0, len(c.tenureStats)), + } + + for policyType, stats := range c.tenureStats { + advice.Reports = append(advice.Reports, PolicyReport{ + Policy: policyType, + Hits: stats.Hits, + Misses: stats.Misses, + Active: policyType == c.activePolicy, + }) + } + + // Ties are broken by policy so the answer is stable. Ranging over a map + // gives a random order, and a stable sort preserves it among equal rates, + // so without this a cache whose arms are performing identically would name + // a different "best" policy on every call. + sort.SliceStable(advice.Reports, func(i, j int) bool { + left, right := advice.Reports[i], advice.Reports[j] + if left.HitRate() != right.HitRate() { + return left.HitRate() > right.HitRate() + } + + return left.Policy < right.Policy + }) + + if len(advice.Reports) == 0 { + return advice + } + + best := advice.Reports[0] + advice.Best = best.Policy + + for _, r := range advice.Reports { + if r.Active { + advice.Improvement = best.HitRate() - r.HitRate() + + break + } + } + + return advice +} diff --git a/advice_test.go b/advice_test.go new file mode 100644 index 0000000..46e1ef5 --- /dev/null +++ b/advice_test.go @@ -0,0 +1,340 @@ +package ascache + +import ( + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// makeObserver builds an observe-only cache over two mock policies. It passes +// no bandit at all, which is the point: observing should require no strategy. +func makeObserver(t *testing.T) ( + *AdaptiveCache[string, int], + *mockPolicy[string, int], + *mockPolicy[string, int], +) { + t.Helper() + + lru := newMockPolicy[string, int](LRU, 100) + lfu := newMockPolicy[string, int](LFU, 100) + + ac, err := NewAdaptiveCache[string, int]( + []Policy[string, int]{lru, lfu}, + nil, + &Settings{EpochDuration: 24 * time.Hour, ObserveOnly: true}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + return ac, lru, lfu +} + +func TestObserveOnly_NeedsNoBandit(t *testing.T) { + ac, _, _ := makeObserver(t) + + assert.Equal(t, LRU, ac.ActivePolicy()) +} + +func TestObserveOnly_StillRequiresBanditWhenSwitching(t *testing.T) { + _, err := NewAdaptiveCache[string, int]( + []Policy[string, int]{newMockPolicy[string, int](LRU, 10)}, + nil, + &Settings{EpochDuration: time.Hour}, + ) + + assert.ErrorIs(t, err, ErrNilBandit, + "a cache that switches still needs a strategy to switch by") +} + +// TestObserveOnly_NeverSwitches is the guarantee the mode exists to make: the +// cache behaves exactly like the policy it was built with, whatever the +// measurements say. +func TestObserveOnly_NeverSwitches(t *testing.T) { + lru := newMockPolicy[string, int](LRU, 100) + lfu := newMockPolicy[string, int](LFU, 100) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, + // A bandit that always demands the other policy. It must be ignored. + &mockBandit{next: LFU}, + &Settings{EpochDuration: 24 * time.Hour, ObserveOnly: true}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + ac.Add("a", 1) + primeActiveStats(ac, 1, 99) + primeStats(lfu, 99, 1) + + for i := 0; i < 5; i++ { + ac.runEpoch() + } + + assert.Equal(t, LRU, ac.ActivePolicy(), + "ObserveOnly must never change the active policy, however good another arm looks") +} + +// TestObserveOnly_MeasuresWithoutAFullCache guards the capacity gate: it +// exists to avoid switching on thin evidence, but in observe mode nothing +// switches, so gating would only suppress the measurement being asked for. +func TestObserveOnly_MeasuresWithoutAFullCache(t *testing.T) { + ac, _, lfu := makeObserver(t) + + // EvictPartialCapacityFilling is false and the cache holds 1 of 100 + // entries, which would normally gate the epoch entirely. + ac.Add("a", 1) + primeActiveStats(ac, 10, 90) + primeStats(lfu, 90, 10) + + ac.runEpoch() + + advice := ac.Advice() + require.Len(t, advice.Reports, 2, "both policies must be measured") + assert.Equal(t, LFU, advice.Best) +} + +func TestAdvice_IdentifiesTheBetterPolicy(t *testing.T) { + ac, _, lfu := makeObserver(t) + + primeActiveStats(ac, 30, 70) // active LRU: 30% + primeStats(lfu, 80, 20) // shadow LFU: 80% + ac.runEpoch() + + advice := ac.Advice() + + assert.Equal(t, LFU, advice.Best) + assert.Equal(t, LRU, advice.Active) + assert.InDelta(t, 0.50, advice.Improvement, 1e-9, + "LFU beats LRU by 50 points and the advice should say so") + assert.Equal(t, int64(1), advice.Epochs) + + summary := advice.String() + assert.Contains(t, summary, "LFU beats LRU") + assert.Contains(t, summary, "50.00 points") +} + +func TestAdvice_AccumulatesAcrossEpochs(t *testing.T) { + ac, _, lfu := makeObserver(t) + + for i := 0; i < 4; i++ { + primeActiveStats(ac, 10, 10) + primeStats(lfu, 15, 5) + ac.runEpoch() + } + + advice := ac.Advice() + require.Equal(t, int64(4), advice.Epochs) + + byPolicy := map[PolicyType]PolicyReport{} + for _, r := range advice.Reports { + byPolicy[r.Policy] = r + } + + assert.Equal(t, int64(40), byPolicy[LRU].Hits, "advice must span every epoch, not just the last") + assert.Equal(t, int64(60), byPolicy[LFU].Hits) + assert.InDelta(t, 0.75, byPolicy[LFU].HitRate(), 1e-9) +} + +func TestAdvice_SaysNothingBeforeMeasuring(t *testing.T) { + ac, _, _ := makeObserver(t) + + advice := ac.Advice() + + assert.Zero(t, advice.Epochs) + assert.Empty(t, advice.Reports) + assert.Equal(t, "no measurements yet", advice.String()) +} + +func TestAdvice_ReportsWhenTheActivePolicyIsAlreadyBest(t *testing.T) { + ac, _, lfu := makeObserver(t) + + primeActiveStats(ac, 90, 10) + primeStats(lfu, 10, 90) + ac.runEpoch() + + advice := ac.Advice() + + assert.Equal(t, LRU, advice.Best) + assert.Zero(t, advice.Improvement) + assert.Contains(t, advice.String(), "LRU is the best") +} + +func TestAdvice_FlagsSampledEstimates(t *testing.T) { + lru := newMockPolicy[string, int](LRU, 100000) + lfu := newMockPolicy[string, int](LFU, 100000) + + ac, err := NewAdaptiveCache[string, int]( + []Policy[string, int]{lru, lfu}, nil, + &Settings{EpochDuration: 24 * time.Hour, ObserveOnly: true, ShadowSampleRate: 0.05}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + for i := 0; i < 2000; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + ac.runEpoch() + + advice := ac.Advice() + + assert.True(t, advice.Sampled, "sampled measurements must be flagged as estimates") + assert.InDelta(t, 0.05, advice.SampleRate, 1e-9) + assert.Contains(t, advice.String(), "estimated from 5.0%") +} + +// TestAdvice_SafeUnderConcurrentUse checks that reading advice while the cache +// is serving traffic neither races nor blocks writers out. +func TestAdvice_SafeUnderConcurrentUse(t *testing.T) { + ac, _, _ := makeObserver(t) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + for i := 0; i < 4; i++ { + wg.Add(1) + go func(seed int) { + defer wg.Done() + for j := 0; ; j++ { + select { + case <-stop: + return + default: + key := "k" + strconv.Itoa((seed+j)%50) + ac.Add(key, j) + ac.Get(key) + } + } + }(i) + } + + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 200; i++ { + advice := ac.Advice() + _ = advice.String() + } + }() + + time.Sleep(50 * time.Millisecond) + close(stop) + wg.Wait() +} + +// TestAdvice_StringIsReadable checks the summary a user actually reads. +func TestAdvice_StringIsReadable(t *testing.T) { + ac, _, lfu := makeObserver(t) + + primeActiveStats(ac, 25, 75) + primeStats(lfu, 60, 40) + ac.runEpoch() + + summary := ac.Advice().String() + t.Logf("\n%s", summary) + + for _, want := range []string{"LFU", "LRU", "hit rate", "currently active"} { + assert.True(t, strings.Contains(summary, want), + "summary should mention %q:\n%s", want, summary) + } +} + +// TestAdvice_DoesNotRecommendRevertingACorrectSwitch guards the defect that +// made advice actively harmful in adaptive mode. Accumulating a policy's +// measurements across a role change pooled its active tenure (full capacity, +// all traffic) with its shadow tenure (miniature capacity, a sample), and left +// the outgoing policy's long good history outweighing the incoming policy's +// short one - so right after a correct switch, Advice named the policy the +// cache had just moved away from as best, for as many epochs as the history +// was long. +func TestAdvice_DoesNotRecommendRevertingACorrectSwitch(t *testing.T) { + lru := newMockPolicy[string, int](LRU, 100) + lfu := newMockPolicy[string, int](LFU, 100) + bandit := &mockBandit{next: LRU} + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, bandit, + &Settings{EpochDuration: 24 * time.Hour, EvictPartialCapacityFilling: true}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + // A long stretch where LRU is active and excellent. + for i := 0; i < 30; i++ { + primeActiveStats(ac, 90, 10) + primeStats(lfu, 10, 90) + ac.runEpoch() + } + require.Equal(t, LRU, ac.ActivePolicy()) + + // The workload turns, the bandit switches to LFU, and LFU is now the + // better policy while LRU shadows badly. + bandit.next = LFU + primeActiveStats(ac, 10, 90) + primeStats(lfu, 95, 5) + ac.runEpoch() + require.Equal(t, LFU, ac.ActivePolicy(), "expected the switch to land") + + primeActiveStats(ac, 95, 5) + primeStats(lru, 10, 90) + ac.runEpoch() + + advice := ac.Advice() + + assert.Equal(t, LFU, advice.Best, + "advice must reflect the policies in their current roles, not recommend reverting a correct switch") + assert.Zero(t, advice.Improvement, + "the active policy is the best one, so there is nothing being left on the table") +} + +// TestAdvice_EpochsCountsOnlyMeasuredEpochs guards a counter that reported +// elapsed ticks as evidence. The capacity gate can skip measurement for +// thousands of ticks, and reporting those as epochs behind a recommendation +// overstated the evidence by exactly that much. +func TestAdvice_EpochsCountsOnlyMeasuredEpochs(t *testing.T) { + lru := newMockPolicy[string, int](LRU, 100) + lfu := newMockPolicy[string, int](LFU, 100) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, &mockBandit{next: LRU}, + // The gate is on and the cache holds far less than its capacity, so + // no epoch measures anything. + &Settings{EpochDuration: 24 * time.Hour, EvictPartialCapacityFilling: false}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + ac.Add("a", 1) + for i := 0; i < 50; i++ { + ac.runEpoch() + } + + advice := ac.Advice() + + assert.Zero(t, advice.Epochs, + "50 gated ticks measured nothing, so they are not evidence") + assert.Empty(t, advice.Reports) + assert.Equal(t, "no measurements yet", advice.String()) +} + +// TestAdvice_BestIsDeterministicOnTies guards a Best that flapped between arms +// on an unchanged cache: Reports was built by ranging a map and sorted stably +// on hit rate alone, so tied policies kept their random order. +func TestAdvice_BestIsDeterministicOnTies(t *testing.T) { + ac, _, lfu := makeObserver(t) + + // Both arms measure exactly the same, so every ordering is a valid sort. + primeActiveStats(ac, 50, 50) + primeStats(lfu, 50, 50) + ac.runEpoch() + + first := ac.Advice().Best + for i := 0; i < 200; i++ { + assert.Equal(t, first, ac.Advice().Best, + "Best must not change while the cache does not") + } +} diff --git a/bench/bandit.go b/bench/bandit.go new file mode 100644 index 0000000..59a606c --- /dev/null +++ b/bench/bandit.go @@ -0,0 +1,183 @@ +package bench + +import ( + "math" + "math/rand/v2" + "sync" + + ascache "github.com/sshaplygin/as-cache" +) + +// ThompsonBandit picks a policy by Thompson sampling over Beta posteriors. +// +// Each arm's hit rate is modelled as a Beta distribution updated from the +// hits and misses reported for it. Selecting means drawing one sample per arm +// and taking the best draw, so an arm is chosen in proportion to the +// probability that it is genuinely the best - which explores uncertain arms +// without ever committing to a fixed exploration schedule. +// +// Evidence is discounted as it ages. Without that the posteriors only sharpen, +// and an arm that was best over the first ten thousand requests keeps winning +// long after the workload has moved on. Discounting is what makes the bandit +// able to change its mind, which is the entire point on a shifting workload. +// +// The as-cache module deliberately ships no Bandit implementation, so this +// exists here to make the benchmarks runnable. It is small enough to copy. +type ThompsonBandit struct { + mu sync.Mutex + // hits and misses hold the discounted evidence per arm. + hits map[ascache.PolicyType]float64 + misses map[ascache.PolicyType]float64 + // discount multiplies existing evidence at each update, in (0,1]. A value + // of 1 never forgets. + discount float64 + rng *rand.Rand +} + +// NewThompsonBandit returns a bandit that discounts prior evidence by the +// given factor on each report. A discount of 1 never forgets; 0.7 is a +// reasonable starting point for a workload expected to change. +func NewThompsonBandit(discount float64, seed uint64) *ThompsonBandit { + if discount <= 0 || discount > 1 { + discount = 1 + } + + return &ThompsonBandit{ + hits: map[ascache.PolicyType]float64{}, + misses: map[ascache.PolicyType]float64{}, + discount: discount, + rng: newRNG(seed), + } +} + +// RecordStats folds one policy's epoch result into its posterior. +func (b *ThompsonBandit) RecordStats(stats ascache.ShadowStats) { + b.mu.Lock() + defer b.mu.Unlock() + + b.hits[stats.Policy] = b.hits[stats.Policy]*b.discount + float64(stats.Hits) + b.misses[stats.Policy] = b.misses[stats.Policy]*b.discount + float64(stats.Misses) +} + +// SelectPolicy draws one sample from each arm's posterior and returns the arm +// with the highest draw. +func (b *ThompsonBandit) SelectPolicy() ascache.PolicyType { + b.mu.Lock() + defer b.mu.Unlock() + + best := ascache.Undefined + bestSample := -1.0 + + for policy, hits := range b.hits { + // Beta(1+hits, 1+misses): the +1s are a uniform prior, so an arm with + // no evidence yet is sampled across the whole range rather than being + // pinned at zero and never tried. + sample := betaSample(b.rng, 1+hits, 1+b.misses[policy]) + if sample > bestSample { + best, bestSample = policy, sample + } + } + + return best +} + +// Arms returns the arms the bandit has seen, for reporting. +func (b *ThompsonBandit) Arms() []ascache.PolicyType { + b.mu.Lock() + defer b.mu.Unlock() + + arms := make([]ascache.PolicyType, 0, len(b.hits)) + for policy := range b.hits { + arms = append(arms, policy) + } + + return arms +} + +// betaSample draws from Beta(a, b) as the ratio of two Gamma draws, which is +// the standard construction: if X ~ Gamma(a,1) and Y ~ Gamma(b,1) then +// X/(X+Y) ~ Beta(a,b). +func betaSample(rng *rand.Rand, a, b float64) float64 { + x := gammaSample(rng, a) + y := gammaSample(rng, b) + if x+y == 0 { + return 0 + } + + return x / (x + y) +} + +// gammaSample draws from Gamma(shape, 1) using Marsaglia and Tsang's method, +// with the standard boost for shapes below 1. +func gammaSample(rng *rand.Rand, shape float64) float64 { + if shape < 1 { + // Gamma(a) == Gamma(a+1) * U^(1/a) for a < 1. + return gammaSample(rng, shape+1) * math.Pow(rng.Float64(), 1/shape) + } + + d := shape - 1.0/3.0 + c := 1 / math.Sqrt(9*d) + + for { + x := rng.NormFloat64() + v := 1 + c*x + if v <= 0 { + continue + } + v = v * v * v + + u := rng.Float64() + if u < 1-0.0331*x*x*x*x { + return d * v + } + if math.Log(u) < 0.5*x*x+d*(1-v+math.Log(v)) { + return d * v + } + } +} + +// GreedyBandit always selects the arm with the best hit rate so far. It is a +// useful control: it shows what the adaptive layer achieves without any +// exploration, and it makes switching behaviour deterministic in tests. +type GreedyBandit struct { + mu sync.Mutex + rates map[ascache.PolicyType]float64 + counts map[ascache.PolicyType]int64 +} + +// NewGreedyBandit returns a bandit that always picks the best-measured arm. +func NewGreedyBandit() *GreedyBandit { + return &GreedyBandit{ + rates: map[ascache.PolicyType]float64{}, + counts: map[ascache.PolicyType]int64{}, + } +} + +// RecordStats stores the arm's hit rate for the epoch just measured. +func (b *GreedyBandit) RecordStats(stats ascache.ShadowStats) { + b.mu.Lock() + defer b.mu.Unlock() + + total := stats.Hits + stats.Misses + if total == 0 { + return + } + b.rates[stats.Policy] = float64(stats.Hits) / float64(total) + b.counts[stats.Policy] = total +} + +// SelectPolicy returns the arm with the highest measured hit rate. +func (b *GreedyBandit) SelectPolicy() ascache.PolicyType { + b.mu.Lock() + defer b.mu.Unlock() + + best := ascache.Undefined + bestRate := -1.0 + for policy, rate := range b.rates { + if rate > bestRate { + best, bestRate = policy, rate + } + } + + return best +} diff --git a/bench/evidence_test.go b/bench/evidence_test.go new file mode 100644 index 0000000..1cf7a1d --- /dev/null +++ b/bench/evidence_test.go @@ -0,0 +1,325 @@ +package bench_test + +import ( + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/bench" +) + +// cacheSize is small relative to each workload's keyspace, so the policies are +// actually forced to choose what to keep. A cache large enough to hold the +// working set makes every policy look identical. +const cacheSize = 500 + +// workloads returns the suite every comparison runs over. +func workloads() []bench.Workload { + return []bench.Workload{ + bench.Zipf(200000, 20000, 1.1, 1), + bench.Uniform(200000, 5000, 2), + // Working set just over capacity: the classic LRU pathology. + bench.Loop(200000, cacheSize+50), + bench.Scan(200, 100, 400, 600), + bench.PhaseShift(20, 10000, 20000, cacheSize+50, 3), + } +} + +// runFixed measures every shipped policy on a workload. +func runFixed(t *testing.T, w bench.Workload) []bench.Result { + t.Helper() + + results := make([]bench.Result, 0, len(bench.FixedPolicies())) + for _, builder := range bench.FixedPolicies() { + policy, err := builder.Build(cacheSize) + require.NoError(t, err, "build %s", builder.Name) + results = append(results, bench.Replay(builder.Name, policy, w)) + } + + return results +} + +// TestFixedPolicyEvidence reports what each policy achieves on each workload. +// It asserts the properties these workloads exist to demonstrate; the printed +// tables are the artifact. +func TestFixedPolicyEvidence(t *testing.T) { + if testing.Short() { + // These replay millions of requests through timing-driven epochs. + // They are evidence, not correctness checks, and under -race the + // epoch pacing changes enough to make them meaningless. + t.Skip("evidence run; use make evidence") + } + + for _, w := range workloads() { + t.Run(w.Name, func(t *testing.T) { + results := runFixed(t, w) + + t.Logf("\n%s (%d requests, cache %d)\n%s\n%s", + w.Name, w.Len(), cacheSize, w.Description, bench.Table(results)) + + byPolicy := map[string]bench.Result{} + for _, r := range results { + byPolicy[r.Policy] = r + } + + switch w.Name { + case "loop": + // Every key is evicted exactly before it is needed again. + assert.Less(t, byPolicy["LRU"].HitRate(), 0.05, + "a cyclic scan just over capacity should defeat LRU almost completely") + assert.Greater(t, byPolicy["Random"].HitRate(), byPolicy["LRU"].HitRate(), + "random eviction should beat LRU on its pathological case") + + case "scan": + assert.Greater(t, byPolicy["W-TinyLFU"].HitRate(), byPolicy["LRU"].HitRate(), + "frequency-based admission should hold the hot set through sweeps that flush LRU") + + case "uniform": + // With no reuse structure, nothing can do much better than + // anything else; bookkeeping earns nothing. + spread := byPolicy["W-TinyLFU"].HitRate() - byPolicy["Random"].HitRate() + assert.Less(t, spread, 0.10, + "on a workload with no structure, a sophisticated policy should not beat random by much") + } + }) + } +} + +// TestAdaptiveVersusFixed is the question the whole library exists to answer: +// does choosing a policy at runtime beat committing to one up front? +// +// The honest comparison is against the best fixed policy per workload, not +// against LRU. Adaptive selection is only worth its overhead if it tracks the +// best arm without knowing in advance which that is. +func TestAdaptiveVersusFixed(t *testing.T) { + if testing.Short() { + // These replay millions of requests through timing-driven epochs. + // They are evidence, not correctness checks, and under -race the + // epoch pacing changes enough to make them meaningless. + t.Skip("evidence run; use make evidence") + } + + var rows []summaryRow + + for _, w := range workloads() { + t.Run(w.Name, func(t *testing.T) { + fixed := runFixed(t, w) + + arms, err := bench.AdaptiveArms(cacheSize) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache(arms, + bench.NewThompsonBandit(0.7, 7), + &ascache.Settings{ + // Short enough that many epochs elapse during a replay, so + // the bandit gets the chance to react within a phase. + EpochDuration: 2 * time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: ascache.MigrationWarm, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + adaptive := bench.Replay("adaptive", cache, w) + + all := append([]bench.Result{}, fixed...) + all = append(all, adaptive) + t.Logf("\n%s vs fixed policies\n%s", w.Name, bench.Table(all)) + + bestFixed, worstFixed := fixed[0], fixed[0] + for _, r := range fixed { + if r.HitRate() > bestFixed.HitRate() { + bestFixed = r + } + if r.HitRate() < worstFixed.HitRate() { + worstFixed = r + } + } + + rows = append(rows, summaryRow{w.Name, adaptive.HitRate(), bestFixed.Policy, + bestFixed.HitRate(), worstFixed.HitRate()}) + + // The claim worth defending is not that adaptive always wins, but + // that it never lands near the bottom: a cache that can pick the + // worst arm is worse than any fixed choice. + assert.Greater(t, adaptive.HitRate(), worstFixed.HitRate(), + "adaptive selection must beat the worst fixed policy on %s", w.Name) + }) + } + + t.Log("\n" + summarise(rows)) +} + +// summaryRow is one workload's line in the headline comparison. +type summaryRow struct { + workload string + adaptive float64 + best string + bestRate float64 + worstRate float64 +} + +func summarise(rows []summaryRow) string { + out := "| Workload | Adaptive | Best fixed | Worst fixed | Adaptive vs best |\n| --- | --- | --- | --- | --- |\n" + for _, r := range rows { + out += fmt.Sprintf("| %s | %.2f%% | %s %.2f%% | %.2f%% | %+.2f pts |\n", + r.workload, r.adaptive*100, r.best, r.bestRate*100, r.worstRate*100, + (r.adaptive-r.bestRate)*100) + } + + return out +} + +// TestSamplingPreservesPolicyRanking settles the debt Milestone 2 left behind. +// +// Sampled shadows only help if a 5% miniature ranks policies the way full-size +// shadows would. That rests on the miniature-simulation result, which is +// established for stack algorithms under stationary workloads and shakier for +// policies whose bookkeeping is sized in absolute terms - 2Q's ghost queues, +// ARC's list balance, W-TinyLFU's admission sketch. If the ranking inverts, +// sampling silently makes the bandit choose wrong while everything still looks +// healthy, so this measures it rather than trusting the theory. +func TestSamplingPreservesPolicyRanking(t *testing.T) { + if testing.Short() { + // These replay millions of requests through timing-driven epochs. + // They are evidence, not correctness checks, and under -race the + // epoch pacing changes enough to make them meaningless. + t.Skip("evidence run; use make evidence") + } + + // Large enough that a 5% miniature clears the MinShadowCapacity floor and + // is still a real cache, small enough that the arms genuinely differ: if + // every arm holds the whole working set they all score the same and there + // is no ranking to preserve. + const size = 8000 + + for _, w := range []bench.Workload{ + bench.Zipf(300000, 200000, 1.1, 11), + bench.Scan(30, 4000, 16000, 40000), + } { + t.Run(w.Name, func(t *testing.T) { + full := ratesUnderSampling(t, w, size, 0) + sampled := ratesUnderSampling(t, w, size, 0.05) + + bestFull := argmax(full) + bestSampled := argmax(sampled) + + t.Logf("\n%s\nfull-size: %s\nsampled: %s\nfull picks %s, sampled picks %s", + w.Name, formatRates(full), formatRates(sampled), bestFull, bestSampled) + + // The argmax itself is not the property worth asserting: arms that + // are within noise of each other reorder run to run even with no + // sampling at all, because epoch boundaries fall differently. + // + // What matters is regret - whether the arm sampling picks is + // actually worse than the one full-size measurement would have + // picked, judged by their true full-size rates. + regret := full[bestFull] - full[bestSampled] + + assert.Less(t, regret, 0.03, + "sampling picked %s (true rate %.2f%%) over %s (%.2f%%): %.2f points of regret on %s", + bestSampled, full[bestSampled]*100, bestFull, full[bestFull]*100, regret*100, w.Name) + }) + } +} + +// argmax returns the name with the highest rate. +func argmax(rates map[string]float64) string { + best, bestRate := "", -1.0 + for name, rate := range rates { + if rate > bestRate { + best, bestRate = name, rate + } + } + + return best +} + +func formatRates(rates map[string]float64) string { + names := make([]string, 0, len(rates)) + for name := range rates { + names = append(names, name) + } + sort.SliceStable(names, func(i, j int) bool { return rates[names[i]] > rates[names[j]] }) + + parts := make([]string, 0, len(names)) + for _, name := range names { + parts = append(parts, fmt.Sprintf("%s=%.2f%%", name, rates[name]*100)) + } + + return strings.Join(parts, " ") +} + +// ratesUnderSampling measures every arm inside one AdaptiveCache at the given +// sample rate and returns each arm's measured hit rate. +func ratesUnderSampling(t *testing.T, w bench.Workload, size int, rate float64) map[string]float64 { + t.Helper() + + arms, err := bench.AdaptiveArms(size) + require.NoError(t, err) + + recorder := &rankingBandit{} + + cache, err := ascache.NewAdaptiveCache(arms, recorder, &ascache.Settings{ + EpochDuration: 5 * time.Millisecond, + EvictPartialCapacityFilling: true, + ShadowSampleRate: rate, + MinShadowCapacity: 256, + // Hold the active policy still: this measures the arms, and a switch + // would change which arm is being measured in which role. + SwitchCooldownEpochs: 1 << 30, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + bench.Replay("ranking", cache, w) + + return recorder.rates() +} + +// rankingBandit accumulates each arm's reported hits and misses and never +// switches, so a run measures every arm in a fixed role. +type rankingBandit struct { + mu sync.Mutex + hits map[ascache.PolicyType]int64 + misses map[ascache.PolicyType]int64 +} + +func (b *rankingBandit) RecordStats(stats ascache.ShadowStats) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.hits == nil { + b.hits, b.misses = map[ascache.PolicyType]int64{}, map[ascache.PolicyType]int64{} + } + b.hits[stats.Policy] += stats.Hits + b.misses[stats.Policy] += stats.Misses +} + +func (b *rankingBandit) SelectPolicy() ascache.PolicyType { + // Never switch: whichever arm is active stays active. + return ascache.Undefined +} + +func (b *rankingBandit) rates() map[string]float64 { + b.mu.Lock() + defer b.mu.Unlock() + + out := make(map[string]float64, len(b.hits)) + for policy, hits := range b.hits { + total := hits + b.misses[policy] + if total == 0 { + continue + } + out[policy.String()] = float64(hits) / float64(total) + } + + return out +} diff --git a/bench/go.mod b/bench/go.mod new file mode 100644 index 0000000..bd0a78d --- /dev/null +++ b/bench/go.mod @@ -0,0 +1,31 @@ +module github.com/sshaplygin/as-cache/bench + +go 1.25.2 + +require ( + github.com/sshaplygin/as-cache v0.0.0 + github.com/sshaplygin/as-cache/policies v0.0.0 + github.com/sshaplygin/as-cache/policies/arc v0.0.0 + github.com/sshaplygin/as-cache/policies/tinylfu v0.0.0 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/hashicorp/golang-lru/arc/v2 v2.0.6 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.6 // indirect + github.com/maypok86/otter/v2 v2.3.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sshaplygin/as-cache/lfu v0.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/sshaplygin/as-cache => .. + +replace github.com/sshaplygin/as-cache/lfu => ../lfu + +replace github.com/sshaplygin/as-cache/policies => ../policies + +replace github.com/sshaplygin/as-cache/policies/arc => ../policies/arc + +replace github.com/sshaplygin/as-cache/policies/tinylfu => ../policies/tinylfu diff --git a/bench/go.sum b/bench/go.sum new file mode 100644 index 0000000..20cb381 --- /dev/null +++ b/bench/go.sum @@ -0,0 +1,16 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/golang-lru/arc/v2 v2.0.6 h1:4NU7uP5vSoK6TbaMj3NtY478TTAWLso/vL1gpNrInHg= +github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w= +github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/bench/harness.go b/bench/harness.go new file mode 100644 index 0000000..494cde6 --- /dev/null +++ b/bench/harness.go @@ -0,0 +1,146 @@ +package bench + +import ( + "fmt" + "sort" + "strings" + "time" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/policies" + "github.com/sshaplygin/as-cache/policies/arc" + "github.com/sshaplygin/as-cache/policies/tinylfu" +) + +// Result is one policy's performance on one workload. +type Result struct { + Policy string + Hits int64 + Misses int64 + Duration time.Duration +} + +// HitRate returns the fraction of requests served from cache. +func (r Result) HitRate() float64 { + total := r.Hits + r.Misses + if total == 0 { + return 0 + } + + return float64(r.Hits) / float64(total) +} + +// NsPerOp returns the average time per request. +func (r Result) NsPerOp() float64 { + total := r.Hits + r.Misses + if total == 0 { + return 0 + } + + return float64(r.Duration.Nanoseconds()) / float64(total) +} + +// Cache is the subset of cache behaviour the harness replays against, so a +// bare policy and a full AdaptiveCache can be measured the same way. +type Cache interface { + Get(key string) (int, bool) + Add(key string, value int) bool +} + +// Replay runs a workload against a cache, filling on every miss exactly as a +// read-through cache would, and reports what it served. +// +// Hits and misses are counted here rather than read from the cache's own +// statistics so that every subject is measured identically, including ones +// whose internal accounting is sampled. +func Replay(name string, c Cache, w Workload) Result { + result := Result{Policy: name} + + start := time.Now() + for i, key := range w.Keys { + if _, ok := c.Get(key); ok { + result.Hits++ + + continue + } + result.Misses++ + c.Add(key, i) + } + result.Duration = time.Since(start) + + return result +} + +// PolicyBuilder constructs a policy of the given capacity. +type PolicyBuilder struct { + Name string + Build func(size int) (ascache.Policy[string, int], error) +} + +// FixedPolicies returns every policy this repository ships, for measuring what +// each achieves on its own. +func FixedPolicies() []PolicyBuilder { + return []PolicyBuilder{ + {"LRU", policies.NewLRU[string, int]}, + {"LFU", policies.NewLFU[string, int]}, + {"2Q", policies.NewTwoQueue[string, int]}, + {"Random", func(size int) (ascache.Policy[string, int], error) { + return policies.NewRandomPolicy[string, int](size), nil + }}, + {"TTL", func(size int) (ascache.Policy[string, int], error) { + // A TTL longer than any run, so this measures its LRU behaviour + // rather than expiry: the workloads carry no notion of staleness. + return policies.NewTTL[string, int](size, time.Hour), nil + }}, + {"ARC", arc.NewPolicy[string, int]}, + {"W-TinyLFU", tinylfu.NewPolicy[string, int]}, + } +} + +// AdaptiveArms builds a fresh set of arms for an AdaptiveCache. Every arm +// needs its own instance per run, since policies carry state. +func AdaptiveArms(size int) ([]ascache.Policy[string, int], error) { + arms := make([]ascache.Policy[string, int], 0, len(FixedPolicies())) + for _, builder := range FixedPolicies() { + policy, err := builder.Build(size) + if err != nil { + return nil, fmt.Errorf("build %s arm: %w", builder.Name, err) + } + arms = append(arms, policy) + } + + return arms, nil +} + +// Table renders results as a markdown table, best hit rate first. +func Table(results []Result) string { + sorted := make([]Result, len(results)) + copy(sorted, results) + sort.SliceStable(sorted, func(i, j int) bool { + return sorted[i].HitRate() > sorted[j].HitRate() + }) + + var b strings.Builder + b.WriteString("| Policy | Hit rate | ns/op |\n| --- | --- | --- |\n") + for _, r := range sorted { + fmt.Fprintf(&b, "| %s | %.2f%% | %.0f |\n", r.Policy, r.HitRate()*100, r.NsPerOp()) + } + + return b.String() +} + +// Ranking returns policy names ordered best hit rate first. +func Ranking(results []Result) []string { + sorted := make([]Result, len(results)) + copy(sorted, results) + sort.SliceStable(sorted, func(i, j int) bool { + return sorted[i].HitRate() > sorted[j].HitRate() + }) + + names := make([]string, len(sorted)) + for i, r := range sorted { + names[i] = r.Policy + } + + return names +} diff --git a/bench/memory_test.go b/bench/memory_test.go new file mode 100644 index 0000000..adea919 --- /dev/null +++ b/bench/memory_test.go @@ -0,0 +1,220 @@ +package bench_test + +import ( + "runtime" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/bench" + "github.com/sshaplygin/as-cache/policies" + "github.com/sshaplygin/as-cache/policies/arc" + "github.com/sshaplygin/as-cache/policies/tinylfu" +) + +// valueBytes is the payload size per entry. Real caches hold objects, not +// ints, and the whole point of dropping values from demoted policies is only +// visible when a value costs more than a pointer. +const valueBytes = 256 + +// retainedBytes measures the heap still held after build runs, by settling the +// heap, building, settling again, and keeping the result alive across the +// second measurement so it cannot be collected early. +func retainedBytes(build func() any) uint64 { + var before, after runtime.MemStats + + runtime.GC() + runtime.GC() + runtime.ReadMemStats(&before) + + held := build() + + runtime.GC() + runtime.GC() + runtime.ReadMemStats(&after) + + runtime.KeepAlive(held) + + if after.HeapAlloc < before.HeapAlloc { + return 0 + } + + return after.HeapAlloc - before.HeapAlloc +} + +// fillKeys returns the key set used by every memory configuration, so they are +// all measured holding exactly the same data. +func fillKeys(n int) []string { + keys := make([]string, n) + for i := range keys { + keys[i] = "memory-key-" + strconv.Itoa(i) + } + + return keys +} + +// TestMemoryMultiplier measures what the adaptive layer costs in memory +// relative to a single plain LRU holding the same entries. +// +// This is the number the README's disclaimer used to assert without measuring: +// running N policies in parallel was said to multiply memory by N. Shadow +// policies hold no real values, and with sampling they hold only a fraction of +// the keys, so the true multiplier should be far below N. +func TestMemoryMultiplier(t *testing.T) { + if testing.Short() { + t.Skip("evidence run; use make evidence") + } + + const entries = 50000 + + keys := fillKeys(entries) + payload := func() []byte { return make([]byte, valueBytes) } + + baseline := retainedBytes(func() any { + cache, err := policies.NewLRU[string, []byte](entries) + require.NoError(t, err) + for _, key := range keys { + cache.Add(key, payload()) + } + + return cache + }) + + adaptive := func(rate float64) uint64 { + return retainedBytes(func() any { + arms := buildArms(t, entries) + cache, err := ascache.NewAdaptiveCache(arms, NewNoSwitchBandit(), &ascache.Settings{ + EpochDuration: time.Hour, + EvictPartialCapacityFilling: true, + ShadowSampleRate: rate, + MinShadowCapacity: 256, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + for _, key := range keys { + cache.Add(key, payload()) + } + + return cache + }) + } + + full := adaptive(0) + sampled := adaptive(0.05) + + mib := func(b uint64) float64 { return float64(b) / (1 << 20) } + + t.Logf("\nmemory holding %d entries of %d-byte values, %d policies\n"+ + " single LRU %7.1f MiB (1.00x)\n"+ + " adaptive, no sampling %7.1f MiB (%.2fx)\n"+ + " adaptive, sample 0.05 %7.1f MiB (%.2fx)", + entries, valueBytes, len(bench.FixedPolicies()), + mib(baseline), + mib(full), float64(full)/float64(baseline), + mib(sampled), float64(sampled)/float64(baseline)) + + require.Positive(t, baseline, "baseline measurement failed") + + assert.Less(t, sampled, full, + "sampling should reduce what the shadows retain") + + // Shadows hold keys and bookkeeping but never real values, so even with no + // sampling the multiplier must be far below the number of policies. + assert.Less(t, float64(full)/float64(baseline), 3.0, + "shadow policies hold no values, so six policies must not cost six times one") +} + +// TestAllocationsPerOperation reports allocations on the hot path, which is +// the other half of the overhead question: bytes retained is what the cache +// costs at rest, allocations per op is what it costs to run. +func TestAllocationsPerOperation(t *testing.T) { + if testing.Short() { + t.Skip("evidence run; use make evidence") + } + + const size = 10000 + + keys := fillKeys(size) + + measure := func(name string, c interface { + Get(string) ([]byte, bool) + Add(string, []byte) bool + }, + ) { + for _, key := range keys { + c.Add(key, make([]byte, valueBytes)) + } + + result := testing.Benchmark(func(b *testing.B) { + b.ReportAllocs() + for i := 0; b.Loop(); i++ { + c.Get(keys[i%size]) + } + }) + + t.Logf(" %-24s %8.1f ns/op %6d B/op %4d allocs/op", + name, float64(result.NsPerOp()), + result.AllocedBytesPerOp(), result.AllocsPerOp()) + } + + t.Log("\nGet on a warm cache:") + + lru, err := policies.NewLRU[string, []byte](size) + require.NoError(t, err) + measure("single LRU", lru) + + for _, rate := range []float64{0, 0.05} { + arms := buildArms(t, size) + cache, err := ascache.NewAdaptiveCache(arms, NewNoSwitchBandit(), &ascache.Settings{ + EpochDuration: time.Hour, + EvictPartialCapacityFilling: true, + ShadowSampleRate: rate, + MinShadowCapacity: 256, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + name := "adaptive, no sampling" + if rate > 0 { + name = "adaptive, sample 0.05" + } + measure(name, cache) + } +} + +// buildArms builds one arm per shipped policy over []byte values. +func buildArms(t *testing.T, size int) []ascache.Policy[string, []byte] { + t.Helper() + + lru, err := policies.NewLRU[string, []byte](size) + require.NoError(t, err) + twoQ, err := policies.NewTwoQueue[string, []byte](size) + require.NoError(t, err) + arcPolicy, err := arc.NewPolicy[string, []byte](size) + require.NoError(t, err) + tiny, err := tinylfu.NewPolicy[string, []byte](size) + require.NoError(t, err) + + return []ascache.Policy[string, []byte]{ + lru, + twoQ, + arcPolicy, + tiny, + policies.NewRandomPolicy[string, []byte](size), + policies.NewTTL[string, []byte](size, time.Hour), + } +} + +// NewNoSwitchBandit returns a bandit that never switches, so a memory +// measurement is not perturbed by a migration mid-fill. +func NewNoSwitchBandit() ascache.Bandit { return noSwitchBandit{} } + +type noSwitchBandit struct{} + +func (noSwitchBandit) RecordStats(_ ascache.ShadowStats) {} +func (noSwitchBandit) SelectPolicy() ascache.PolicyType { return ascache.Undefined } diff --git a/bench/timeline_test.go b/bench/timeline_test.go new file mode 100644 index 0000000..0a835f3 --- /dev/null +++ b/bench/timeline_test.go @@ -0,0 +1,182 @@ +package bench_test + +import ( + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/bench" +) + +// timelineCache wraps an AdaptiveCache and records which policy was active as +// the workload is replayed, so switching behaviour can be plotted against the +// phase it was reacting to. +type timelineCache struct { + inner *ascache.AdaptiveCache[string, int] + + mu sync.Mutex + samples []ascache.PolicyType + interval int + seen int +} + +func (c *timelineCache) sample() { + c.mu.Lock() + c.seen++ + due := c.seen%c.interval == 0 + c.mu.Unlock() + + if !due { + return + } + + active := c.inner.ActivePolicy() + + c.mu.Lock() + c.samples = append(c.samples, active) + c.mu.Unlock() +} + +func (c *timelineCache) Get(key string) (int, bool) { + c.sample() + + return c.inner.Get(key) +} + +func (c *timelineCache) Add(key string, value int) bool { + return c.inner.Add(key, value) +} + +// TestActivePolicyTimeline replays a phase-shifting workload and plots which +// policy was active through it. +// +// This is the artifact that shows whether switching does anything real: on a +// workload that alternates between regimes, a cache that adapts should be seen +// changing arms, and changing them at the phase boundaries rather than at +// random. +func TestActivePolicyTimeline(t *testing.T) { + if testing.Short() { + // These replay millions of requests through timing-driven epochs. + // They are evidence, not correctness checks, and under -race the + // epoch pacing changes enough to make them meaningless. + t.Skip("evidence run; use make evidence") + } + + const ( + size = 500 + phases = 12 + perPhase = 20000 + ) + + w := bench.PhaseShift(phases, perPhase, 20000, size+50, 5) + + arms, err := bench.AdaptiveArms(size) + require.NoError(t, err) + + inner, err := ascache.NewAdaptiveCache(arms, + bench.NewThompsonBandit(0.6, 9), + &ascache.Settings{ + EpochDuration: 2 * time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: ascache.MigrationWarm, + ShadowSampleRate: 0.05, + MinShadowCapacity: 64, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = inner.Close() }) + + // One sample per 1/40th of a phase, so a phase is legible in the plot. + cache := &timelineCache{inner: inner, interval: perPhase / 40} + + result := bench.Replay("adaptive", cache, w) + + cache.mu.Lock() + samples := append([]ascache.PolicyType(nil), cache.samples...) + cache.mu.Unlock() + + require.NotEmpty(t, samples, "expected timeline samples") + + t.Logf("\nphase-shift timeline (%d requests, cache %d, %d phases)\n%s\nhit rate %.2f%%", + w.Len(), size, phases, plotTimeline(samples, phases), result.HitRate()*100) + + distinct := map[ascache.PolicyType]int{} + for _, p := range samples { + distinct[p]++ + } + + assert.Greater(t, len(distinct), 1, + "the cache should change arms on a workload that changes regime, saw only %v", distinct) +} + +// plotTimeline renders the active policy over time as one row per policy, with +// phase boundaries marked, so the reader can see whether switches line up with +// the workload changing regime. +func plotTimeline(samples []ascache.PolicyType, phases int) string { + present := map[ascache.PolicyType]bool{} + for _, p := range samples { + present[p] = true + } + + names := make([]ascache.PolicyType, 0, len(present)) + for p := range present { + names = append(names, p) + } + sort.Slice(names, func(i, j int) bool { return names[i] < names[j] }) + + width := len(samples) + perPhase := width / phases + + var b strings.Builder + + // Phase ruler: alternating regime labels above the plot. + b.WriteString(fmt.Sprintf("%-10s ", "phase")) + for i := 0; i < width; i++ { + if perPhase > 0 && i%perPhase == 0 { + if (i/perPhase)%2 == 0 { + b.WriteString("Z") + + continue + } + b.WriteString("L") + + continue + } + b.WriteString("-") + } + b.WriteString(" (Z = zipf phase, L = loop phase)\n") + + for _, policy := range names { + fmt.Fprintf(&b, "%-10s ", policy.String()) + for _, s := range samples { + if s == policy { + b.WriteString("#") + + continue + } + b.WriteString(" ") + } + b.WriteString("\n") + } + + // Share of time each policy held the active slot. + counts := map[ascache.PolicyType]int{} + for _, s := range samples { + counts[s]++ + } + b.WriteString("\nshare of time active: ") + parts := make([]string, 0, len(names)) + for _, policy := range names { + parts = append(parts, fmt.Sprintf("%s %.0f%%", + policy, float64(counts[policy])/float64(len(samples))*100)) + } + b.WriteString(strings.Join(parts, ", ")) + + return b.String() +} diff --git a/bench/trace.go b/bench/trace.go new file mode 100644 index 0000000..33b8091 --- /dev/null +++ b/bench/trace.go @@ -0,0 +1,297 @@ +package bench + +import ( + "bufio" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" +) + +// TraceDirEnv names the environment variable holding a directory of trace +// files. Traces are not committed to this repository: they are large, and +// their licences generally do not permit redistribution. +const TraceDirEnv = "AS_CACHE_TRACES" + +// ErrNoTraceDir is returned when no trace directory has been configured. +var ErrNoTraceDir = errors.New("no trace directory configured") + +// TraceFormat describes how to pull a cache key out of each line of a trace. +// +// Trace formats differ in trivial ways - a block number per line, a CSV with a +// key column, a space-separated offset and length - so rather than a parser +// per source there is one parser and a description of the layout. +type TraceFormat struct { + // Delimiter splits a line into fields. Empty means split on whitespace. + Delimiter string + // KeyColumn is the zero-based field holding the key. Lines with fewer + // fields are skipped. + KeyColumn int + // Comment, when not empty, marks lines to ignore. + Comment string + // SkipHeader drops the first non-comment line. + SkipHeader bool +} + +// LineFormat reads one key per line, the simplest and most common layout. +var LineFormat = TraceFormat{KeyColumn: 0} + +// CSVFormat reads a comma-separated trace, taking the key from the given +// column and skipping a header row. +func CSVFormat(keyColumn int) TraceFormat { + return TraceFormat{Delimiter: ",", KeyColumn: keyColumn, SkipHeader: true} +} + +// field extracts the key column from a line, reporting whether it found one. +func (f TraceFormat) field(line string) (string, bool) { + if f.Comment != "" && strings.HasPrefix(line, f.Comment) { + return "", false + } + + var fields []string + if f.Delimiter == "" { + fields = strings.Fields(line) + } else { + fields = strings.Split(line, f.Delimiter) + } + + if f.KeyColumn >= len(fields) { + return "", false + } + + key := strings.TrimSpace(fields[f.KeyColumn]) + if key == "" { + return "", false + } + + return key, true +} + +// LoadTrace reads up to limit requests from a trace file, transparently +// decompressing a .gz file. A limit of zero or less reads the whole file. +// +// Keys are interned: a real trace repeats a small key set across millions of +// requests, so sharing one string per distinct key is the difference between a +// workload that fits in memory and one that does not. +func LoadTrace(path string, format TraceFormat, limit int) (Workload, error) { + file, err := os.Open(path) + if err != nil { + return Workload{}, fmt.Errorf("open trace: %w", err) + } + defer file.Close() + + var reader io.Reader = file + if strings.HasSuffix(path, ".gz") { + gz, gzErr := gzip.NewReader(file) + if gzErr != nil { + return Workload{}, fmt.Errorf("decompress trace: %w", gzErr) + } + defer gz.Close() + reader = gz + } + + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + intern := map[string]string{} + keys := make([]string, 0, 1024) + skippedHeader := false + + for scanner.Scan() { + line := scanner.Text() + + key, ok := format.field(line) + if !ok { + continue + } + + if format.SkipHeader && !skippedHeader { + skippedHeader = true + + continue + } + + if shared, seen := intern[key]; seen { + key = shared + } else { + intern[key] = key + } + + keys = append(keys, key) + if limit > 0 && len(keys) >= limit { + break + } + } + + if err := scanner.Err(); err != nil { + return Workload{}, fmt.Errorf("read trace: %w", err) + } + + if len(keys) == 0 { + return Workload{}, fmt.Errorf("trace %s yielded no keys: wrong format?", filepath.Base(path)) + } + + return Workload{ + Name: strings.TrimSuffix(strings.TrimSuffix(filepath.Base(path), ".gz"), ".txt"), + Keys: keys, + Description: fmt.Sprintf("real trace: %d requests over %d distinct keys", + len(keys), len(intern)), + }, nil +} + +// Known trace layouts, as verified against the published files. Each is named +// for the dataset it reads; see docs/TRACES.md for where to obtain them. +var ( + // TwitterFormat reads the Twitter Twemcache traces: seven comma-separated + // columns, no header, key in column 1. + TwitterFormat = TraceFormat{Delimiter: ",", KeyColumn: 1} + + // LIRSFormat reads the LIRS research traces: one page number per line. + // Lines consisting of a single asterisk are checkpoint markers rather than + // keys, and must not be replayed as accesses. + LIRSFormat = TraceFormat{KeyColumn: 0, Comment: "*"} +) + +// LoadARCTrace reads a trace in the ARC paper's layout: whitespace-separated +// records of "startBlock blockCount ...", where each record stands for +// blockCount consecutive block accesses. +// +// The expansion is the whole point and is easy to miss: a record is not one +// access. Reading the first column as a key would produce a workload with a +// different length and different locality from the one every published result +// refers to, so the numbers would not be comparable to the literature. +func LoadARCTrace(path string, limit int) (Workload, error) { + file, err := os.Open(path) + if err != nil { + return Workload{}, fmt.Errorf("open trace: %w", err) + } + defer file.Close() + + var reader io.Reader = file + if strings.HasSuffix(path, ".gz") { + gz, gzErr := gzip.NewReader(file) + if gzErr != nil { + return Workload{}, fmt.Errorf("decompress trace: %w", gzErr) + } + defer gz.Close() + reader = gz + } + + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + intern := map[uint64]string{} + keys := make([]string, 0, 1024) + records := 0 + + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 2 { + continue + } + + start, startErr := strconv.ParseUint(fields[0], 10, 64) + count, countErr := strconv.ParseUint(fields[1], 10, 64) + if startErr != nil || countErr != nil { + continue + } + records++ + + for i := uint64(0); i < count; i++ { + block := start + i + + shared, seen := intern[block] + if !seen { + shared = strconv.FormatUint(block, 10) + intern[block] = shared + } + + keys = append(keys, shared) + if limit > 0 && len(keys) >= limit { + break + } + } + + if limit > 0 && len(keys) >= limit { + break + } + } + + if err := scanner.Err(); err != nil { + return Workload{}, fmt.Errorf("read trace: %w", err) + } + + if len(keys) == 0 { + return Workload{}, fmt.Errorf("trace %s yielded no keys: wrong format?", filepath.Base(path)) + } + + return Workload{ + Name: strings.TrimSuffix(filepath.Base(path), ".gz"), + Keys: keys, + Description: fmt.Sprintf("ARC trace: %d records expanded to %d accesses over %d distinct blocks", + records, len(keys), len(intern)), + }, nil +} + +// TraceDir returns the configured trace directory, or ErrNoTraceDir when the +// environment variable is unset. Callers are expected to skip rather than fail +// when traces are absent, so a checkout without them still builds and tests. +func TraceDir() (string, error) { + dir := os.Getenv(TraceDirEnv) + if dir == "" { + return "", ErrNoTraceDir + } + + info, err := os.Stat(dir) + if err != nil { + return "", fmt.Errorf("stat %s: %w", TraceDirEnv, err) + } + if !info.IsDir() { + return "", fmt.Errorf("%s is not a directory: %s", TraceDirEnv, dir) + } + + return dir, nil +} + +// DiscoverTraces lists the trace files in the configured directory. +func DiscoverTraces() ([]string, error) { + dir, err := TraceDir() + if err != nil { + return nil, err + } + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read trace dir: %w", err) + } + + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".md") { + continue + } + paths = append(paths, filepath.Join(dir, name)) + } + + return paths, nil +} + +// DistinctKeys counts the distinct keys in a workload, which is what decides +// whether a given cache capacity is interesting: a cache larger than the key +// set makes every policy look identical. +func DistinctKeys(w Workload) int { + seen := make(map[string]struct{}, len(w.Keys)/4+1) + for _, key := range w.Keys { + seen[key] = struct{}{} + } + + return len(seen) +} diff --git a/bench/trace_test.go b/bench/trace_test.go new file mode 100644 index 0000000..8b6a45b --- /dev/null +++ b/bench/trace_test.go @@ -0,0 +1,233 @@ +package bench_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/bench" +) + +// traceSpec names a trace file and how to read it. +type traceSpec struct { + file string + load func(path string) (bench.Workload, error) + cache int + source string +} + +// knownTraces are the traces ./scripts/fetch-traces.sh downloads. Each is +// skipped individually when absent, so a partial download still reports on +// what is there. +func knownTraces() []traceSpec { + return []traceSpec{ + { + file: "twitter_cluster052.csv", + load: func(p string) (bench.Workload, error) { return bench.LoadTrace(p, bench.TwitterFormat, 0) }, + cache: 10000, + source: "Twitter Twemcache production KV cache (OSDI '20)", + }, + { + file: "lirs_loop.trace.gz", + load: func(p string) (bench.Workload, error) { return bench.LoadTrace(p, bench.LIRSFormat, 0) }, + cache: 500, + source: "LIRS loop: cyclic scan, adversarial for LRU (SIGMETRICS '02)", + }, + { + file: "lirs_2_pools.trace.gz", + load: func(p string) (bench.Workload, error) { return bench.LoadTrace(p, bench.LIRSFormat, 0) }, + cache: 1000, + source: "LIRS 2_pools: two interleaved pools with different locality", + }, + { + file: "arc_p3.gz", + load: func(p string) (bench.Workload, error) { return bench.LoadARCTrace(p, 2000000) }, + cache: 20000, + source: "ARC paper P3 workstation trace (FAST '03)", + }, + { + file: "arc_oltp.gz", + load: func(p string) (bench.Workload, error) { return bench.LoadARCTrace(p, 2000000) }, + cache: 20000, + source: "ARC paper OLTP database trace (FAST '03)", + }, + } +} + +// loadKnownTraces returns the traces present locally, skipping the test when +// none are configured. +func loadKnownTraces(t *testing.T) []struct { + spec traceSpec + workload bench.Workload +} { + t.Helper() + + dir, err := bench.TraceDir() + if err != nil { + t.Skipf("%s; run ./scripts/fetch-traces.sh and set %s", err, bench.TraceDirEnv) + } + + var found []struct { + spec traceSpec + workload bench.Workload + } + + for _, spec := range knownTraces() { + path := filepath.Join(dir, spec.file) + if _, statErr := os.Stat(path); statErr != nil { + t.Logf("absent, skipping: %s", spec.file) + + continue + } + + w, loadErr := spec.load(path) + require.NoError(t, loadErr, "load %s", spec.file) + found = append(found, struct { + spec traceSpec + workload bench.Workload + }{spec, w}) + } + + if len(found) == 0 { + t.Skipf("no known traces in %s; run ./scripts/fetch-traces.sh", dir) + } + + return found +} + +// TestTraceEvidence is the real-workload counterpart to TestAdaptiveVersusFixed. +// The synthetic result - that adaptive selection never beats the best fixed +// policy - rests on workloads chosen by the author of the library, which is +// exactly the kind of evidence that should not be trusted on its own. +func TestTraceEvidence(t *testing.T) { + if testing.Short() { + t.Skip("evidence run; use make evidence") + } + + for _, found := range loadKnownTraces(t) { + spec, w := found.spec, found.workload + + t.Run(w.Name, func(t *testing.T) { + distinct := bench.DistinctKeys(w) + t.Logf("\n%s\n%s\n%s\ncache %d entries, %.1f%% of the %d distinct keys", + w.Name, spec.source, w.Description, spec.cache, + float64(spec.cache)/float64(distinct)*100, distinct) + + results := make([]bench.Result, 0, len(bench.FixedPolicies())+1) + for _, builder := range bench.FixedPolicies() { + policy, err := builder.Build(spec.cache) + require.NoError(t, err) + results = append(results, bench.Replay(builder.Name, policy, w)) + } + + arms, err := bench.AdaptiveArms(spec.cache) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache(arms, + bench.NewThompsonBandit(0.7, 13), + &ascache.Settings{ + EpochDuration: 2 * time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: ascache.MigrationWarm, + ShadowSampleRate: 0.05, + MinShadowCapacity: 64, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + adaptive := bench.Replay("adaptive", cache, w) + results = append(results, adaptive) + + t.Logf("\n%s", bench.Table(results)) + + best, worst := results[0], results[0] + for _, r := range results { + if r.Policy == "adaptive" { + continue + } + if r.HitRate() > best.HitRate() { + best = r + } + if r.HitRate() < worst.HitRate() { + worst = r + } + } + + t.Logf("adaptive %.2f%% | best fixed %s %.2f%% (%+.2f pts) | worst fixed %s %.2f%%", + adaptive.HitRate()*100, best.Policy, best.HitRate()*100, + (adaptive.HitRate()-best.HitRate())*100, worst.Policy, worst.HitRate()*100) + + // The same claim the synthetic suite makes: the value on offer is a + // bound on the downside of choosing wrong, not beating the best. + assert.Greater(t, adaptive.HitRate(), worst.HitRate(), + "adaptive selection must beat the worst fixed policy on %s", w.Name) + }) + } +} + +// TestTraceLoaders checks the parsers against the published ground truth for +// each trace, so a format misread cannot quietly produce a plausible-looking +// key stream and wrong evidence with it. +func TestTraceLoaders(t *testing.T) { + if testing.Short() { + t.Skip("evidence run; use make evidence") + } + + dir, err := bench.TraceDir() + if err != nil { + t.Skipf("%s; run ./scripts/fetch-traces.sh", err) + } + + // Counts measured from the published files. + expectations := map[string]struct { + requests int + distinct int + load func(path string) (bench.Workload, error) + }{ + "lirs_loop.trace.gz": { + requests: 505500, distinct: 1011, + load: func(p string) (bench.Workload, error) { return bench.LoadTrace(p, bench.LIRSFormat, 0) }, + }, + "lirs_2_pools.trace.gz": { + requests: 100000, distinct: 9939, + load: func(p string) (bench.Workload, error) { return bench.LoadTrace(p, bench.LIRSFormat, 0) }, + }, + } + + for file, want := range expectations { + path := filepath.Join(dir, file) + if _, statErr := os.Stat(path); statErr != nil { + continue + } + + t.Run(file, func(t *testing.T) { + w, loadErr := want.load(path) + require.NoError(t, loadErr) + + assert.Equal(t, want.requests, w.Len(), + "request count must match the published file; a parser that drops or invents records invalidates every number derived from it") + assert.Equal(t, want.distinct, bench.DistinctKeys(w), "distinct key count") + }) + } + + // The ARC layout expands each record into blockCount accesses, so the + // access count must exceed the line count. Getting this wrong yields a + // workload incomparable with the published literature. + arcPath := filepath.Join(dir, "arc_p3.gz") + if _, statErr := os.Stat(arcPath); statErr == nil { + t.Run("arc_p3 expansion", func(t *testing.T) { + w, loadErr := bench.LoadARCTrace(arcPath, 0) + require.NoError(t, loadErr) + + assert.True(t, strings.Contains(w.Description, "expanded")) + assert.Greater(t, w.Len(), 3000000, + "each ARC record stands for blockCount accesses; without the expansion p3 yields far too few") + }) + } +} diff --git a/bench/tuning_test.go b/bench/tuning_test.go new file mode 100644 index 0000000..9113e96 --- /dev/null +++ b/bench/tuning_test.go @@ -0,0 +1,82 @@ +package bench_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/bench" +) + +// TestAdaptiveTuning asks whether the gap between adaptive selection and the +// best fixed policy is a property of the idea or of how it was configured. +// +// The first trace runs used a 2ms epoch with warm migration, which on a 20k +// cache means copying 20,000 entries several hundred times during a replay. +// That is a configuration that spends most of its time migrating, and blaming +// the approach for it would be a measurement error rather than a finding. +func TestAdaptiveTuning(t *testing.T) { + if testing.Short() { + t.Skip("evidence run; use make evidence") + } + + configs := []struct { + name string + epoch time.Duration + strategy ascache.MigrationStrategy + gates bool + }{ + {"2ms epoch, warm migration", 2 * time.Millisecond, ascache.MigrationWarm, false}, + {"2ms epoch, cold migration", 2 * time.Millisecond, ascache.MigrationCold, false}, + {"50ms epoch, warm migration", 50 * time.Millisecond, ascache.MigrationWarm, false}, + {"50ms epoch, warm + stability gates", 50 * time.Millisecond, ascache.MigrationWarm, true}, + } + + for _, found := range loadKnownTraces(t) { + spec, w := found.spec, found.workload + + t.Run(w.Name, func(t *testing.T) { + // The bar: the best any single policy manages on this trace. + best := bench.Result{} + bestName := "" + for _, builder := range bench.FixedPolicies() { + policy, err := builder.Build(spec.cache) + require.NoError(t, err) + r := bench.Replay(builder.Name, policy, w) + if r.HitRate() > best.HitRate() { + best, bestName = r, builder.Name + } + } + + t.Logf("\n%s: best fixed is %s at %.2f%%", w.Name, bestName, best.HitRate()*100) + + for _, cfg := range configs { + arms, err := bench.AdaptiveArms(spec.cache) + require.NoError(t, err) + + settings := &ascache.Settings{ + EpochDuration: cfg.epoch, + EvictPartialCapacityFilling: true, + MigrationStrategy: cfg.strategy, + ShadowSampleRate: 0.05, + MinShadowCapacity: 64, + } + if cfg.gates { + settings.MinHitRateImprovement = 0.02 + settings.SwitchCooldownEpochs = 3 + } + + cache, err := ascache.NewAdaptiveCache(arms, bench.NewThompsonBandit(0.7, 13), settings) + require.NoError(t, err) + + r := bench.Replay("adaptive", cache, w) + _ = cache.Close() + + t.Logf(" %-36s %6.2f%% (%+6.2f pts vs best) %7.0f ns/op", + cfg.name, r.HitRate()*100, (r.HitRate()-best.HitRate())*100, r.NsPerOp()) + } + }) + } +} diff --git a/bench/workload.go b/bench/workload.go new file mode 100644 index 0000000..efcef31 --- /dev/null +++ b/bench/workload.go @@ -0,0 +1,182 @@ +// Package bench generates cache workloads and replays them against policies, +// so claims about which policy wins where can be checked rather than asserted. +// +// Every generator is deterministic given its seed: a run is reproducible, and +// two policies are always compared on the identical request sequence. +package bench + +import ( + "math/rand/v2" + "strconv" +) + +// Workload is a named request sequence to replay against a cache. +type Workload struct { + // Name identifies the workload in reports. + Name string + // Keys is the request sequence, in order. + Keys []string + // Description says what the workload is meant to represent and which + // policies it is expected to favour, so a surprising result is + // recognisable as surprising. + Description string +} + +// Len returns the number of requests. +func (w Workload) Len() int { return len(w.Keys) } + +// key renders a key id. Keys are strings because that is what a real cache +// usually holds, and it keeps hashing costs realistic. +func key(id int) string { + return "k" + strconv.Itoa(id) +} + +// newRNG returns a deterministic source for a given seed. A reproducible +// sequence is the requirement here: two policies must be compared on the +// identical request stream, which a cryptographic source would defeat. +// +//nolint:gosec // deliberate: determinism is the point, see above +func newRNG(seed uint64) *rand.Rand { + return rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) +} + +// newZipf builds a Zipf source over ids in [0, keyspace). It returns ids as +// ints, bounded by keyspace, so no conversion can wrap. +func newZipf(rng *rand.Rand, s float64, keyspace int) func() int { + if keyspace < 2 { + return func() int { return 0 } + } + + // keyspace is a positive int here, so both conversions are exact in either + // direction and the drawn id is clamped into range before it is used. + maxID := uint64(keyspace - 1) //nolint:gosec // keyspace is >= 2 here, so this is exact + zipf := rand.NewZipf(rng, s, 1, maxID) + + return func() int { + drawn := min(zipf.Uint64(), maxID) + + return int(drawn) //nolint:gosec // clamped to maxID on the line above + } +} + +// Zipf returns a workload whose key popularity follows a Zipf distribution: +// a small head of very hot keys and a long tail of cold ones. This is the +// shape most real caches see, and it rewards policies that track frequency. +func Zipf(requests, keyspace int, s float64, seed uint64) Workload { + // v = 1 so the most popular key is id 0; keyspace bounds the support. + draw := newZipf(newRNG(seed), s, keyspace) + + keys := make([]string, requests) + for i := range keys { + keys[i] = key(draw()) + } + + return Workload{ + Name: "zipf", + Keys: keys, + Description: "skewed popularity; favours frequency-aware policies (LFU, W-TinyLFU)", + } +} + +// Uniform returns a workload with no reuse structure: every key is equally +// likely. Nothing can predict it, so elaborate bookkeeping earns nothing and +// random eviction is competitive. It is the control that shows when a policy +// is paying for information the workload does not contain. +func Uniform(requests, keyspace int, seed uint64) Workload { + rng := newRNG(seed) + + keys := make([]string, requests) + for i := range keys { + keys[i] = key(rng.IntN(keyspace)) + } + + return Workload{ + Name: "uniform", + Keys: keys, + Description: "no reuse structure; random eviction is competitive", + } +} + +// Loop returns a cyclic scan over a working set slightly larger than the +// cache. It is the classic LRU pathology: every key is evicted exactly before +// it is needed again, so LRU hits nothing at all, while random eviction keeps +// an arbitrary fraction resident and does far better. +func Loop(requests, workingSet int) Workload { + keys := make([]string, requests) + for i := range keys { + keys[i] = key(i % workingSet) + } + + return Workload{ + Name: "loop", + Keys: keys, + Description: "cyclic scan just over capacity; pathological for LRU, fine for random", + } +} + +// Scan returns a workload that repeatedly reads a small hot set and then +// sweeps a long run of one-off keys. The sweep is what flushes a naive +// recency cache: policies that admit on frequency keep the hot set, policies +// that admit on recency lose it every sweep. +func Scan(rounds, hotSet, hotReads, scanLen int) Workload { + keys := make([]string, 0, rounds*(hotReads+scanLen)) + scanID := 1000000 + + for range rounds { + for i := range hotReads { + keys = append(keys, key(i%hotSet)) + } + for range scanLen { + keys = append(keys, key(scanID)) + scanID++ + } + } + + return Workload{ + Name: "scan", + Keys: keys, + Description: "hot set plus repeated one-off sweeps; favours scan-resistant policies (2Q, ARC, W-TinyLFU)", + } +} + +// PhaseShift alternates between two regimes that reward different policies: +// a Zipf phase, where frequency wins, and a loop phase, where frequency is +// exactly the wrong signal because every key is equally and briefly popular. +// +// This is the workload adaptive selection exists for. A fixed policy must be +// mediocre in one phase to be good in the other; a cache that switches can be +// good in both, if the switching actually works and is fast enough to matter. +func PhaseShift(phases, perPhase, keyspace, workingSet int, seed uint64) Workload { + draw := newZipf(newRNG(seed), 1.1, keyspace) + + keys := make([]string, 0, phases*perPhase) + for p := range phases { + if p%2 == 0 { + for range perPhase { + keys = append(keys, key(draw())) + } + + continue + } + for i := range perPhase { + keys = append(keys, key(i%workingSet)) + } + } + + return Workload{ + Name: "phase-shift", + Keys: keys, + Description: "alternating zipf and loop phases; no fixed policy is good in both", + } +} + +// PhaseBoundaries returns the request indices at which PhaseShift changes +// regime, for plotting which policy was active when. +func PhaseBoundaries(phases, perPhase int) []int { + bounds := make([]int, 0, phases) + for p := 1; p < phases; p++ { + bounds = append(bounds, p*perPhase) + } + + return bounds +} diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000..21dfd74 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,245 @@ +package ascache + +import ( + "strconv" + "sync" + "testing" + "time" +) + +// benchPolicy is a minimal thread-safe Policy used to measure AdaptiveCache's +// own orchestration overhead. It is deliberately simple - a mutex around a map, +// like any real cache - so the numbers reflect the cost the adaptive layer adds +// on top of a policy, not the policy's own algorithm. +type benchPolicy[K comparable, V any] struct { + mu sync.RWMutex + data map[K]V + cap int + policyType PolicyType + hits int64 + misses int64 +} + +func newBenchPolicy[K comparable, V any](policyType PolicyType, capacity int) *benchPolicy[K, V] { + return &benchPolicy[K, V]{ + data: make(map[K]V, capacity), + cap: capacity, + policyType: policyType, + } +} + +func (p *benchPolicy[K, V]) Add(key K, value V) bool { + p.mu.Lock() + defer p.mu.Unlock() + _, existed := p.data[key] + p.data[key] = value + return existed +} + +func (p *benchPolicy[K, V]) Get(key K) (V, bool) { + p.mu.Lock() + defer p.mu.Unlock() + v, ok := p.data[key] + if ok { + p.hits++ + } else { + p.misses++ + } + return v, ok +} + +func (p *benchPolicy[K, V]) Peek(key K) (V, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + v, ok := p.data[key] + return v, ok +} + +func (p *benchPolicy[K, V]) Contains(key K) bool { + p.mu.RLock() + defer p.mu.RUnlock() + _, ok := p.data[key] + return ok +} + +func (p *benchPolicy[K, V]) Remove(key K) bool { + p.mu.Lock() + defer p.mu.Unlock() + _, ok := p.data[key] + delete(p.data, key) + return ok +} + +func (p *benchPolicy[K, V]) Purge() { + p.mu.Lock() + defer p.mu.Unlock() + p.data = make(map[K]V, p.cap) +} + +func (p *benchPolicy[K, V]) Keys() []K { + p.mu.RLock() + defer p.mu.RUnlock() + keys := make([]K, 0, len(p.data)) + for k := range p.data { + keys = append(keys, k) + } + return keys +} + +func (p *benchPolicy[K, V]) Values() []V { + p.mu.RLock() + defer p.mu.RUnlock() + vals := make([]V, 0, len(p.data)) + for _, v := range p.data { + vals = append(vals, v) + } + return vals +} + +func (p *benchPolicy[K, V]) Len() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.data) +} + +func (p *benchPolicy[K, V]) Cap() int { return p.cap } + +func (p *benchPolicy[K, V]) Resize(size int) int { + p.mu.Lock() + defer p.mu.Unlock() + p.cap = size + return 0 +} + +func (p *benchPolicy[K, V]) GetStats() PolicyStats { + p.mu.RLock() + defer p.mu.RUnlock() + return PolicyStats{Hits: p.hits, Misses: p.misses} +} + +func (p *benchPolicy[K, V]) ResetStats() { + p.mu.Lock() + defer p.mu.Unlock() + p.hits, p.misses = 0, 0 +} + +func (p *benchPolicy[K, V]) GetType() PolicyType { return p.policyType } + +const benchKeys = 1000 + +// newBenchCache builds a cache with shadowCount shadow policies alongside the +// active one, pre-populated with benchKeys keys, and an epoch long enough that +// no switch happens mid-benchmark. A sampleRate of 0 leaves sampling off. +func newBenchCache(b *testing.B, shadowCount int, sampleRate float64) (*AdaptiveCache[string, int], []string) { + b.Helper() + + types := []PolicyType{LRU, LFU, PolicyType(3), PolicyType(4)} + policies := make([]Policy[string, int], 0, shadowCount+1) + for i := 0; i <= shadowCount; i++ { + policies = append(policies, newBenchPolicy[string, int](types[i], benchKeys*2)) + } + + ac, err := NewAdaptiveCache(policies, &mockBandit{next: LRU}, &Settings{ + EpochDuration: time.Hour, + EvictPartialCapacityFilling: true, + ShadowSampleRate: sampleRate, + // The floor would otherwise disable sampling at this cache size. + MinShadowCapacity: 8, + }) + if err != nil { + b.Fatalf("NewAdaptiveCache: %v", err) + } + b.Cleanup(func() { _ = ac.Close() }) + + keys := make([]string, benchKeys) + for i := range keys { + keys[i] = "key-" + strconv.Itoa(i) + ac.Add(keys[i], i) + } + + return ac, keys +} + +// benchRates is the set of shadow sample rates each benchmark reports, so the +// overhead reduction from sampling is visible as a single comparison. +var benchRates = []struct { + name string + rate float64 +}{ + {"sample=off", 0}, + {"sample=0.05", 0.05}, +} + +// BenchmarkGet measures the single-goroutine read path. +func BenchmarkGet(b *testing.B) { + for _, r := range benchRates { + for _, shadows := range []int{1, 3} { + b.Run(r.name+"/shadows="+strconv.Itoa(shadows), func(b *testing.B) { + ac, keys := newBenchCache(b, shadows, r.rate) + b.ResetTimer() + for i := 0; b.Loop(); i++ { + ac.Get(keys[i%benchKeys]) + } + }) + } + } +} + +// BenchmarkGetParallel measures read scalability: every goroutine contends on +// the cache's RWMutex on top of each policy's own lock, so this is where the +// per-operation shadow fan-out costs the most. +func BenchmarkGetParallel(b *testing.B) { + for _, r := range benchRates { + for _, shadows := range []int{1, 3} { + b.Run(r.name+"/shadows="+strconv.Itoa(shadows), func(b *testing.B) { + ac, keys := newBenchCache(b, shadows, r.rate) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + ac.Get(keys[i%benchKeys]) + i++ + } + }) + }) + } + } +} + +// BenchmarkMixedParallel measures a read-heavy workload with writes mixed in. +// Writes take the cache's write lock, so this is where readers being blocked +// by an unrelated Add shows up. +func BenchmarkMixedParallel(b *testing.B) { + for _, r := range benchRates { + b.Run(r.name, func(b *testing.B) { + ac, keys := newBenchCache(b, 1, r.rate) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + if i%10 == 0 { + ac.Add(keys[i%benchKeys], i) + } else { + ac.Get(keys[i%benchKeys]) + } + i++ + } + }) + }) + } +} + +// BenchmarkAdd measures the write path, which fans out to every shadow policy. +func BenchmarkAdd(b *testing.B) { + for _, r := range benchRates { + for _, shadows := range []int{1, 3} { + b.Run(r.name+"/shadows="+strconv.Itoa(shadows), func(b *testing.B) { + ac, keys := newBenchCache(b, shadows, r.rate) + b.ResetTimer() + for i := 0; b.Loop(); i++ { + ac.Add(keys[i%benchKeys], i) + } + }) + } + } +} diff --git a/cache.go b/cache.go index 91817ba..966e557 100644 --- a/cache.go +++ b/cache.go @@ -2,55 +2,11 @@ package ascache import ( "context" - "errors" "sync" + "sync/atomic" "time" ) -var ErrEmptyPolicies = errors.New("must provide non zero policies size") - -// Settings configures the behaviour of AdaptiveCache. -type Settings struct { - EpochDuration time.Duration - // EvictPartialCapacityFilling allows policy switching even when the cache - // is not yet full. - EvictPartialCapacityFilling bool - // MigrationStrategy determines how data is moved when the active policy - // changes. Defaults to MigrationCold (zero value). - MigrationStrategy MigrationStrategy -} - -func NewAdaptiveCache[K comparable, V any]( - policies []Policy[K, V], - bandit Bandit, - settings *Settings, -) (*AdaptiveCache[K, V], error) { - if len(policies) == 0 { - return nil, ErrEmptyPolicies - } - - ctx, cancel := context.WithCancel(context.Background()) - - availablePolicies := make(map[PolicyType]Policy[K, V], len(policies)) - for _, policy := range policies { - availablePolicies[policy.GetType()] = policy - } - - ac := &AdaptiveCache[K, V]{ - policies: availablePolicies, - activePolicy: policies[0].GetType(), - bandit: bandit, - epochTicker: time.NewTicker(settings.EpochDuration), - ctx: ctx, - cancel: cancel, - settings: settings, - } - - go ac.runAdaptiveSelect() - - return ac, nil -} - // AdaptiveCache is a cache that automatically selects the best replacement // policy at runtime using a Multi-Armed Bandit algorithm. type AdaptiveCache[K comparable, V any] struct { @@ -60,6 +16,34 @@ type AdaptiveCache[K comparable, V any] struct { activePolicy PolicyType policies map[PolicyType]Policy[K, V] + // sampler decides which keys shadow policies track. It is shared by every + // policy so they all measure the same substream, and is fixed for the + // lifetime of the cache. + sampler *keySampler[K] + + // nominalCap is each policy's capacity as the caller built it, restored + // when the policy takes over active duty. shadowCap is the miniature + // capacity it runs at while shadowing, and minShadowCap is the floor + // applied when recomputing that capacity after a Resize. + nominalCap map[PolicyType]int + shadowCap map[PolicyType]int + minShadowCap int + + // activeSampledHits and activeSampledMisses count the active policy's + // results for sampled keys only. The bandit is fed these rather than the + // policy's full counters so that every arm is judged on the same sampled + // substream, with the same weight of evidence. They are mutated on the + // read path, so they must be atomic. + activeSampledHits atomic.Int64 + activeSampledMisses atomic.Int64 + + // globalStats accumulates the hit/miss counts the active policy earned up + // to the last reporting epoch. Per-policy counters are reset at each + // reporting epoch after being delivered to the bandit (epochs gated by + // EvictPartialCapacityFilling skip both the report and the reset), so + // cumulative totals must be kept here. + globalStats GlobalStats + // --- Migration (gradual) --- migrating bool migrateFrom PolicyType @@ -69,242 +53,153 @@ type AdaptiveCache[K comparable, V any] struct { // --- Control Plane --- bandit Bandit + // epochStats holds the per-policy stats measured in the epoch the last + // report covered, keyed by policy. The switch-stability gates in + // allowSwitchLocked read it; it is empty on epochs that skipped reporting. + epochStats map[PolicyType]PolicyStats + + // tenureStats accumulates a policy's measurements for as long as it stays + // in one role, which is what Advice draws on. Per-epoch counters are reset + // after each report, so an answer about the traffic has to be accumulated + // somewhere. + // + // It is cleared for both policies involved in a switch. Pooling a policy's + // active tenure with its shadow tenure would mix two different measurement + // regimes - full capacity over all traffic against miniature capacity over + // a sample - and, worse, would leave the just-demoted policy's long good + // history outweighing the promoted one's short history, so Advice would + // recommend reverting a switch the cache had just made correctly. + tenureStats map[PolicyType]PolicyStats + + // reportingEpochs counts only the epochs that actually measured something. + // epochID counts ticks, including those the capacity gate skipped, and + // reporting that as the evidence behind a recommendation would overstate + // it - sometimes by thousands of epochs to none at all. + reportingEpochs int64 + + // lastSwitchEpoch is the epoch in which the active policy last changed, + // used by the SwitchCooldownEpochs gate. + lastSwitchEpoch int64 + // --- Settings --- epochID int64 epochTicker *time.Ticker settings *Settings - ctx context.Context - cancel context.CancelFunc -} - -func (c *AdaptiveCache[K, V]) runAdaptiveSelect() { - for { - select { - case <-c.ctx.Done(): - c.epochTicker.Stop() - return - case <-c.epochTicker.C: - newPolicy := c.tryChangePolicy() - if c.activePolicy != newPolicy { - c.migrateData(c.activePolicy, newPolicy) - c.activePolicy = newPolicy - } - - c.epochID++ - } - } + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + closeOnce sync.Once } -func (c *AdaptiveCache[K, V]) tryChangePolicy() PolicyType { - c.mu.Lock() - defer c.mu.Unlock() - - currentPolicy := c.activePolicy - - if !c.settings.EvictPartialCapacityFilling && - c.policies[currentPolicy].Len() != c.policies[currentPolicy].Cap() { - return currentPolicy +// recordActiveSample counts the active policy's result for a key that is part +// of the measured sample. Unsampled keys are served normally but not counted, +// so the active arm's evidence covers the same substream as every shadow's. +func (c *AdaptiveCache[K, V]) recordActiveSample(sampled, hit bool) { + if !sampled { + return } - for _, policy := range c.policies { - if policy.GetType() == c.activePolicy { - continue - } - - stats := policy.GetStats() - policy.ResetStats() - - c.bandit.RecordStats(ShadowStats{ - Policy: policy.GetType(), - Hits: stats.Hits, - Misses: stats.Misses, - }) + if hit { + c.activeSampledHits.Add(1) + } else { + c.activeSampledMisses.Add(1) } - - return c.bandit.SelectPolicy() } -// migrateData transfers key/value pairs from the old active policy to the new -// one according to the configured MigrationStrategy. It must be called while -// the write lock is held. -// -// MigrationCold: no-op. -// MigrationWarm: purge stale shadow entries from target, copy all key/value pairs. -// MigrationGradual: purge stale shadow entries from target, snapshot key list, -// and set up the gradual migration window. -func (c *AdaptiveCache[K, V]) migrateData(from, to PolicyType) { - // Abandon any incomplete gradual migration from the previous epoch. - c.clearMigrationState() - - switch c.settings.MigrationStrategy { - case MigrationCold: - // Purge zero-value shadow entries so callers never observe a cached - // zero as if it were a real value. - c.policies[to].Purge() - return - - case MigrationWarm: - fromPolicy := c.policies[from] - toPolicy := c.policies[to] - - // Remove stale zero-value shadow entries so callers never observe a zero - // value as if it were a real cached result. - toPolicy.Purge() +func (c *AdaptiveCache[K, V]) Get(key K) (V, bool) { + sampled := c.sampler.sampled(key) - keys := fromPolicy.Keys() - for _, key := range keys { - val, ok := fromPolicy.Peek(key) - if !ok { - continue + c.mu.RLock() + if !c.migrating { + if sampled { + for _, policy := range c.policies { + if policy.GetType() == c.activePolicy { + continue + } + policy.Get(key) } - toPolicy.Add(key, val) - } - - case MigrationGradual: - // Remove stale zero-value shadow entries from the new active policy. - c.policies[to].Purge() - - keys := c.policies[from].Keys() - realKeys := make(map[K]struct{}, len(keys)) - for _, k := range keys { - realKeys[k] = struct{}{} } - c.migrating = true - c.migrateFrom = from - c.migrationKeys = keys - c.migrationRealKeys = realKeys - } -} - -// clearMigrationState resets all gradual migration fields. It must be called -// while the write lock is held. -func (c *AdaptiveCache[K, V]) clearMigrationState() { - c.migrating = false - c.migrateFrom = Undefined - c.migrationKeys = nil - c.migrationRealKeys = nil -} - -// drainOneKey migrates one pending key from the migration source policy into -// the current active policy. It must be called while the write lock is held. -func (c *AdaptiveCache[K, V]) drainOneKey() { - for len(c.migrationKeys) > 0 { - // Pop from the end (O(1)). - key := c.migrationKeys[len(c.migrationKeys)-1] - c.migrationKeys = c.migrationKeys[:len(c.migrationKeys)-1] + val, found := c.policies[c.activePolicy].Get(key) + c.mu.RUnlock() + c.recordActiveSample(sampled, found) - // Skip keys already promoted via Get or overwritten by a shadow Add. - if _, ok := c.migrationRealKeys[key]; !ok { - continue - } - - val, ok := c.policies[c.migrateFrom].Peek(key) - if !ok { - delete(c.migrationRealKeys, key) - continue - } - - c.policies[c.activePolicy].Add(key, val) - delete(c.migrationRealKeys, key) - - // Close the migration window when the last real key is drained. - if len(c.migrationRealKeys) == 0 { - c.clearMigrationState() - } - return + return val, found } + c.mu.RUnlock() - // Queue exhausted with no promotable keys remaining. - c.clearMigrationState() -} - -// tryPromote attempts to move key from the migration source policy into the -// current active policy. It acquires a write lock and must NOT be called while -// any lock is held. -func (c *AdaptiveCache[K, V]) tryPromote(key K) (V, bool) { + // Gradual migration window: resolve the whole lookup under the write lock, + // promoting an eligible key into the active policy BEFORE its Get is + // counted. The active policy then records a hit for a request the cache + // serves; promoting after the Get would leave a spurious miss in the + // active arm's stats for a served request, skewing both Stats() and the + // bandit's posterior toward the demoted policy. c.mu.Lock() defer c.mu.Unlock() - // Double-check: migration may have ended between the RUnlock and this Lock. - if !c.migrating { - var zero V - return zero, false - } - - // Skip keys whose values have been overwritten by a shadow Add. - if _, ok := c.migrationRealKeys[key]; !ok { - var zero V - return zero, false - } - - val, ok := c.policies[c.migrateFrom].Peek(key) - if !ok { - delete(c.migrationRealKeys, key) - var zero V - return zero, false + if sampled { + for _, policy := range c.policies { + if policy.GetType() == c.activePolicy { + continue + } + policy.Get(key) + } } - c.policies[c.activePolicy].Add(key, val) - delete(c.migrationRealKeys, key) - - return val, true -} - -func (c *AdaptiveCache[K, V]) Get(key K) (V, bool) { - c.mu.RLock() - for _, policy := range c.policies { - if policy.GetType() == c.activePolicy { - continue - } - policy.Get(key) + // Re-check: the window may have closed between the RUnlock and this Lock. + if c.migrating { + c.promoteLocked(key) } val, found := c.policies[c.activePolicy].Get(key) - migrating := c.migrating - c.mu.RUnlock() - - if found || !migrating { - return val, found - } + c.recordActiveSample(sampled, found) - // Miss in the new active policy during a gradual migration window: attempt - // to promote the key from the old active policy. - return c.tryPromote(key) + return val, found } func (c *AdaptiveCache[K, V]) Add(key K, value V) bool { c.mu.Lock() defer c.mu.Unlock() - for _, policy := range c.policies { - if policy.GetType() == c.activePolicy { - continue + if c.sampler.sampled(key) { + for _, policy := range c.policies { + if policy.GetType() == c.activePolicy { + continue + } + var zeroValue V + _ = policy.Add(key, zeroValue) } - var zeroValue V - _ = policy.Add(key, zeroValue) } if c.migrating { - // The shadow Add above just overwrote this key's real value in the - // migration source. Mark it as corrupted so it is never promoted. + // The key is about to be written to the active policy with its real + // value, so it needs no promotion; and if the shadow pass above ran, + // it just overwrote the value held by the migration source. Either + // way the key must not be promoted later. delete(c.migrationRealKeys, key) - // Opportunistically migrate one additional key per Add call. - c.drainOneKey() + if len(c.migrationRealKeys) == 0 { + c.closeMigrationLocked() + } else { + // Opportunistically migrate one additional key per Add call. + c.drainOneKey() + } } return c.policies[c.activePolicy].Add(key, value) } +// Stats returns the cumulative hits and misses served by the cache: totals +// folded up to the last reporting epoch (globalStats) plus the active +// policy's counters accumulated since then. func (c *AdaptiveCache[K, V]) Stats() GlobalStats { c.mu.RLock() defer c.mu.RUnlock() ps := c.policies[c.activePolicy].GetStats() return GlobalStats{ - Hits: ps.Hits, - Misses: ps.Misses, + Hits: c.globalStats.Hits + ps.Hits, + Misses: c.globalStats.Misses + ps.Misses, } } @@ -321,6 +216,11 @@ func (c *AdaptiveCache[K, V]) Remove(key K) bool { if c.migrating { delete(c.migrationRealKeys, key) + // Close the window when the last pending key is removed; a lingering + // window would keep routing every Get through the write lock. + if len(c.migrationRealKeys) == 0 { + c.closeMigrationLocked() + } } return c.policies[c.activePolicy].Remove(key) @@ -333,16 +233,38 @@ func (c *AdaptiveCache[K, V]) Purge() { for _, policy := range c.policies { policy.Purge() } - c.clearMigrationState() + c.closeMigrationLocked() } +// Resize sets the cache's capacity to size and returns the total number of +// entries evicted across all policies. Shadow policies are resized to the +// miniature capacity that corresponds to size rather than to size itself, so +// they stay faithful simulations of a cache of the requested capacity. +// +// The sample rate itself is fixed for the life of the cache: changing it would +// change which keys are sampled, invalidating every shadow's accumulated state. +// The miniature capacity therefore follows the rate directly here, without the +// MinShadowCapacity floor that construction applies - see scaledCapacity. func (c *AdaptiveCache[K, V]) Resize(size int) int { c.mu.Lock() defer c.mu.Unlock() + shadowSize := scaledCapacity(size, c.sampler.rate) + evicted := 0 - for _, policy := range c.policies { - evicted += policy.Resize(size) + for policyType, policy := range c.policies { + c.nominalCap[policyType] = size + c.shadowCap[policyType] = shadowSize + + target := shadowSize + // The active policy serves every key, and a policy that is still + // draining into it under a gradual migration holds the only copy of + // everything not yet promoted. Shrinking either to miniature capacity + // would evict real data the cache is still responsible for. + if policyType == c.activePolicy || (c.migrating && policyType == c.migrateFrom) { + target = size + } + evicted += policy.Resize(target) } return evicted @@ -392,8 +314,14 @@ func (c *AdaptiveCache[K, V]) ActivePolicy() PolicyType { return c.activePolicy } +// Close stops the background epoch goroutine and waits for it to exit. It is +// idempotent and safe to call concurrently; every call returns nil after the +// goroutine has stopped. func (c *AdaptiveCache[K, V]) Close() error { - c.cancel() + c.closeOnce.Do(func() { + c.cancel() + c.wg.Wait() + }) return nil } diff --git a/cache_test.go b/cache_test.go index 0b2daf1..e328a09 100644 --- a/cache_test.go +++ b/cache_test.go @@ -162,10 +162,8 @@ func makeCache(t *testing.T, strategy MigrationStrategy) ( return ac, lru, lfu, bandit } -// forceSwitchTo triggers a policy switch by manipulating the bandit and calling -// tryChangePolicy directly (it is unexported, so we call it via the background -// ticker using a very-short epoch duration cache built just for that purpose). -// For unit tests we instead call the internal method directly via a thin helper. +// triggerSwitch applies a policy switch synchronously, taking the same path +// runEpoch does so that promotion, migration and demotion are all exercised. func triggerSwitch(ac *AdaptiveCache[string, int], to PolicyType) { ac.mu.Lock() defer ac.mu.Unlock() @@ -174,9 +172,7 @@ func triggerSwitch(ac *AdaptiveCache[string, int], to PolicyType) { if from == to { return } - ac.clearMigrationState() - ac.migrateData(from, to) - ac.activePolicy = to + ac.switchLocked(from, to) } // --- MigrationCold --- @@ -299,6 +295,28 @@ func TestMigrationGradual_ZeroValueNotPromoted(t *testing.T) { assert.Equal(t, 77, val, "gradual: expected latest Add value") } +// TestMigrationGradual_PromotedGetCountsAsHit verifies that a Get served via +// promotion from the migration source is recorded as a hit on the active +// policy — not as a miss — so Stats() and the bandit report reflect a request +// the cache actually served. +func TestMigrationGradual_PromotedGetCountsAsHit(t *testing.T) { + ac, _, lfu, _ := makeCache(t, MigrationGradual) + + ac.Add("a", 42) + triggerSwitch(ac, LFU) + + val, ok := ac.Get("a") + require.True(t, ok, "promoted Get must serve the value") + require.Equal(t, 42, val) + + stats := lfu.GetStats() + assert.Equal(t, int64(1), stats.Hits, "promoted Get must count as a hit on the active policy") + assert.Equal(t, int64(0), stats.Misses, "promoted Get must not leave a spurious miss") + + assert.Equal(t, GlobalStats{Hits: 1, Misses: 0}, ac.Stats(), + "Stats must report the served request as a hit") +} + func TestMigrationGradual_EpochClearsMigration(t *testing.T) { ac, _, _, _ := makeCache(t, MigrationGradual) @@ -346,6 +364,38 @@ func TestMigrationGradual_RemovePreventsPromotion(t *testing.T) { assert.False(t, ok, "gradual: expected miss after Remove, got (%d, true)", val) } +// TestMigrationGradual_RemoveLastKeyClosesWindow guards against a phantom +// migration window: Remove of the last pending key must close the window, +// otherwise every subsequent Get keeps taking the write lock until some other +// event (Add, Purge, policy switch) happens to end the migration. +func TestMigrationGradual_RemoveLastKeyClosesWindow(t *testing.T) { + ac, _, _, _ := makeCache(t, MigrationGradual) + + ac.Add("a", 1) + triggerSwitch(ac, LFU) + require.True(t, ac.migrating, "expected migration window to open") + + ac.Remove("a") + + ac.mu.RLock() + migrating := ac.migrating + ac.mu.RUnlock() + assert.False(t, migrating, "window must close when Remove empties the pending key set") +} + +// TestMigrationGradual_EmptySourceOpensNoWindow verifies that switching away +// from an empty policy does not open a migration window at all. +func TestMigrationGradual_EmptySourceOpensNoWindow(t *testing.T) { + ac, _, _, _ := makeCache(t, MigrationGradual) + + triggerSwitch(ac, LFU) + + ac.mu.RLock() + migrating := ac.migrating + ac.mu.RUnlock() + assert.False(t, migrating, "empty source must not open a migration window") +} + func TestMigrationGradual_DrainCompletesNaturally(t *testing.T) { ac, _, lfu, _ := makeCache(t, MigrationGradual) @@ -398,7 +448,6 @@ func TestMigrationGradual_Concurrent(t *testing.T) { wg.Add(goroutines * 2) for g := 0; g < goroutines; g++ { - g := g go func() { defer wg.Done() for i := 0; i < 50; i++ { @@ -536,6 +585,56 @@ func TestNewAdaptiveCache_NilPolicies(t *testing.T) { assert.ErrorIs(t, err, ErrEmptyPolicies) } +func TestNewAdaptiveCache_NilBandit(t *testing.T) { + _, err := NewAdaptiveCache( + []Policy[string, int]{newMockPolicy[string, int](LRU, 10)}, + nil, + &Settings{EpochDuration: time.Hour}, + ) + assert.ErrorIs(t, err, ErrNilBandit) +} + +func TestNewAdaptiveCache_NilSettings(t *testing.T) { + _, err := NewAdaptiveCache( + []Policy[string, int]{newMockPolicy[string, int](LRU, 10)}, + &mockBandit{next: LRU}, + nil, + ) + assert.ErrorIs(t, err, ErrNilSettings) +} + +func TestNewAdaptiveCache_NonPositiveEpochDuration(t *testing.T) { + for _, d := range []time.Duration{0, -time.Second} { + _, err := NewAdaptiveCache( + []Policy[string, int]{newMockPolicy[string, int](LRU, 10)}, + &mockBandit{next: LRU}, + &Settings{EpochDuration: d}, + ) + assert.ErrorIs(t, err, ErrInvalidEpochDuration, "duration %s must be rejected", d) + } +} + +func TestNewAdaptiveCache_NilPolicyEntry(t *testing.T) { + _, err := NewAdaptiveCache( + []Policy[string, int]{newMockPolicy[string, int](LRU, 10), nil}, + &mockBandit{next: LRU}, + &Settings{EpochDuration: time.Hour}, + ) + assert.ErrorIs(t, err, ErrNilPolicy) +} + +func TestNewAdaptiveCache_DuplicatePolicyType(t *testing.T) { + _, err := NewAdaptiveCache( + []Policy[string, int]{ + newMockPolicy[string, int](LRU, 10), + newMockPolicy[string, int](LRU, 10), + }, + &mockBandit{next: LRU}, + &Settings{EpochDuration: time.Hour}, + ) + assert.ErrorIs(t, err, ErrDuplicatePolicy) +} + // --------------------------------------------------------------------------- // AdaptiveCache: tryChangePolicy via epoch ticker // --------------------------------------------------------------------------- @@ -634,6 +733,171 @@ func TestAdaptiveCache_TryChangePolicy_SkipsWhenNotFull(t *testing.T) { assert.Equal(t, LRU, selected, "expected no switch when cache is not full and EvictPartialCapacityFilling=false") } +// --------------------------------------------------------------------------- +// AdaptiveCache: switch stability (cool-down, minimum improvement, min samples) +// --------------------------------------------------------------------------- + +// makeStabilityCache builds a two-policy cache whose bandit always picks LFU, +// with a 24h epoch so only explicit runEpoch calls advance it. +func makeStabilityCache(t *testing.T, s *Settings) ( + *AdaptiveCache[string, int], + *mockPolicy[string, int], + *mockPolicy[string, int], +) { + t.Helper() + + lru := newMockPolicy[string, int](LRU, 10) + lfu := newMockPolicy[string, int](LFU, 10) + + s.EpochDuration = 24 * time.Hour + s.EvictPartialCapacityFilling = true + + ac, err := NewAdaptiveCache([]Policy[string, int]{lru, lfu}, &mockBandit{next: LFU}, s) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + return ac, lru, lfu +} + +// primeStats gives the active policy `activeHits` hits out of `total` requests +// and the shadow the complement, by driving Get against keys that do or do not +// exist. Both policies see every Get, so we control the split by pre-seeding +// only the policy we want to hit. +func primeStats(p *mockPolicy[string, int], hits, misses int64) { + p.mu.Lock() + p.stats = PolicyStats{Hits: hits, Misses: misses} + p.mu.Unlock() +} + +// primeActiveStats sets the evidence the active arm reports to the bandit. +// The active policy is judged on the sampled substream rather than on its own +// full counters, so its epoch evidence lives on the cache, not on the policy. +func primeActiveStats(ac *AdaptiveCache[string, int], hits, misses int64) { + ac.activeSampledHits.Store(hits) + ac.activeSampledMisses.Store(misses) +} + +func TestSwitchStability_ZeroSettingsSwitchesAsBefore(t *testing.T) { + ac, lru, lfu := makeStabilityCache(t, &Settings{}) + + primeActiveStats(ac, 5, 5) + primeStats(lfu, 5, 5) + _ = lru + + ac.runEpoch() + + assert.Equal(t, LFU, ac.ActivePolicy(), + "a zero-valued Settings must apply every bandit selection") +} + +func TestSwitchStability_MinHitRateImprovementBlocksMarginalSwitch(t *testing.T) { + ac, lru, lfu := makeStabilityCache(t, &Settings{MinHitRateImprovement: 0.10}) + + // Candidate is better, but only by 0.02 - below the 0.10 threshold. + primeActiveStats(ac, 50, 50) // active hit rate 0.50 + primeStats(lfu, 52, 48) // candidate hit rate 0.52 + _ = lru + + ac.runEpoch() + + assert.Equal(t, LRU, ac.ActivePolicy(), + "a marginal improvement must not trigger a switch") +} + +func TestSwitchStability_MinHitRateImprovementAllowsClearWin(t *testing.T) { + ac, lru, lfu := makeStabilityCache(t, &Settings{MinHitRateImprovement: 0.10}) + + primeActiveStats(ac, 40, 60) // active hit rate 0.40 + primeStats(lfu, 70, 30) // candidate hit rate 0.70, +0.30 + _ = lru + + ac.runEpoch() + + assert.Equal(t, LFU, ac.ActivePolicy(), + "an improvement above the threshold must trigger a switch") +} + +func TestSwitchStability_CooldownBlocksConsecutiveSwitches(t *testing.T) { + ac, lru, lfu := makeStabilityCache(t, &Settings{SwitchCooldownEpochs: 3}) + + // Epoch 0: lastSwitchEpoch is 0 and epochID is 0, so the cool-down has + // not elapsed yet and the switch is held back. + primeStats(lru, 1, 1) + primeStats(lfu, 1, 1) + ac.runEpoch() + require.Equal(t, LRU, ac.ActivePolicy(), "cool-down must hold the first switch") + + // Epochs 1 and 2 remain inside the window. + ac.runEpoch() + ac.runEpoch() + require.Equal(t, LRU, ac.ActivePolicy(), "cool-down must still hold at epoch 2") + + // Epoch 3: three epochs have elapsed, the switch is allowed. + ac.runEpoch() + assert.Equal(t, LFU, ac.ActivePolicy(), "switch must be allowed once the cool-down elapses") +} + +func TestSwitchStability_CooldownRearmsAfterSwitch(t *testing.T) { + ac, _, _ := makeStabilityCache(t, &Settings{SwitchCooldownEpochs: 2}) + + // Drive epochs until the first switch lands. + for i := 0; i < 3; i++ { + ac.runEpoch() + } + require.Equal(t, LFU, ac.ActivePolicy(), "expected the first switch to land") + + ac.mu.RLock() + lastSwitch, epochID := ac.lastSwitchEpoch, ac.epochID + ac.mu.RUnlock() + + assert.Equal(t, epochID-1, lastSwitch, "lastSwitchEpoch must record the switching epoch") +} + +func TestSwitchStability_MinEpochRequestsBlocksThinEvidence(t *testing.T) { + ac, _, lfu := makeStabilityCache(t, &Settings{MinEpochRequests: 100}) + + primeActiveStats(ac, 1, 1) // 2 requests + primeStats(lfu, 5, 0) // 5 requests, perfect hit rate but far too few + ac.runEpoch() + require.Equal(t, LRU, ac.ActivePolicy(), "a handful of samples must not trigger a switch") + + primeActiveStats(ac, 100, 100) + primeStats(lfu, 200, 0) + ac.runEpoch() + assert.Equal(t, LFU, ac.ActivePolicy(), "enough samples must allow the switch") +} + +func TestSwitchStability_GatedEpochClearsEvidence(t *testing.T) { + lru := newMockPolicy[string, int](LRU, 10) + lfu := newMockPolicy[string, int](LFU, 10) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, + &mockBandit{next: LFU}, + &Settings{ + EpochDuration: 24 * time.Hour, + // Require a full cache before switching; the mocks stay empty. + EvictPartialCapacityFilling: false, + MinHitRateImprovement: 0.01, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + primeActiveStats(ac, 10, 90) + primeStats(lfu, 90, 10) + _ = lru + + ac.runEpoch() + + assert.Equal(t, LRU, ac.ActivePolicy(), "a gated epoch must not switch") + + ac.mu.RLock() + stats := len(ac.epochStats) + ac.mu.RUnlock() + assert.Zero(t, stats, "a gated epoch must not leave stale evidence for the gates") +} + // --------------------------------------------------------------------------- // AdaptiveCache: epoch-based background switching // --------------------------------------------------------------------------- @@ -664,6 +928,108 @@ func TestAdaptiveCache_EpochBasedSwitch(t *testing.T) { assert.Equal(t, LFU, ac.ActivePolicy(), "expected epoch-based switch to LFU") } +// --------------------------------------------------------------------------- +// AdaptiveCache: bandit receives active policy stats; Stats() stays cumulative +// --------------------------------------------------------------------------- + +// TestAdaptiveCache_BanditReceivesActivePolicyStats verifies that the active +// policy's epoch stats are reported to the bandit alongside the shadows and +// that every policy's counters are reset after reporting. +func TestAdaptiveCache_BanditReceivesActivePolicyStats(t *testing.T) { + lruP := newMockPolicy[string, int](LRU, 10) + lfuP := newMockPolicy[string, int](LFU, 10) + bandit := &recordingBandit{next: LRU} + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lruP, lfuP}, + bandit, + &Settings{ + EpochDuration: 24 * time.Hour, + EvictPartialCapacityFilling: true, + MigrationStrategy: MigrationCold, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + ac.Add("x", 1) + ac.Get("x") // hit in active LRU and in shadow LFU (shadow-added key) + ac.Get("missing") // miss everywhere + + _ = ac.tryChangePolicy() + + byPolicy := map[PolicyType]ShadowStats{} + for _, r := range bandit.getRecords() { + byPolicy[r.Policy] = r + } + + require.Contains(t, byPolicy, LRU, "active policy stats must reach the bandit") + require.Contains(t, byPolicy, LFU, "shadow policy stats must reach the bandit") + assert.Equal(t, int64(1), byPolicy[LRU].Hits, "active policy hits mismatch") + assert.Equal(t, int64(1), byPolicy[LRU].Misses, "active policy misses mismatch") + assert.Equal(t, int64(1), byPolicy[LFU].Hits, "shadow policy hits mismatch") + assert.Equal(t, int64(1), byPolicy[LFU].Misses, "shadow policy misses mismatch") + + assert.Equal(t, PolicyStats{}, lruP.GetStats(), "active counters must reset after reporting") + assert.Equal(t, PolicyStats{}, lfuP.GetStats(), "shadow counters must reset after reporting") +} + +// TestAdaptiveCache_StatsCumulativeAcrossEpochs verifies that Stats() keeps +// cumulative totals even though per-policy counters are reset every epoch. +func TestAdaptiveCache_StatsCumulativeAcrossEpochs(t *testing.T) { + ac, _, _, _ := makeCache(t, MigrationCold) + + ac.Add("a", 1) + ac.Get("a") // hit + ac.Get("missing") // miss + + before := ac.Stats() + require.Equal(t, GlobalStats{Hits: 1, Misses: 1}, before) + + // Simulate an epoch boundary: stats are reported to the bandit and the + // per-policy counters reset. + _ = ac.tryChangePolicy() + + assert.Equal(t, before, ac.Stats(), "Stats must be cumulative across epoch resets") + + ac.Get("a") + assert.Equal(t, GlobalStats{Hits: 2, Misses: 1}, ac.Stats(), "Stats must keep accumulating after the reset") +} + +// TestAdaptiveCache_DemotionResetsCounters verifies that a demoted policy +// starts its first shadow epoch with clean counters: its active-tenure stats +// must not leak into the next epoch's shadow report. +func TestAdaptiveCache_DemotionResetsCounters(t *testing.T) { + lruP := newMockPolicy[string, int](LRU, 10) + lfuP := newMockPolicy[string, int](LFU, 10) + bandit := &recordingBandit{next: LFU} + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lruP, lfuP}, + bandit, + &Settings{ + EpochDuration: 24 * time.Hour, + EvictPartialCapacityFilling: true, + MigrationStrategy: MigrationCold, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + ac.Add("x", 1) + ac.Get("x") // hit while LRU is active + + // Epoch boundary: the bandit picks LFU, demoting LRU to a shadow. + ac.runEpoch() + require.Equal(t, LFU, ac.ActivePolicy()) + + ac.Get("y") // miss in active LFU and in shadow LRU + + // The demoted LRU must carry only post-demotion traffic. + assert.Equal(t, PolicyStats{Misses: 1}, lruP.GetStats(), + "active-tenure stats leaked into the first shadow epoch") +} + // --------------------------------------------------------------------------- // AdaptiveCache: context cancellation stops background goroutine // --------------------------------------------------------------------------- @@ -683,6 +1049,142 @@ func TestAdaptiveCache_Close(t *testing.T) { } } +func TestAdaptiveCache_Close_Idempotent(t *testing.T) { + ac, _, _, _ := makeCache(t, MigrationCold) + + require.NoError(t, ac.Close()) + require.NoError(t, ac.Close()) +} + +func TestAdaptiveCache_Close_Concurrent(t *testing.T) { + ac, _, _, _ := makeCache(t, MigrationCold) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + assert.NoError(t, ac.Close()) + }() + } + wg.Wait() +} + +// TestAdaptiveCache_Close_StopsEpochGoroutine verifies Close waits for the +// background goroutine: once Close returns, the epoch counter cannot advance. +func TestAdaptiveCache_Close_StopsEpochGoroutine(t *testing.T) { + lruP := newMockPolicy[string, int](LRU, 10) + lfuP := newMockPolicy[string, int](LFU, 10) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lruP, lfuP}, + &mockBandit{next: LRU}, + &Settings{ + EpochDuration: time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: MigrationCold, + }, + ) + require.NoError(t, err) + + require.NoError(t, ac.Close()) + + ac.mu.RLock() + before := ac.epochID + ac.mu.RUnlock() + + time.Sleep(20 * time.Millisecond) + + ac.mu.RLock() + after := ac.epochID + ac.mu.RUnlock() + + assert.Equal(t, before, after, "epoch goroutine kept running after Close returned") +} + +// blockingBandit parks inside SelectPolicy until released, so a test can hold +// the background epoch goroutine provably in-flight. +type blockingBandit struct { + entered chan struct{} // buffered(1); signalled on first SelectPolicy entry + release chan struct{} // closed to let SelectPolicy return +} + +func (b *blockingBandit) RecordStats(_ ShadowStats) {} + +func (b *blockingBandit) SelectPolicy() PolicyType { + select { + case b.entered <- struct{}{}: + default: + } + <-b.release + return LRU +} + +// TestAdaptiveCache_Close_WaitsForInFlightEpoch verifies that Close blocks on +// the WaitGroup: while the epoch goroutine is parked inside the bandit, Close +// must not return, and it must return promptly once the goroutine can exit. +// Without wg.Wait in Close this fails at the "Close returned while in-flight" +// select. +func TestAdaptiveCache_Close_WaitsForInFlightEpoch(t *testing.T) { + bandit := &blockingBandit{ + entered: make(chan struct{}, 1), + release: make(chan struct{}), + } + + ac, err := NewAdaptiveCache( + []Policy[string, int]{ + newMockPolicy[string, int](LRU, 10), + newMockPolicy[string, int](LFU, 10), + }, + bandit, + &Settings{ + EpochDuration: time.Millisecond, + // Load-bearing: with partial filling disallowed the not-yet-full + // cache would make selectPolicyLocked return before ever calling + // bandit.SelectPolicy, and the goroutine would never park. + EvictPartialCapacityFilling: true, + MigrationStrategy: MigrationCold, + }, + ) + require.NoError(t, err) + + // If an assertion below fails, unblock the bandit so the epoch goroutine + // (parked while holding the cache mutex) does not leak, then Close. + var releaseOnce sync.Once + t.Cleanup(func() { + releaseOnce.Do(func() { close(bandit.release) }) + _ = ac.Close() + }) + + select { + case <-bandit.entered: + case <-time.After(5 * time.Second): + require.FailNow(t, "epoch goroutine never reached the bandit") + } + + closed := make(chan struct{}) + go func() { + _ = ac.Close() + close(closed) + }() + + select { + case <-closed: + require.FailNow(t, "Close returned while the epoch goroutine was still in-flight") + case <-time.After(50 * time.Millisecond): + // expected: Close is blocked in wg.Wait + } + + releaseOnce.Do(func() { close(bandit.release) }) + + select { + case <-closed: + // expected: goroutine exited, Close returned + case <-time.After(5 * time.Second): + require.FailNow(t, "Close did not return after the epoch goroutine exited") + } +} + // --------------------------------------------------------------------------- // AdaptiveCache: Remove propagates to shadow policies // --------------------------------------------------------------------------- @@ -771,3 +1273,199 @@ func TestPolicyType_String(t *testing.T) { assert.Equal(t, tt.want, got, "PolicyType(%d).String() mismatch", tt.pt) } } + +// --------------------------------------------------------------------------- +// Concurrency regression tests +// --------------------------------------------------------------------------- + +// flipBandit alternates its selection on every call so the active policy +// changes on every epoch tick, exercising the migrate + activePolicy swap path. +type flipBandit struct { + mu sync.Mutex + n int +} + +func (b *flipBandit) RecordStats(_ ShadowStats) {} +func (b *flipBandit) SelectPolicy() PolicyType { + b.mu.Lock() + defer b.mu.Unlock() + b.n++ + if b.n%2 == 0 { + return LRU + } + return LFU +} + +// TestAdaptiveCache_ConcurrentSwitchAndAccess_NoRace guards against the data +// race where runEpoch mutated activePolicy / migration state outside the lock. +// Before the fix this fails under `go test -race`. +func TestAdaptiveCache_ConcurrentSwitchAndAccess_NoRace(t *testing.T) { + lruP := newMockPolicy[string, int](LRU, 100) + lfuP := newMockPolicy[string, int](LFU, 100) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lruP, lfuP}, + &flipBandit{}, + &Settings{ + EpochDuration: time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: MigrationWarm, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + var wg sync.WaitGroup + stop := make(chan struct{}) + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + ac.Add("k", 1) + ac.Get("k") + ac.Contains("k") + _ = ac.Keys() + _ = ac.Len() + _ = ac.ActivePolicy() + } + } + }() + } + + time.Sleep(150 * time.Millisecond) + close(stop) + wg.Wait() +} + +// TestCacheWrapper_ConcurrentGet_NoRace guards against the data race where +// CacheWrapper.Get mutated its stats counters non-atomically while callers +// invoked it concurrently. Before the fix this fails under `go test -race`. +func TestCacheWrapper_ConcurrentGet_NoRace(t *testing.T) { + underlying := newMockPolicy[string, int](LRU, 100) + w := NewCache[string, int](underlying, LRU, 100) + w.Add("k", 1) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 20000; j++ { + w.Get("k") + w.Get("missing") + } + }() + } + wg.Wait() + + stats := w.GetStats() + // 8 goroutines * 20000 iters each of one hit + one miss. + assert.Equal(t, int64(8*20000), stats.Hits, "hit count must not be lost to races") + assert.Equal(t, int64(8*20000), stats.Misses, "miss count must not be lost to races") +} + +// TestAdaptiveCache_StressAcrossEpochBoundaries hammers the full public API +// from many goroutines while 1ms epochs force a policy switch on every tick, +// for each migration strategy. It asserts nothing beyond survival: its job is +// to let the race detector observe every cross-epoch interleaving. +func TestAdaptiveCache_StressAcrossEpochBoundaries(t *testing.T) { + strategies := map[string]MigrationStrategy{ + "cold": MigrationCold, + "warm": MigrationWarm, + "gradual": MigrationGradual, + } + + for name, strategy := range strategies { + t.Run(name, func(t *testing.T) { + lruP := newMockPolicy[string, int](LRU, 100) + lfuP := newMockPolicy[string, int](LFU, 100) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lruP, lfuP}, + &flipBandit{}, + &Settings{ + EpochDuration: time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: strategy, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + const goroutines = 4 + var wg sync.WaitGroup + stop := make(chan struct{}) + + // Writers: Add with periodic Remove. + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(seed int) { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + key := string(rune('a' + (seed+i)%26)) + ac.Add(key, seed*1000+i) + if i%7 == 0 { + ac.Remove(key) + } + } + } + }(g) + } + + // Readers: every read-path method. + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(seed int) { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + key := string(rune('a' + (seed+i)%26)) + ac.Get(key) + ac.Peek(key) + ac.Contains(key) + _ = ac.Keys() + _ = ac.Len() + _ = ac.Stats() + _ = ac.ActivePolicy() + } + } + }(g) + } + + // Maintenance: occasional Purge and Resize. + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + time.Sleep(10 * time.Millisecond) + if i%2 == 0 { + ac.Purge() + } else { + ac.Resize(100) + } + } + } + }() + + time.Sleep(200 * time.Millisecond) + close(stop) + wg.Wait() + }) + } +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..1516878 --- /dev/null +++ b/doc.go @@ -0,0 +1,67 @@ +// Package ascache is a cache that chooses its own eviction policy. +// +// Choosing a replacement policy normally means guessing which one suits your +// traffic, and the cost of guessing wrong is large: on a cyclic access pattern +// just larger than the cache, LRU serves a 0% hit rate where W-TinyLFU serves +// 92%. This library removes the guess. It runs candidate policies side by side, +// measures them against your real traffic, and either tells you which one wins +// or switches to it for you. +// +// # How it works +// +// One policy is active and serves real data. The others are shadows: they +// receive the same key stream with zero values, purely so their hit rates can +// be compared. Every epoch each policy reports what it measured to a [Bandit], +// which picks the policy for the next epoch. +// +// cache, err := ascache.NewAdaptiveCache( +// []ascache.Policy[string, int]{lru, twoQ, tinyLFU}, +// myBandit, +// &ascache.Settings{EpochDuration: time.Minute}, +// ) +// defer cache.Close() +// +// The API is a superset of hashicorp/golang-lru/v2, so an existing cache can be +// swapped for one of these without changing call sites. [AdaptiveCache.Stats], +// [AdaptiveCache.Advice], [AdaptiveCache.ActivePolicy] and +// [AdaptiveCache.Close] are the additions. +// +// Ready-made policies live in companion modules, so the core has no +// dependencies: github.com/sshaplygin/as-cache/policies for LRU, 2Q, Random +// and TTL, .../policies/arc for ARC, .../policies/tinylfu for W-TinyLFU. +// +// # Start by observing +// +// The lowest-risk way to adopt this is not to let it switch anything. With +// [Settings.ObserveOnly] the cache behaves exactly like the first policy it was +// given, while every other policy is measured in the background, and +// [AdaptiveCache.Advice] reports what it found. No bandit is needed in this +// mode. +// +// cache, _ := ascache.NewAdaptiveCache(policies, nil, &ascache.Settings{ +// EpochDuration: time.Minute, +// ObserveOnly: true, +// }) +// // ... later ... +// fmt.Println(cache.Advice()) +// +// # Cost +// +// Shadow policies hold keys and eviction bookkeeping but never values, so they +// cost far less than a full copy: six policies measure at 2.65x the memory of +// one, and 1.32x with [Settings.ShadowSampleRate] set. Sampling has shadows +// track a deterministic fraction of the keyspace, which stops per-operation +// cost scaling with the number of policies. +// +// # What to expect +// +// Adaptive selection reliably beats the worst policy you might have picked and +// lands close to the best. On published traces it comes within about a point +// of the best fixed policy and occasionally beats it, without being told in +// advance which that is. It will not dramatically outperform a policy you have +// already measured and know suits your traffic. +// +// Epoch duration is the setting that matters most: too short and the cache +// spends its time migrating between policies rather than serving. See the +// README for measurements and configuration guidance. +package ascache diff --git a/epoch.go b/epoch.go new file mode 100644 index 0000000..efe0af9 --- /dev/null +++ b/epoch.go @@ -0,0 +1,130 @@ +package ascache + +func (c *AdaptiveCache[K, V]) runAdaptiveSelect() { + defer c.wg.Done() + defer c.epochTicker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-c.epochTicker.C: + c.runEpoch() + } + } +} + +// runEpoch performs one epoch tick: it selects the next policy, migrates data +// when the policy changes and the stability gates allow it, and advances the +// epoch counter. The entire sequence runs under the write lock so concurrent +// cache operations never observe a half-applied switch (a torn activePolicy or +// partially migrated state). +func (c *AdaptiveCache[K, V]) runEpoch() { + c.mu.Lock() + defer c.mu.Unlock() + + // A gradual migration window lasts at most one epoch. Left open it would + // never close on a workload that stops touching the keys still pending: + // the source would hold real values at full capacity indefinitely, compete + // as an arm measured at a capacity no other shadow runs at, and keep every + // Get on the write-locked path. Closing here also demotes it, so it is a + // comparable miniature by the time stats are collected below. + c.closeMigrationLocked() + + newPolicy := c.selectPolicyLocked() + if c.settings.ObserveOnly { + // Measure, report, advise - but never act. The cache keeps behaving + // exactly like the policy it was built with. + c.epochID++ + + return + } + + if c.activePolicy != newPolicy && c.allowSwitchLocked(newPolicy) { + c.switchLocked(c.activePolicy, newPolicy) + c.lastSwitchEpoch = c.epochID + } + + c.epochID++ +} + +// tryChangePolicy records every policy's stats with the bandit (nothing on a +// gated epoch — see selectPolicyLocked) and returns the policy selected for +// the next epoch. It acquires the write lock and performs no migration. It +// exists as a lock-acquiring entry point; callers that already hold the lock +// must use selectPolicyLocked instead. +func (c *AdaptiveCache[K, V]) tryChangePolicy() PolicyType { + c.mu.Lock() + defer c.mu.Unlock() + + return c.selectPolicyLocked() +} + +// selectPolicyLocked reports every policy's stats to the bandit — the active +// policy included, so its posterior does not go stale — and returns the +// bandit's chosen policy for the next epoch. When +// EvictPartialCapacityFilling is false and the active policy is not yet full, +// it returns early without reporting or resetting anything; counters then +// accumulate until the next reporting epoch. On a reporting epoch counters +// are reset after delivery; the active policy's counts are folded into +// globalStats first so Stats() stays cumulative and no active-tenure counts +// leak into a policy's first shadow epoch after demotion. It must be called +// while the write lock is held. +func (c *AdaptiveCache[K, V]) selectPolicyLocked() PolicyType { + currentPolicy := c.activePolicy + + // The capacity gate exists to avoid switching on the strength of a + // half-full cache. In ObserveOnly mode nothing switches, so the gate would + // only suppress the measurement the caller is running the cache for. + if !c.settings.ObserveOnly && !c.settings.EvictPartialCapacityFilling && + c.policies[currentPolicy].Len() != c.policies[currentPolicy].Cap() { + // Nothing was measured this epoch: drop the previous epoch's numbers + // so the stability gates never compare against stale evidence. + clear(c.epochStats) + return currentPolicy + } + + if c.epochStats == nil { + c.epochStats = make(map[PolicyType]PolicyStats, len(c.policies)) + } + if c.tenureStats == nil { + c.tenureStats = make(map[PolicyType]PolicyStats, len(c.policies)) + } + c.reportingEpochs++ + + for _, policy := range c.policies { + stats := policy.GetStats() + policy.ResetStats() + + reported := stats + if policy.GetType() == currentPolicy { + // Stats() reports everything the cache served, so the active + // policy's full counters are what accumulate there. + c.globalStats.Hits += stats.Hits + c.globalStats.Misses += stats.Misses + + // The bandit instead sees the active policy measured over the + // sampled substream, the same one the shadows are measured over, + // so no arm is judged on more evidence than another. + reported = PolicyStats{ + Hits: c.activeSampledHits.Swap(0), + Misses: c.activeSampledMisses.Swap(0), + } + } + + c.epochStats[policy.GetType()] = reported + + tenure := c.tenureStats[policy.GetType()] + tenure.Hits += reported.Hits + tenure.Misses += reported.Misses + c.tenureStats[policy.GetType()] = tenure + + c.bandit.RecordStats(ShadowStats{ + Policy: policy.GetType(), + Hits: reported.Hits, + Misses: reported.Misses, + }) + } + + return c.bandit.SelectPolicy() +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..60a8e84 --- /dev/null +++ b/errors.go @@ -0,0 +1,26 @@ +package ascache + +import "errors" + +// ErrEmptyPolicies is returned by NewAdaptiveCache when the policies slice is +// nil or empty. +var ErrEmptyPolicies = errors.New("must provide non zero policies size") + +// ErrNilPolicy is returned by NewAdaptiveCache when one of the provided +// policies is nil. +var ErrNilPolicy = errors.New("policy must not be nil") + +// ErrDuplicatePolicy is returned by NewAdaptiveCache when two policies report +// the same PolicyType. +var ErrDuplicatePolicy = errors.New("duplicate policy type") + +// ErrNilBandit is returned by NewAdaptiveCache when the bandit is nil. +var ErrNilBandit = errors.New("bandit must not be nil") + +// ErrNilSettings is returned by NewAdaptiveCache when settings is nil. +var ErrNilSettings = errors.New("settings must not be nil") + +// ErrInvalidEpochDuration is returned by NewAdaptiveCache when +// Settings.EpochDuration is zero or negative: time.NewTicker panics on +// non-positive durations. +var ErrInvalidEpochDuration = errors.New("epoch duration must be positive") diff --git a/evicting_test.go b/evicting_test.go new file mode 100644 index 0000000..d735191 --- /dev/null +++ b/evicting_test.go @@ -0,0 +1,308 @@ +package ascache + +import ( + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// evictingPolicy is a Policy that actually enforces its capacity, evicting in +// insertion order when full and on a shrinking Resize. +// +// mockPolicy deliberately does not: its Resize only records the new capacity. +// That blind spot hid three defects in the capacity handling around sampling, +// because every test that resized a policy saw its data survive regardless. +// Any test whose subject is capacity must use this type instead. +type evictingPolicy[K comparable, V any] struct { + mu sync.Mutex + data map[K]V + order []K + cap int + policyType PolicyType + stats PolicyStats +} + +func newEvictingPolicy[K comparable, V any](policyType PolicyType, capacity int) *evictingPolicy[K, V] { + return &evictingPolicy[K, V]{ + data: make(map[K]V, capacity), + cap: capacity, + policyType: policyType, + } +} + +// evictLocked drops oldest-first until the cache is within capacity, returning +// how many entries it removed. +func (p *evictingPolicy[K, V]) evictLocked() int { + evicted := 0 + for p.cap >= 0 && len(p.data) > p.cap { + oldest := p.order[0] + p.order = p.order[1:] + if _, ok := p.data[oldest]; ok { + delete(p.data, oldest) + evicted++ + } + } + + return evicted +} + +func (p *evictingPolicy[K, V]) Add(key K, value V) bool { + p.mu.Lock() + defer p.mu.Unlock() + + if _, existed := p.data[key]; !existed { + p.order = append(p.order, key) + } + p.data[key] = value + + return p.evictLocked() > 0 +} + +func (p *evictingPolicy[K, V]) Get(key K) (V, bool) { + p.mu.Lock() + defer p.mu.Unlock() + + v, ok := p.data[key] + if ok { + p.stats.Hits++ + } else { + p.stats.Misses++ + } + + return v, ok +} + +func (p *evictingPolicy[K, V]) Peek(key K) (V, bool) { + p.mu.Lock() + defer p.mu.Unlock() + v, ok := p.data[key] + + return v, ok +} + +func (p *evictingPolicy[K, V]) Contains(key K) bool { + p.mu.Lock() + defer p.mu.Unlock() + _, ok := p.data[key] + + return ok +} + +func (p *evictingPolicy[K, V]) Remove(key K) bool { + p.mu.Lock() + defer p.mu.Unlock() + _, ok := p.data[key] + delete(p.data, key) + + return ok +} + +func (p *evictingPolicy[K, V]) Purge() { + p.mu.Lock() + defer p.mu.Unlock() + p.data = make(map[K]V, p.cap) + p.order = nil +} + +func (p *evictingPolicy[K, V]) Keys() []K { + p.mu.Lock() + defer p.mu.Unlock() + + keys := make([]K, 0, len(p.data)) + for _, k := range p.order { + if _, ok := p.data[k]; ok { + keys = append(keys, k) + } + } + + return keys +} + +func (p *evictingPolicy[K, V]) Values() []V { + p.mu.Lock() + defer p.mu.Unlock() + + vals := make([]V, 0, len(p.data)) + for _, v := range p.data { + vals = append(vals, v) + } + + return vals +} + +func (p *evictingPolicy[K, V]) Len() int { + p.mu.Lock() + defer p.mu.Unlock() + + return len(p.data) +} + +func (p *evictingPolicy[K, V]) Cap() int { + p.mu.Lock() + defer p.mu.Unlock() + + return p.cap +} + +func (p *evictingPolicy[K, V]) Resize(size int) int { + p.mu.Lock() + defer p.mu.Unlock() + p.cap = size + + return p.evictLocked() +} + +func (p *evictingPolicy[K, V]) GetStats() PolicyStats { + p.mu.Lock() + defer p.mu.Unlock() + + return p.stats +} + +func (p *evictingPolicy[K, V]) ResetStats() { + p.mu.Lock() + defer p.mu.Unlock() + p.stats = PolicyStats{} +} + +func (p *evictingPolicy[K, V]) GetType() PolicyType { return p.policyType } + +// makeEvictingCache builds a cache over two capacity-enforcing policies whose +// bandit always selects banditPick, so a test controls whether an epoch tick +// switches policies or leaves the active one in place. +func makeEvictingCache(t *testing.T, capacity int, banditPick PolicyType, settings *Settings) ( + *AdaptiveCache[string, int], + *evictingPolicy[string, int], + *evictingPolicy[string, int], +) { + t.Helper() + + lru := newEvictingPolicy[string, int](LRU, capacity) + lfu := newEvictingPolicy[string, int](LFU, capacity) + + settings.EpochDuration = 24 * time.Hour + settings.EvictPartialCapacityFilling = true + + ac, err := NewAdaptiveCache([]Policy[string, int]{lru, lfu}, &mockBandit{next: banditPick}, settings) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + return ac, lru, lfu +} + +// TestResize_KeepsGradualMigrationSourceIntact guards a resize that shrank the +// policy still draining into the active one down to miniature capacity, +// evicting the only copy of every key not yet promoted. +func TestResize_KeepsGradualMigrationSourceIntact(t *testing.T) { + const capacity = 2000 + + ac, _, _ := makeEvictingCache(t, capacity, LRU, &Settings{ + MigrationStrategy: MigrationGradual, + ShadowSampleRate: 0.05, + MinShadowCapacity: 8, + }) + + for i := 0; i < capacity; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + + triggerSwitch(ac, LFU) + require.True(t, ac.migrating, "expected a gradual window to open") + + // A resize to the capacity the cache already has must not lose anything. + ac.Resize(capacity) + + found := 0 + for i := 0; i < capacity; i++ { + if got, ok := ac.Get("key-" + strconv.Itoa(i)); ok { + require.Equal(t, i, got, "key-%d returned a value that was never stored", i) + found++ + } + } + + assert.Equal(t, capacity, found, + "a no-op resize during a gradual migration must not discard pending data") +} + +// TestResize_ShadowCapacityTracksTheSampleRate guards a shrinking resize that +// left shadows running at a capacity larger than their share of the traffic. +// A shadow of capacity C fed an r-sampled stream simulates a cache of C/r, so +// breaking that identity makes every shadow look better than the active policy +// regardless of which policy is actually better. +func TestResize_ShadowCapacityTracksTheSampleRate(t *testing.T) { + ac, _, lfu := makeEvictingCache(t, 20000, LRU, &Settings{ShadowSampleRate: 0.05}) + + rate := ac.sampler.rate + require.InDelta(t, 0.05, rate, 1e-9, "expected the requested rate to survive construction") + + for _, size := range []int{20000, 10000, 2000, 500, 100} { + ac.Resize(size) + + got := float64(lfu.Cap()) / float64(size) + assert.InDelta(t, rate, got, 0.01, + "after Resize(%d) the shadow runs at capacity %d, which is %.4f of the cache but samples %.4f of the keys", + size, lfu.Cap(), got, rate) + } +} + +// TestEpoch_ClosesAbandonedGradualWindow guards a window that stayed open +// forever when the workload stopped touching the keys still pending: the +// source kept real values at full capacity, competed as an arm no other shadow +// was comparable with, and forced every Get onto the write-locked path. +func TestEpoch_ClosesAbandonedGradualWindow(t *testing.T) { + // The bandit keeps naming LFU, the policy that becomes active, so the epoch + // tick does not switch and only the window-closing path can demote LRU. + ac, lru, _ := makeEvictingCache(t, 1000, LFU, &Settings{ + MigrationStrategy: MigrationGradual, + ShadowSampleRate: 0.05, + MinShadowCapacity: 8, + }) + + for i := 0; i < 200; i++ { + ac.Add("key-"+strconv.Itoa(i), i+1) + } + + triggerSwitch(ac, LFU) + require.True(t, ac.migrating, "expected a gradual window to open") + require.Equal(t, 1000, lru.Cap(), "the source keeps full capacity while the window is open") + + // The workload never touches a pending key again; only an epoch tick can + // end the window. + ac.runEpoch() + + ac.mu.RLock() + migrating := ac.migrating + ac.mu.RUnlock() + + assert.False(t, migrating, "an epoch boundary must close an abandoned gradual window") + assert.Equal(t, ac.shadowCap[LRU], lru.Cap(), + "the source must be demoted to miniature capacity once the window closes") + + lru.mu.Lock() + defer lru.mu.Unlock() + for key, value := range lru.data { + require.Zero(t, value, "the demoted source must not keep real values (key %q)", key) + } +} + +// TestDemotion_ResetsStats guards a demoted policy reporting measurements it +// took in its previous role, at a different capacity and over all traffic +// rather than the sample. +func TestDemotion_ResetsStats(t *testing.T) { + ac, lru, _ := makeEvictingCache(t, 1000, LRU, &Settings{ShadowSampleRate: 0.05}) + + for i := 0; i < 100; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + ac.Get("key-" + strconv.Itoa(i)) + } + require.NotEqual(t, PolicyStats{}, lru.GetStats(), "the active policy should have measured something") + + triggerSwitch(ac, LFU) + + assert.Equal(t, PolicyStats{}, lru.GetStats(), + "a demoted policy must start its shadow tenure with no carried-over measurements") +} diff --git a/examples/basic/basic b/examples/basic/basic deleted file mode 100755 index 9ab031c..0000000 Binary files a/examples/basic/basic and /dev/null differ diff --git a/examples/basic/main.go b/examples/basic/main.go index d1d49a4..8caa805 100644 --- a/examples/basic/main.go +++ b/examples/basic/main.go @@ -29,16 +29,15 @@ func main() { panic(err) } - policiesList := []ascache.Policy[string, *UserProfile]{ - ascache.NewCache(lruCache, ascache.LRU, 100), - } - lfuCache, err := slfu.New[string, *UserProfile](100) if err != nil { panic(err) } - policiesList = append(policiesList, ascache.NewCache(lfuCache, ascache.LFU, 100)) + policiesList := []ascache.Policy[string, *UserProfile]{ + ascache.NewCache(lruCache, ascache.LRU, 100), + ascache.NewCache(lfuCache, ascache.LFU, 100), + } armNames := []ascache.PolicyType{ascache.LRU, ascache.LFU} @@ -58,9 +57,6 @@ func main() { } defer cache.Close() - val, ok := cache.Get("key") - fmt.Println(val, ok) - mux := http.NewServeMux() mux.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) { @@ -76,7 +72,7 @@ func main() { return } - w.Write([]byte(val.Name)) + fmt.Fprint(w, val.Name) }) mux.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) { @@ -98,8 +94,12 @@ func main() { return } - _ = cache.Add(key, &UserProfile{}) - w.Write([]byte("ok")) + _ = cache.Add(key, &UserProfile{ + Name: name, + Email: email, + CreatedAt: time.Now(), + }) + fmt.Fprint(w, "ok") }) server := &http.Server{ @@ -200,9 +200,9 @@ func (crs *CacheRewardSource) updateStats(policy ascache.PolicyType, hits, misse s.Misses += float64(misses) } -//==================================================================== +// ===================================================================== // 2. ADAPTER IMPLEMENTING THE `Bandit` INTERFACE -//==================================================================== +// ===================================================================== // StitchFixBanditAdapter wraps the stitchfix bandit and implements the // ascache.Bandit interface. diff --git a/examples/migration/main.go b/examples/migration/main.go index 85b0cde..ec49a03 100644 --- a/examples/migration/main.go +++ b/examples/migration/main.go @@ -28,10 +28,11 @@ import ( "sync" "time" - ascache "github.com/sshaplygin/as-cache" - slfu "github.com/sshaplygin/as-cache/lfu" hlru "github.com/hashicorp/golang-lru/v2" "github.com/stitchfix/mab" + + ascache "github.com/sshaplygin/as-cache" + slfu "github.com/sshaplygin/as-cache/lfu" ) // ─── Thompson Sampling reward source ───────────────────────────────────────── @@ -437,8 +438,8 @@ func main() { bandit := &controllableBandit{inner: inner} policies := []ascache.Policy[string, string]{ - ascache.NewCache[string, string](lruCache, ascache.LRU, 100), - ascache.NewCache[string, string](lfuCache, ascache.LFU, 100), + ascache.NewCache(lruCache, ascache.LRU, 100), + ascache.NewCache(lfuCache, ascache.LFU, 100), } cache, err := ascache.NewAdaptiveCache( diff --git a/examples/migration/migration b/examples/migration/migration deleted file mode 100755 index 377fa2b..0000000 Binary files a/examples/migration/migration and /dev/null differ diff --git a/interfaces.go b/interfaces.go index d708f29..06b298f 100644 --- a/interfaces.go +++ b/interfaces.go @@ -2,7 +2,10 @@ package ascache var _ Cacher[int, string] = (*AdaptiveCache[int, string])(nil) -// cache interface comparable from hashicorp/golang-lru/v2 cache's +// Cacher is the cache interface an eviction policy must satisfy to be used as +// an arm. It is deliberately identical to the method set of +// hashicorp/golang-lru/v2, so an existing cache is usually already a Cacher, +// and so an AdaptiveCache is a drop-in replacement for one. type Cacher[K comparable, V any] interface { Add(key K, value V) (evicted bool) Contains(key K) bool @@ -21,23 +24,41 @@ type Cacher[K comparable, V any] interface { // RemoveOldest() (key K, value V, ok bool) } +// CacheStats is the hit/miss accounting a policy exposes so its performance +// can be compared with the other arms. type CacheStats interface { GetStats() PolicyStats ResetStats() } +// Policy is a cache that can serve as one arm of an AdaptiveCache: a Cacher +// that also reports its capacity, its measurements, and which policy it is. type Policy[K comparable, V any] interface { Cacher[K, V] - // hashicorp/golang-lru/v2 doesn't have this method + + // Cap reports the capacity. hashicorp/golang-lru/v2 has no such method, + // but the adaptive layer needs it: shadow policies run at a reduced + // capacity and are restored to their full one when promoted. Cap() int CacheStats GetType() PolicyType } +// Bandit chooses which policy should be active, given what each has measured. +// +// This package ships no implementation, because the choice of strategy is the +// interesting part and depends on how quickly the traffic changes. A +// Thompson-sampling implementation with evidence discounting is in the bench +// module and is short enough to copy. type Bandit interface { - // RecordStats delivers a performance report from one of the shadow caches - // for the previous epoch. + // RecordStats delivers one policy's performance report. On every + // reporting epoch each policy reports — the active policy included — so + // implementations receive a full set of arms and must not synthesize + // stats for the active arm themselves. When + // Settings.EvictPartialCapacityFilling is false, epochs where the active + // policy is not yet full skip reporting entirely; counters then + // accumulate and the next report spans the skipped epochs. RecordStats(stats ShadowStats) // SelectPolicy asks the bandit to choose which policy should become the diff --git a/lfu/lfu_test.go b/lfu/lfu_test.go index 48ed3b1..6b7ad4d 100644 --- a/lfu/lfu_test.go +++ b/lfu/lfu_test.go @@ -452,3 +452,191 @@ func TestAdd_EvictionCallbackCorrectValues(t *testing.T) { require.Contains(t, evicted, "b", "expected 'b' to be evicted") assert.Equal(t, 2, evicted["b"]) } + +// --------------------------------------------------------------------------- +// Resize / ContainsOrAdd / PeekOrAdd / RemoveOldest / GetOldest +// +// These exported wrapper methods previously had no test coverage at all, which +// is why the nil-pointer panics in the underlying simplelfu bucket index went +// unnoticed: Resize, RemoveOldest and GetOldest are their public entry points. +// --------------------------------------------------------------------------- + +func TestResize_ShrinkEvictsLeastFrequentlyUsed(t *testing.T) { + c, err := New[string, int](4) + require.NoError(t, err) + + c.Add("a", 1) + c.Add("b", 2) + c.Add("c", 3) + c.Get("a") // raise "a" above the rest + + evicted := c.Resize(2) + assert.Equal(t, 1, evicted, "shrinking 3 entries to capacity 2 evicts one") + assert.Equal(t, 2, c.Len()) + assert.True(t, c.Contains("a"), "the most frequently used entry must survive") +} + +func TestResize_GrowEvictsNothing(t *testing.T) { + c, err := New[string, int](2) + require.NoError(t, err) + + c.Add("a", 1) + c.Add("b", 2) + + assert.Zero(t, c.Resize(10), "growing must not evict") + assert.Equal(t, 2, c.Len()) + + c.Add("c", 3) // now fits without eviction + assert.Equal(t, 3, c.Len()) +} + +func TestResize_ToZeroThenAddStaysBounded(t *testing.T) { + c, err := New[string, int](2) + require.NoError(t, err) + + c.Add("a", 1) + assert.Equal(t, 1, c.Resize(0), "Resize(0) drains the cache") + assert.Zero(t, c.Len()) + + c.Add("b", 2) // must not panic through the wrapper + c.Add("c", 3) + assert.LessOrEqual(t, c.Len(), 1, "a degenerate capacity must stay bounded") +} + +func TestResize_EvictionCallback(t *testing.T) { + evicted := make(map[string]int) + c, err := NewWithEvict[string, int](4, func(k string, v int) { evicted[k] = v }) + require.NoError(t, err) + + c.Add("a", 1) + c.Add("b", 2) + c.Add("c", 3) + + n := c.Resize(1) + assert.Equal(t, 2, n) + assert.Len(t, evicted, 2, "callback must fire for every evicted entry") +} + +func TestContainsOrAdd(t *testing.T) { + c, err := New[string, int](2) + require.NoError(t, err) + + ok, evicted := c.ContainsOrAdd("a", 1) + assert.False(t, ok, "'a' did not exist yet") + assert.False(t, evicted) + + ok, evicted = c.ContainsOrAdd("a", 99) + assert.True(t, ok, "'a' now exists") + assert.False(t, evicted) + + v, found := c.Peek("a") + require.True(t, found) + assert.Equal(t, 1, v, "ContainsOrAdd must not overwrite an existing value") +} + +func TestContainsOrAdd_EvictionCallback(t *testing.T) { + var gotKey string + c, err := NewWithEvict[string, int](1, func(k string, _ int) { gotKey = k }) + require.NoError(t, err) + + c.Add("a", 1) + ok, evicted := c.ContainsOrAdd("b", 2) + assert.False(t, ok) + assert.True(t, evicted, "adding past capacity evicts") + assert.Equal(t, "a", gotKey, "callback must report the evicted key") +} + +func TestPeekOrAdd(t *testing.T) { + c, err := New[string, int](2) + require.NoError(t, err) + + prev, ok, evicted := c.PeekOrAdd("a", 1) + assert.Zero(t, prev) + assert.False(t, ok, "'a' did not exist yet") + assert.False(t, evicted) + + prev, ok, evicted = c.PeekOrAdd("a", 99) + assert.Equal(t, 1, prev, "must return the existing value") + assert.True(t, ok) + assert.False(t, evicted) +} + +func TestPeekOrAdd_EvictionCallback(t *testing.T) { + var gotKey string + c, err := NewWithEvict[string, int](1, func(k string, _ int) { gotKey = k }) + require.NoError(t, err) + + c.Add("a", 1) + _, ok, evicted := c.PeekOrAdd("b", 2) + assert.False(t, ok) + assert.True(t, evicted) + assert.Equal(t, "a", gotKey) +} + +func TestRemoveOldest(t *testing.T) { + c, err := New[string, int](3) + require.NoError(t, err) + + _, _, ok := c.RemoveOldest() + assert.False(t, ok, "an empty cache has no oldest entry") + + c.Add("a", 1) + c.Add("b", 2) + c.Get("a") // "b" is now the least frequently used + + k, v, ok := c.RemoveOldest() + require.True(t, ok) + assert.Equal(t, "b", k) + assert.Equal(t, 2, v) + assert.False(t, c.Contains("b"), "the oldest entry must be gone") + assert.Equal(t, 1, c.Len()) +} + +func TestRemoveOldest_EvictionCallback(t *testing.T) { + var gotKey string + var gotVal int + c, err := NewWithEvict[string, int](3, func(k string, v int) { gotKey, gotVal = k, v }) + require.NoError(t, err) + + c.Add("a", 1) + _, _, ok := c.RemoveOldest() + require.True(t, ok) + assert.Equal(t, "a", gotKey) + assert.Equal(t, 1, gotVal) +} + +func TestGetOldest(t *testing.T) { + c, err := New[string, int](3) + require.NoError(t, err) + + _, _, ok := c.GetOldest() + assert.False(t, ok, "an empty cache has no oldest entry") + + c.Add("a", 1) + c.Add("b", 2) + c.Get("a") // "b" is now the least frequently used + + k, v, ok := c.GetOldest() + require.True(t, ok) + assert.Equal(t, "b", k) + assert.Equal(t, 2, v) + assert.Equal(t, 2, c.Len(), "GetOldest must not remove the entry") +} + +// Removing an entry that empties the minimum-frequency bucket previously left +// the index pointing at an empty bucket, panicking on the next GetOldest. +func TestGetOldest_AfterRemoveEmptiesMinBucket(t *testing.T) { + c, err := New[string, int](3) + require.NoError(t, err) + + c.Add("a", 1) + c.Add("b", 2) + c.Get("a") // "a" -> freq 2, "b" alone in the freq-1 bucket + + require.True(t, c.Remove("b")) + + k, v, ok := c.GetOldest() + require.True(t, ok, "GetOldest must survive an emptied minimum bucket") + assert.Equal(t, "a", k) + assert.Equal(t, 1, v) +} diff --git a/lfu/simplelfu/lfu.go b/lfu/simplelfu/lfu.go index 4852308..93a98dc 100644 --- a/lfu/simplelfu/lfu.go +++ b/lfu/simplelfu/lfu.go @@ -37,6 +37,14 @@ func NewLFU[K comparable, V any](size int, onEvict EvictCallback[K, V]) (*LFU[K, } func (c *LFU[K, V]) Add(key K, value V) (evicted bool) { + if c.size <= 0 { + // A cache of zero capacity holds nothing. Without this an Add into a + // cache left at size 0 by Resize skips the eviction below (there is + // nothing to evict) and then stores the entry anyway, so the cache + // holds an entry it has no room for. + return false + } + ent, ok := c.items[key] if ok { ent.Value = value @@ -44,16 +52,11 @@ func (c *LFU[K, V]) Add(key K, value V) (evicted bool) { return } - evicted = len(c.items) == c.size - if evicted { - ent := c.evictList[c.minFreq].Back() - c.evictList[c.minFreq].Remove(ent) - - delete(c.items, ent.Key) - - if c.onEvict != nil { - c.onEvict(ent.Key, ent.Value) - } + // Evict only when the cache actually holds an entry. Gating on the item + // count as well as the capacity keeps a degenerate size - such as the one + // left behind by Resize(0) - from addressing a bucket that does not exist. + if len(c.items) > 0 && len(c.items) >= c.size { + _, _, evicted = c.evictOldest() } newFreq := 1 @@ -165,13 +168,29 @@ func (c *LFU[K, V]) Resize(size int) (evicted int) { // GetOldest returns the least-frequently-used item without removing it. func (c *LFU[K, V]) GetOldest() (key K, value V, ok bool) { - if len(c.items) == 0 { + ent := c.oldest() + if ent == nil { return } - ent := c.evictList[c.minFreq].Back() return ent.Key, ent.Value, true } +// oldest returns the least-frequently-used entry, or nil when the cache holds +// nothing. It never assumes minFreq addresses a live bucket, so a corrupted +// index degrades to a miss instead of a panic. +func (c *LFU[K, V]) oldest() *internal.Entry[K, V] { + if len(c.items) == 0 { + return nil + } + + bucket, found := c.evictList[c.minFreq] + if !found { + return nil + } + + return bucket.Back() +} + // RemoveOldest removes the least-frequently-used item and returns it. func (c *LFU[K, V]) RemoveOldest() (key K, value V, ok bool) { return c.evictOldest() @@ -179,25 +198,20 @@ func (c *LFU[K, V]) RemoveOldest() (key K, value V, ok bool) { // evictOldest removes the back entry of the minFreq bucket and returns it. func (c *LFU[K, V]) evictOldest() (key K, value V, ok bool) { - if len(c.items) == 0 { + ent := c.oldest() + if ent == nil { return } - ent := c.evictList[c.minFreq].Back() + key, value, ok = ent.Key, ent.Value, true - c.evictList[c.minFreq].Remove(ent) - if c.evictList[c.minFreq].Length() == 0 { - delete(c.evictList, c.minFreq) - c.minFreq = 0 - for freq := range c.evictList { - if c.minFreq == 0 || freq < c.minFreq { - c.minFreq = freq - } - } - } + + c.detach(ent) delete(c.items, ent.Key) + if c.onEvict != nil { c.onEvict(ent.Key, ent.Value) } + return } @@ -236,9 +250,41 @@ func (c *LFU[K, V]) updateFreq(ent *internal.Entry[K, V]) { // removeElement is used to remove a given list element from the cache func (c *LFU[K, V]) removeElement(e *internal.Entry[K, V]) { - c.evictList[e.Freq].Remove(e) + c.detach(e) delete(c.items, e.Key) if c.onEvict != nil { c.onEvict(e.Key, e.Value) } } + +// detach unlinks ent from its frequency bucket, dropping the bucket once it is +// empty and repairing minFreq when the minimum bucket disappears. It maintains +// the invariant that every bucket in evictList holds at least one entry and +// that minFreq addresses a live bucket whenever the cache is non-empty. +func (c *LFU[K, V]) detach(ent *internal.Entry[K, V]) { + bucket, found := c.evictList[ent.Freq] + if !found { + return + } + + bucket.Remove(ent) + if bucket.Length() > 0 { + return + } + + delete(c.evictList, ent.Freq) + if c.minFreq == ent.Freq { + c.recomputeMinFreq() + } +} + +// recomputeMinFreq points minFreq at the smallest surviving bucket, or resets it +// to 0 when the cache holds no buckets at all. +func (c *LFU[K, V]) recomputeMinFreq() { + c.minFreq = 0 + for freq := range c.evictList { + if c.minFreq == 0 || freq < c.minFreq { + c.minFreq = freq + } + } +} diff --git a/lfu/simplelfu/lfu_test.go b/lfu/simplelfu/lfu_test.go index 538b4fd..ff3a0d8 100644 --- a/lfu/simplelfu/lfu_test.go +++ b/lfu/simplelfu/lfu_test.go @@ -1,10 +1,13 @@ package simplelfu import ( + "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/sshaplygin/as-cache/lfu/internal" ) func TestNewLFU_PositiveSize(t *testing.T) { @@ -475,3 +478,182 @@ func TestFrequencyPromotionAcrossMultipleGets(t *testing.T) { assert.True(t, c.Contains("b"), "expected 'b' to remain") assert.True(t, c.Contains("c"), "expected 'c' to remain") } + +// --------------------------------------------------------------------------- +// Bucket-invariant regression tests +// +// Every bucket in evictList must hold at least one entry, and minFreq must +// address a live bucket whenever the cache is non-empty. Each test below +// panicked with a nil-pointer dereference before that invariant was enforced. +// --------------------------------------------------------------------------- + +// assertBucketInvariant fails if any frequency bucket is empty or if minFreq +// does not address a live bucket while entries remain. +func assertBucketInvariant[K comparable, V any](t *testing.T, c *LFU[K, V]) { + t.Helper() + + for freq, bucket := range c.evictList { + assert.NotZero(t, bucket.Length(), "bucket freq=%d must not be empty", freq) + } + + if len(c.items) > 0 { + bucket, found := c.evictList[c.minFreq] + require.True(t, found, "minFreq=%d must address a live bucket", c.minFreq) + assert.NotZero(t, bucket.Length(), "minFreq bucket must not be empty") + } +} + +// Add's eviction path must drop the minFreq bucket once it empties, otherwise a +// later minFreq recompute selects the stale empty bucket and dereferences nil. +func TestAdd_EvictDropsEmptiedBucket(t *testing.T) { + c, err := NewLFU[string, int](2, nil) + require.NoError(t, err) + + c.Add("a", 1) + c.Add("b", 2) + c.Get("a") + c.Get("a") + c.Get("b") + + c.Add("c", 3) // evicts "b" and empties its bucket + assertBucketInvariant(t, c) + + c.Add("d", 4) // evicts "c" and empties its bucket + assertBucketInvariant(t, c) + + k1, _, ok := c.RemoveOldest() + require.True(t, ok, "first RemoveOldest must succeed") + assert.Equal(t, "d", k1) + assertBucketInvariant(t, c) + + k2, _, ok := c.RemoveOldest() + require.True(t, ok, "second RemoveOldest must succeed (previously panicked)") + assert.Equal(t, "a", k2) + assertBucketInvariant(t, c) + + assert.Zero(t, c.Len(), "cache must be drained") +} + +// Resize(0) drains the cache and leaves size==0; the next Add must not take the +// eviction branch and dereference a bucket that no longer exists. +func TestResize_ZeroThenAddDoesNotPanic(t *testing.T) { + c, err := NewLFU[string, int](2, nil) + require.NoError(t, err) + + c.Add("a", 1) + evicted := c.Resize(0) + assert.Equal(t, 1, evicted, "Resize(0) must evict the single entry") + assert.Zero(t, c.Len()) + + c.Add("b", 2) // previously panicked: evictList[minFreq=0] was nil + assertBucketInvariant(t, c) + + // A zero-capacity cache holds nothing. This assertion used to require the + // opposite - that "b" was retrievable - which made LFU the only policy in + // the repository that kept an entry it had no room for. Since the adaptive + // layer resizes policies on its own, that inconsistency was a trap rather + // than a feature. + _, ok := c.Get("b") + assert.False(t, ok, "a zero-capacity cache must not serve an entry it had no room for") + assert.Zero(t, c.Len()) + + // The cache stays bounded rather than growing without limit. + c.Add("c", 3) + assertBucketInvariant(t, c) + assert.Zero(t, c.Len(), "degenerate size must hold nothing") +} + +// Remove must drop an emptied bucket and repair minFreq, otherwise GetOldest +// reads the back of an empty bucket and dereferences nil. +func TestRemove_RepairsMinFreqWhenBucketEmpties(t *testing.T) { + c, err := NewLFU[string, int](2, nil) + require.NoError(t, err) + + c.Add("a", 1) + c.Add("b", 2) + c.Get("a") // "a" -> freq 2, leaving "b" alone in the freq-1 bucket + + require.True(t, c.Remove("b"), "removing 'b' empties the freq-1 bucket") + assertBucketInvariant(t, c) + + k, v, ok := c.GetOldest() // previously panicked: minFreq still pointed at freq 1 + require.True(t, ok, "GetOldest must find 'a'") + assert.Equal(t, "a", k) + assert.Equal(t, 1, v) +} + +// The bucket index is an internal invariant that the code above is responsible +// for upholding. These white-box cases corrupt it deliberately to prove the +// lookup helpers degrade to a miss rather than dereferencing a nil bucket, +// keeping the package free of panics (CLAUDE.md rule 5). +func TestCorruptedIndex_DegradesToMissInsteadOfPanic(t *testing.T) { + t.Run("minFreq addresses a missing bucket", func(t *testing.T) { + c, err := NewLFU[string, int](2, nil) + require.NoError(t, err) + c.Add("a", 1) + c.minFreq = 99 // no such bucket exists + + _, _, ok := c.GetOldest() + assert.False(t, ok, "GetOldest must report a miss") + + _, _, ok = c.RemoveOldest() + assert.False(t, ok, "RemoveOldest must report a miss") + }) + + t.Run("minFreq addresses an empty bucket", func(t *testing.T) { + c, err := NewLFU[string, int](2, nil) + require.NoError(t, err) + c.Add("a", 1) + c.evictList[7] = internal.NewList[string, int]() + c.minFreq = 7 // bucket exists but holds nothing + + _, _, ok := c.GetOldest() + assert.False(t, ok, "GetOldest must report a miss") + + _, _, ok = c.RemoveOldest() + assert.False(t, ok, "RemoveOldest must report a miss") + }) + + t.Run("Resize stops when eviction cannot progress", func(t *testing.T) { + c, err := NewLFU[string, int](4, nil) + require.NoError(t, err) + c.Add("a", 1) + c.Add("b", 2) + c.minFreq = 99 // eviction can no longer find an entry to drop + + assert.Zero(t, c.Resize(1), "Resize must not report phantom evictions") + }) + + t.Run("detach ignores an entry whose bucket is absent", func(t *testing.T) { + c, err := NewLFU[string, int](2, nil) + require.NoError(t, err) + c.Add("a", 1) + + c.detach(&internal.Entry[string, int]{Key: "ghost", Freq: 42}) + assert.Equal(t, 1, c.Len(), "cache must be unchanged") + }) +} + +// TestLFU_ZeroCapacityHoldsNothing guards a cache resized to zero that still +// accepted entries: Add skipped its eviction step because there was nothing to +// evict, then stored the entry regardless. Every other policy in this +// repository holds nothing at zero capacity, and the adaptive layer resizes +// policies on its own, so the odd one out is a trap. +func TestLFU_ZeroCapacityHoldsNothing(t *testing.T) { + cache, err := NewLFU[string, int](4, nil) + require.NoError(t, err) + + for i := 0; i < 4; i++ { + cache.Add("key-"+strconv.Itoa(i), i) + } + require.Equal(t, 4, cache.Len()) + + cache.Resize(0) + assert.Zero(t, cache.Len(), "resizing to zero must empty the cache") + + assert.False(t, cache.Add("fresh", 1), "a zero-capacity cache stores nothing") + assert.Zero(t, cache.Len(), "a zero-capacity cache must stay empty") + + _, ok := cache.Get("fresh") + assert.False(t, ok) +} diff --git a/metrics/go.mod b/metrics/go.mod new file mode 100644 index 0000000..46bde1c --- /dev/null +++ b/metrics/go.mod @@ -0,0 +1,23 @@ +module github.com/sshaplygin/as-cache/metrics + +go 1.25.2 + +require ( + github.com/sshaplygin/as-cache v0.0.0 + github.com/sshaplygin/as-cache/policies v0.0.0 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.6 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sshaplygin/as-cache/lfu v0.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/sshaplygin/as-cache => .. + +replace github.com/sshaplygin/as-cache/policies => ../policies + +replace github.com/sshaplygin/as-cache/lfu => ../lfu diff --git a/metrics/go.sum b/metrics/go.sum new file mode 100644 index 0000000..dc21af3 --- /dev/null +++ b/metrics/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..3d4f28b --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,171 @@ +// Package metrics exposes an AdaptiveCache's measurements to monitoring. +// +// A cache that changes its own eviction policy is only safe to run if you can +// see what it is doing. This package turns the cache's own accounting into a +// snapshot suitable for scraping, and publishes it through expvar. +// +// It depends on nothing outside the standard library. A Prometheus collector +// is a few lines on top of Snapshot; see the package example rather than a +// dependency here, because how metrics are named and labelled is a decision +// that belongs to the application, not to a cache library. +package metrics + +import ( + "encoding/json" + "expvar" + "fmt" + "sort" + "sync" + + ascache "github.com/sshaplygin/as-cache" +) + +// publishMu serialises Publish so its check and its registration cannot be +// interleaved with another Publish from this package. +var publishMu sync.Mutex + +// Advisor is the part of an AdaptiveCache this package reads. Every +// AdaptiveCache satisfies it, whatever its key and value types, which is why +// this is an interface rather than a generic parameter. +type Advisor interface { + Advice() ascache.Advice + Stats() ascache.GlobalStats + ActivePolicy() ascache.PolicyType + Len() int +} + +// PolicySnapshot is one policy's measurements at a point in time. +type PolicySnapshot struct { + Policy string `json:"policy"` + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hit_rate"` + Active bool `json:"active"` +} + +// Snapshot is everything worth exporting about a cache at a point in time. +// +// The fields are chosen so that the two questions an operator actually asks +// are answerable from a dashboard: is the cache working (Served, HitRate, +// Entries), and is it about to do something surprising (ActivePolicy, Epochs, +// BestPolicy, Improvement). +type Snapshot struct { + // ActivePolicy is the policy serving requests. Graph it over time: this is + // the series that shows switching behaviour. + ActivePolicy string `json:"active_policy"` + // Epochs is how many measurement rounds have completed. + Epochs int64 `json:"epochs"` + // Entries is how many entries the active policy currently holds. + Entries int `json:"entries"` + + // Hits, Misses and HitRate describe the traffic the cache actually served, + // unsampled. + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hit_rate"` + + // BestPolicy is the policy with the best measured hit rate, and + // Improvement is how many points it beats the active one by. When + // Improvement stays high, the cache is leaving hit rate on the table - + // either because switching is gated off, or because it is in observe-only + // mode and waiting for a human. + BestPolicy string `json:"best_policy"` + Improvement float64 `json:"improvement"` + + // Sampled reports whether the per-policy numbers are estimates from a + // sampled substream. Hits, Misses and HitRate above are never sampled. + Sampled bool `json:"sampled"` + SampleRate float64 `json:"sample_rate"` + + // Policies holds every arm, best hit rate first. + Policies []PolicySnapshot `json:"policies"` +} + +// Take reads a cache's current measurements. +func Take(cache Advisor) Snapshot { + advice := cache.Advice() + stats := cache.Stats() + + snapshot := Snapshot{ + ActivePolicy: advice.Active.String(), + Epochs: advice.Epochs, + Entries: cache.Len(), + Hits: stats.Hits, + Misses: stats.Misses, + BestPolicy: advice.Best.String(), + Improvement: advice.Improvement, + Sampled: advice.Sampled, + SampleRate: advice.SampleRate, + Policies: make([]PolicySnapshot, 0, len(advice.Reports)), + } + + if total := stats.Hits + stats.Misses; total > 0 { + snapshot.HitRate = float64(stats.Hits) / float64(total) + } + + for _, report := range advice.Reports { + snapshot.Policies = append(snapshot.Policies, PolicySnapshot{ + Policy: report.Policy.String(), + Hits: report.Hits, + Misses: report.Misses, + HitRate: report.HitRate(), + Active: report.Active, + }) + } + + sort.SliceStable(snapshot.Policies, func(i, j int) bool { + return snapshot.Policies[i].HitRate > snapshot.Policies[j].HitRate + }) + + return snapshot +} + +// String renders a snapshot as JSON, which is what expvar serves. +func (s Snapshot) String() string { + encoded, err := json.Marshal(s) + if err != nil { + // Snapshot contains only plain scalars and slices of the same, so + // this cannot fail; report rather than panic if it somehow does. + return fmt.Sprintf("{%q:%q}", "error", err.Error()) + } + + return string(encoded) +} + +// Publish exposes a cache's snapshot under the given name in expvar, so it +// appears in the /debug/vars handler alongside the rest of a process's +// published state. +// +// The value is computed when scraped rather than on a timer, so publishing +// costs nothing until something reads it. +// +// expvar panics if a name is published twice, which would take down a process +// over a metrics-registration mistake, so this reports that as an error +// instead. +// +// Checking with expvar.Get before publishing is not enough on its own: the two +// calls are separate, so a concurrent publisher can register the name in +// between and the panic happens anyway. A mutex closes that window for callers +// of this function, and the recover closes it for a name registered directly +// through expvar by something else - which this package cannot synchronise +// with, and which is exactly the mistake a shared metric name produces. +func Publish(name string, cache Advisor) (err error) { + publishMu.Lock() + defer publishMu.Unlock() + + if expvar.Get(name) != nil { + return fmt.Errorf("metrics: %q is already published", name) + } + + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("metrics: publishing %q: %v", name, recovered) + } + }() + + expvar.Publish(name, expvar.Func(func() any { + return Take(cache) + })) + + return nil +} diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go new file mode 100644 index 0000000..494d005 --- /dev/null +++ b/metrics/metrics_test.go @@ -0,0 +1,159 @@ +package metrics_test + +import ( + "encoding/json" + "expvar" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/metrics" + "github.com/sshaplygin/as-cache/policies" +) + +// newCache builds an observe-only cache with two arms, which is the +// configuration an operator would run first. +func newCache(t *testing.T) *ascache.AdaptiveCache[string, int] { + t.Helper() + + lru, err := policies.NewLRU[string, int](1000) + require.NoError(t, err) + twoQ, err := policies.NewTwoQueue[string, int](1000) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache[string, int]( + []ascache.Policy[string, int]{lru, twoQ}, nil, + &ascache.Settings{EpochDuration: 5 * time.Millisecond, ObserveOnly: true}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + return cache +} + +// drive puts enough traffic through the cache that at least one epoch has been +// measured, so a snapshot has something in it. +func drive(t *testing.T, cache *ascache.AdaptiveCache[string, int]) { + t.Helper() + + deadline := time.Now().Add(200 * time.Millisecond) + for i := 0; time.Now().Before(deadline); i++ { + key := "key-" + strconv.Itoa(i%500) + if _, ok := cache.Get(key); !ok { + cache.Add(key, i) + } + if cache.Advice().Epochs > 2 { + return + } + } + + require.Positive(t, cache.Advice().Epochs, "expected at least one epoch to elapse") +} + +func TestTake_ReportsTheCacheState(t *testing.T) { + cache := newCache(t) + drive(t, cache) + + snapshot := metrics.Take(cache) + + assert.Equal(t, "LRU", snapshot.ActivePolicy, "observe-only keeps the first policy active") + assert.Positive(t, snapshot.Epochs) + assert.Positive(t, snapshot.Entries) + assert.Positive(t, snapshot.Hits+snapshot.Misses, "the cache served traffic") + assert.InDelta(t, float64(snapshot.Hits)/float64(snapshot.Hits+snapshot.Misses), + snapshot.HitRate, 1e-9) + assert.Len(t, snapshot.Policies, 2, "every arm should be reported") +} + +func TestTake_UnsampledTotalsAreRealTraffic(t *testing.T) { + cache := newCache(t) + drive(t, cache) + + snapshot := metrics.Take(cache) + stats := cache.Stats() + + assert.Equal(t, stats.Hits, snapshot.Hits, + "the headline counters must be the traffic actually served, not a sample") + assert.Equal(t, stats.Misses, snapshot.Misses) +} + +func TestTake_PoliciesAreOrderedBestFirst(t *testing.T) { + cache := newCache(t) + drive(t, cache) + + snapshot := metrics.Take(cache) + require.NotEmpty(t, snapshot.Policies) + + for i := 1; i < len(snapshot.Policies); i++ { + assert.GreaterOrEqual(t, snapshot.Policies[i-1].HitRate, snapshot.Policies[i].HitRate, + "policies should be ordered best hit rate first") + } + + assert.Equal(t, snapshot.Policies[0].Policy, snapshot.BestPolicy) +} + +func TestSnapshot_IsValidJSON(t *testing.T) { + cache := newCache(t) + drive(t, cache) + + var decoded map[string]any + require.NoError(t, json.Unmarshal([]byte(metrics.Take(cache).String()), &decoded)) + + for _, field := range []string{ + "active_policy", "epochs", "entries", "hits", "misses", + "hit_rate", "best_policy", "improvement", "policies", + } { + assert.Contains(t, decoded, field) + } +} + +func TestPublish_ExposesThroughExpvar(t *testing.T) { + cache := newCache(t) + drive(t, cache) + + name := "as_cache_test_" + strconv.FormatInt(time.Now().UnixNano(), 36) + require.NoError(t, metrics.Publish(name, cache)) + + published := expvar.Get(name) + require.NotNil(t, published, "the snapshot should appear in expvar") + + var decoded map[string]any + require.NoError(t, json.Unmarshal([]byte(published.String()), &decoded)) + assert.Equal(t, "LRU", decoded["active_policy"]) +} + +// TestPublish_RejectsDuplicateNames guards against expvar's own behaviour: +// publishing the same name twice panics, which would take a process down over +// a metrics registration mistake. +func TestPublish_RejectsDuplicateNames(t *testing.T) { + cache := newCache(t) + + name := "as_cache_dup_" + strconv.FormatInt(time.Now().UnixNano(), 36) + require.NoError(t, metrics.Publish(name, cache)) + + err := metrics.Publish(name, cache) + require.Error(t, err, "a duplicate publish must be an error, not a panic") + assert.Contains(t, err.Error(), "already published") +} + +// TestPublish_IsEvaluatedLazily checks that the published value reflects the +// cache when scraped rather than when registered. +func TestPublish_IsEvaluatedLazily(t *testing.T) { + cache := newCache(t) + + name := "as_cache_lazy_" + strconv.FormatInt(time.Now().UnixNano(), 36) + require.NoError(t, metrics.Publish(name, cache)) + + before := expvar.Get(name).String() + + drive(t, cache) + + after := expvar.Get(name).String() + + assert.NotEqual(t, before, after, + "the published value must be computed on scrape, so a dashboard sees live data") +} diff --git a/metrics/zzz_refute_probe_test.go b/metrics/zzz_refute_probe_test.go new file mode 100644 index 0000000..fb59a77 --- /dev/null +++ b/metrics/zzz_refute_probe_test.go @@ -0,0 +1,74 @@ +package metrics_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/metrics" + "github.com/sshaplygin/as-cache/policies" +) + +// TestProbe_IdleCacheSnapshot prints what an observe-only cache that has served +// NOTHING publishes, after several epochs have elapsed. +func TestProbe_IdleCacheSnapshot(t *testing.T) { + lru, err := policies.NewLRU[string, int](1000) + require.NoError(t, err) + twoQ, err := policies.NewTwoQueue[string, int](1000) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache[string, int]( + []ascache.Policy[string, int]{lru, twoQ}, nil, + &ascache.Settings{EpochDuration: 5 * time.Millisecond, ObserveOnly: true}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + // Snapshot BEFORE any epoch has run at all. + t.Logf("t=0 (no epoch yet): %s", metrics.Take(cache).String()) + + // Let several epochs elapse with zero traffic. + deadline := time.Now().Add(300 * time.Millisecond) + for time.Now().Before(deadline) { + if cache.Advice().Epochs > 3 { + break + } + } + + snap := metrics.Take(cache) + t.Logf("idle (epochs=%d): %s", snap.Epochs, snap.String()) + t.Logf("advice struct: %+v", cache.Advice()) + t.Logf("advice string:\n%s", cache.Advice().String()) + + // Is the "nothing measured" state distinguishable from a genuine 0%? + t.Logf("hits=%d misses=%d hits+misses=%d best=%q active=%q improvement=%v", + snap.Hits, snap.Misses, snap.Hits+snap.Misses, + snap.BestPolicy, snap.ActivePolicy, snap.Improvement) +} + +// TestProbe_GenuineZeroHitRate shows what a cache that genuinely misses on +// everything publishes, for comparison with the idle case. +func TestProbe_GenuineZeroHitRate(t *testing.T) { + lru, err := policies.NewLRU[string, int](10) + require.NoError(t, err) + twoQ, err := policies.NewTwoQueue[string, int](10) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache[string, int]( + []ascache.Policy[string, int]{lru, twoQ}, nil, + &ascache.Settings{EpochDuration: 5 * time.Millisecond, ObserveOnly: true}, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + // Every key distinct -> every Get is a miss. + for i := 0; i < 5000; i++ { + cache.Get(string(rune(i)) + "-never-stored") + } + + snap := metrics.Take(cache) + t.Logf("all-miss: %s", snap.String()) + t.Logf("hits=%d misses=%d hit_rate=%v", snap.Hits, snap.Misses, snap.HitRate) +} diff --git a/migration.go b/migration.go new file mode 100644 index 0000000..e5b86e8 --- /dev/null +++ b/migration.go @@ -0,0 +1,122 @@ +package ascache + +// migrateData transfers key/value pairs from the old active policy to the new +// one according to the configured MigrationStrategy. It always abandons any +// in-progress gradual window first. It must be called while the write lock is +// held. +// +// MigrationCold: purge stale shadow entries from target, start fresh. +// MigrationWarm: purge stale shadow entries from target, copy all key/value pairs. +// MigrationGradual: purge stale shadow entries from target, snapshot key list, +// and open the gradual migration window (unless the source is empty). +func (c *AdaptiveCache[K, V]) migrateData(from, to PolicyType) { + // Abandon any incomplete gradual migration from the previous epoch. + c.clearMigrationState() + + switch c.settings.MigrationStrategy { + case MigrationCold: + // Purge zero-value shadow entries so callers never observe a cached + // zero as if it were a real value. + c.policies[to].Purge() + return + + case MigrationWarm: + fromPolicy := c.policies[from] + toPolicy := c.policies[to] + + // Remove stale zero-value shadow entries so callers never observe a zero + // value as if it were a real cached result. + toPolicy.Purge() + + keys := fromPolicy.Keys() + for _, key := range keys { + val, ok := fromPolicy.Peek(key) + if !ok { + continue + } + toPolicy.Add(key, val) + } + + case MigrationGradual: + // Remove stale zero-value shadow entries from the new active policy. + c.policies[to].Purge() + + keys := c.policies[from].Keys() + if len(keys) == 0 { + // Nothing to migrate: opening an empty window would only force + // Gets through the write lock until something closed it. + return + } + realKeys := make(map[K]struct{}, len(keys)) + for _, k := range keys { + realKeys[k] = struct{}{} + } + + c.migrating = true + c.migrateFrom = from + c.migrationKeys = keys + c.migrationRealKeys = realKeys + } +} + +// clearMigrationState resets all gradual migration fields. It must be called +// while the write lock is held. +func (c *AdaptiveCache[K, V]) clearMigrationState() { + c.migrating = false + c.migrateFrom = Undefined + c.migrationKeys = nil + c.migrationRealKeys = nil +} + +// drainOneKey migrates one pending key from the migration source policy into +// the current active policy. It must be called while the write lock is held. +func (c *AdaptiveCache[K, V]) drainOneKey() { + for len(c.migrationKeys) > 0 { + // Pop from the end (O(1)). + key := c.migrationKeys[len(c.migrationKeys)-1] + c.migrationKeys = c.migrationKeys[:len(c.migrationKeys)-1] + + // Skip keys already promoted via Get or overwritten by a shadow Add. + if _, ok := c.migrationRealKeys[key]; !ok { + continue + } + + val, ok := c.policies[c.migrateFrom].Peek(key) + if !ok { + delete(c.migrationRealKeys, key) + continue + } + + c.policies[c.activePolicy].Add(key, val) + delete(c.migrationRealKeys, key) + + // Close the migration window when the last real key is drained. + if len(c.migrationRealKeys) == 0 { + c.closeMigrationLocked() + } + return + } + + // Queue exhausted with no promotable keys remaining. + c.closeMigrationLocked() +} + +// promoteLocked moves key from the migration source policy into the current +// active policy if it is still eligible: keys overwritten by a shadow Add or +// already promoted are skipped. It closes the migration window when no +// eligible keys remain, whichever path emptied the set (promotion here or an +// earlier Remove). It must be called while the write lock is held during a +// gradual migration window. +func (c *AdaptiveCache[K, V]) promoteLocked(key K) { + // Skip keys whose values have been overwritten by a shadow Add. + if _, ok := c.migrationRealKeys[key]; ok { + if val, ok := c.policies[c.migrateFrom].Peek(key); ok { + c.policies[c.activePolicy].Add(key, val) + } + delete(c.migrationRealKeys, key) + } + + if len(c.migrationRealKeys) == 0 { + c.closeMigrationLocked() + } +} diff --git a/models.go b/models.go index 2d7253e..78a9225 100644 --- a/models.go +++ b/models.go @@ -4,9 +4,30 @@ package ascache type PolicyType uint const ( + // Undefined is the zero value and names no policy. Undefined PolicyType = iota + // LRU evicts the least recently used entry. LRU + // LFU evicts the least frequently used entry. LFU + // TwoQueue evicts using the 2Q algorithm, which keeps a small recent-access + // queue in front of a frequently-accessed queue so a scan cannot flush the + // working set. + TwoQueue + // ARC evicts using Adaptive Replacement Cache, which balances recency + // against frequency on its own. The algorithm is patented by IBM, so its + // adapter lives in a separate module that nothing else depends on. + ARC + // Random evicts an arbitrary entry. It is a useful control arm: a policy + // that cannot beat random on a workload is not earning its bookkeeping. + Random + // TTL evicts by expiry as well as by recency. + TTL + // TinyLFU evicts using the W-TinyLFU family, which gates admission on a + // frequency sketch so a new key must earn its place against the entry it + // would displace. It is the strongest general-purpose baseline in wide + // use, and the one an adaptive cache has to beat to justify itself. + TinyLFU ) // MigrationStrategy controls how key/value pairs are transferred when the @@ -25,9 +46,17 @@ const ( MigrationWarm // MigrationGradual lazily drains the old active policy into the new one. - // Each Get() miss attempts to promote the key from the old policy; each - // Add() call migrates one additional key. The migration window closes at - // the next epoch boundary, on Purge(), or when all keys have been drained. + // During the window each Get() promotes the requested key from the old + // policy into the new active — when it is still eligible (not overwritten + // by a shadow Add, already promoted, or evicted from the source) — before + // the lookup is counted, so served requests register as hits; each Add() + // call migrates at most one additional key. While the window is open, + // Get() takes the write lock, serializing reads. + // + // The window closes when no eligible keys remain, on Purge(), on the next + // policy switch, and in any case at the next epoch boundary - a workload + // that simply stops touching the pending keys must not leave it open + // forever, holding the source at full capacity with its values retained. MigrationGradual ) @@ -42,7 +71,12 @@ type PolicyStats struct { Misses int64 } -// ShadowStats holds the hit/miss result of a shadow cache sensor for one epoch. +// ShadowStats holds one policy's hit/miss counts since its last report — +// normally one epoch, or several when reporting was skipped because the cache +// was not yet full (EvictPartialCapacityFilling=false). Every policy reports +// through this channel on each reporting epoch, the active policy included, +// so the bandit's posterior for the active arm does not go stale while +// reports flow. type ShadowStats struct { Policy PolicyType Hits int64 diff --git a/policies/adapt.go b/policies/adapt.go new file mode 100644 index 0000000..e33a38f --- /dev/null +++ b/policies/adapt.go @@ -0,0 +1,260 @@ +package policies + +import ( + "fmt" + "sync" +) + +// PartialCacher is the shape a cache library commonly ships: lookups and +// mutations, but no eviction or presence reporting and no way to change +// capacity after construction. Both hashicorp's 2Q and ARC caches have exactly +// this shape. +type PartialCacher[K comparable, V any] interface { + Add(key K, value V) + Get(key K) (value V, ok bool) + Peek(key K) (value V, ok bool) + Contains(key K) bool + Remove(key K) + Purge() + Keys() []K + Values() []V + Len() int +} + +// AdaptedCache turns a PartialCacher into a full ascache.Cacher. +// +// It supplies the three things the underlying cache does not: Add reports +// whether it evicted, Remove reports whether the key was present, and Resize +// changes capacity by rebuilding the cache at the new size and replaying the +// entries that fit. +// +// The underlying cache is held behind this type's mutex rather than embedded, +// because Resize replaces it wholesale; an operation that landed on the +// outgoing instance would simply be lost. +type AdaptedCache[K comparable, V any] struct { + mu sync.RWMutex + cache PartialCacher[K, V] + size int + build func(size int) (PartialCacher[K, V], error) +} + +// Adapt wraps a cache built by build, which must return a cache of the +// capacity it is given. build is called again on every Resize. +func Adapt[K comparable, V any]( + size int, + build func(size int) (PartialCacher[K, V], error), +) (*AdaptedCache[K, V], error) { + if build == nil { + return nil, fmt.Errorf("adapt cache: build function must not be nil") + } + + cache, err := build(size) + if err != nil { + return nil, fmt.Errorf("adapt cache: %w", err) + } + + return &AdaptedCache[K, V]{cache: cache, size: size, build: build}, nil +} + +// Add stores a value, reporting whether storing it evicted another entry. +// +// The underlying cache does not report evictions, so this infers one: an +// insert of a key the cache did not already hold, made while the cache is +// full, must have evicted something to make room. +func (c *AdaptedCache[K, V]) Add(key K, value V) bool { + c.mu.Lock() + defer c.mu.Unlock() + + if c.size <= 0 { + // A cache of zero capacity holds nothing. The underlying cache cannot + // be rebuilt at size zero - both 2Q and ARC reject a non-positive size + // - so the entry is refused here instead. Nothing was evicted to make + // room, because nothing was stored. + return false + } + + evicts := !c.cache.Contains(key) && c.cache.Len() >= c.size + c.cache.Add(key, value) + c.enforceCapacityLocked() + + return evicts +} + +// enforceCapacityLocked trims the cache to the configured size. +// +// Normally the underlying cache enforces its own capacity and this does +// nothing. It matters after a Resize whose rebuild failed: the old instance is +// still in use at its original capacity while the configured size is smaller, +// and without this the cache would hold far more than the caller asked for +// while Cap reported the smaller number. +func (c *AdaptedCache[K, V]) enforceCapacityLocked() { + for c.cache.Len() > c.size { + keys := c.cache.Keys() + if len(keys) == 0 { + return + } + c.cache.Remove(keys[0]) + } +} + +// Get returns the value for key, if present, and records the access. +func (c *AdaptedCache[K, V]) Get(key K) (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Get(key) +} + +// Peek returns the value for key without recording an access. +func (c *AdaptedCache[K, V]) Peek(key K) (V, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cache.Peek(key) +} + +// Contains reports whether key is cached, without recording an access. +func (c *AdaptedCache[K, V]) Contains(key K) bool { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cache.Contains(key) +} + +// Remove deletes key, reporting whether it was present. +func (c *AdaptedCache[K, V]) Remove(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + present := c.cache.Contains(key) + c.cache.Remove(key) + + return present +} + +// Purge empties the cache. +func (c *AdaptedCache[K, V]) Purge() { + c.mu.Lock() + defer c.mu.Unlock() + + c.cache.Purge() +} + +// Keys returns the cached keys, oldest first. +func (c *AdaptedCache[K, V]) Keys() []K { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cache.Keys() +} + +// Values returns the cached values, in the same order as Keys. +func (c *AdaptedCache[K, V]) Values() []V { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cache.Values() +} + +// Len returns the number of cached entries. +func (c *AdaptedCache[K, V]) Len() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cache.Len() +} + +// Cap returns the capacity. +func (c *AdaptedCache[K, V]) Cap() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.size +} + +// Resize changes the capacity to size and returns the number of entries +// evicted to reach it. +// +// The underlying cache cannot be resized, so this rebuilds it at the new size +// and replays every entry, letting the rebuilt cache evict down to capacity. +// Whatever internal adaptation the algorithm had accumulated - 2Q's queue +// split, ARC's recency/frequency balance and ghost lists - is lost, and the +// rebuilt cache has to relearn it. Until it has, the policy's measured hit +// rate understates the algorithm, so resizing such a policy repeatedly would +// keep it permanently unadapted. Which entries survive a shrink is not +// meaningful either; see the note in the body. +// +// A size of zero or less empties the cache and holds nothing. +func (c *AdaptedCache[K, V]) Resize(size int) int { + c.mu.Lock() + defer c.mu.Unlock() + + if size < 0 { + size = 0 + } + if size == c.size { + return 0 + } + + before := c.cache.Len() + + if size == 0 { + c.cache.Purge() + c.size = 0 + + return before + } + + // Every entry is replayed and the rebuilt cache evicts down to the new + // capacity itself, rather than this code selecting survivors up front. + // + // There is no correct selection to make. Keys() carries no consistent + // meaning across the caches this adapter serves: 2Q returns its frequent + // queue followed by its recent one, ARC returns its recent list followed + // by its frequent one - opposite groupings, and neither is one global + // recency order. Any "keep the tail" or "keep the head" rule is therefore + // right for one and precisely backwards for the other. + // + // So which entries survive a shrink is not meaningful, and callers should + // not rely on it. That is part of the same cost as losing the algorithm's + // learned state, and a good reason to prefer a natively resizable policy + // as an arm where one exists. + keys := c.cache.Keys() + + type entry struct { + key K + value V + } + kept := make([]entry, 0, len(keys)) + for _, key := range keys { + value, ok := c.cache.Peek(key) + if !ok { + continue + } + kept = append(kept, entry{key: key, value: value}) + } + + rebuilt, err := c.build(size) + if err != nil { + // No cache of the new size could be built - hashicorp's 2Q, for one, + // rejects a size of 1 because its ghost queues round down to zero. + // Keep the existing instance but enforce the requested capacity on it + // anyway, so Cap and behaviour still agree. Reporting the new size + // while silently retaining the old capacity would let the cache hold + // far more than the caller asked for. + c.size = size + before := c.cache.Len() + c.enforceCapacityLocked() + + return before - c.cache.Len() + } + + for _, e := range kept { + rebuilt.Add(e.key, e.value) + } + + c.cache = rebuilt + c.size = size + + return before - rebuilt.Len() +} diff --git a/policies/adapters.go b/policies/adapters.go new file mode 100644 index 0000000..1f4be7f --- /dev/null +++ b/policies/adapters.go @@ -0,0 +1,82 @@ +package policies + +import ( + "fmt" + "time" + + lru "github.com/hashicorp/golang-lru/v2" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/lfu" +) + +// NewLRU returns an LRU policy of the given size, backed by +// hashicorp/golang-lru/v2. +func NewLRU[K comparable, V any](size int) (ascache.Policy[K, V], error) { + cache, err := lru.New[K, V](size) + if err != nil { + return nil, fmt.Errorf("build lru cache: %w", err) + } + + return ascache.NewCache[K, V](cache, ascache.LRU, size), nil +} + +// NewLFU returns an LFU policy of the given size, backed by this repository's +// own O(1) LFU implementation. +// +// LFU evicts the least frequently used entry, which makes it strong where +// popularity is stable and skewed, and weak where it shifts: an entry that was +// hot once accumulates a count that keeps it resident long after the traffic +// has moved on. +func NewLFU[K comparable, V any](size int) (ascache.Policy[K, V], error) { + cache, err := lfu.New[K, V](size) + if err != nil { + return nil, fmt.Errorf("build lfu cache: %w", err) + } + + return ascache.NewCache[K, V](cache, ascache.LFU, size), nil +} + +// NewTwoQueue returns a 2Q policy of the given size, backed by +// hashicorp/golang-lru/v2. +// +// 2Q puts a small recent-access queue in front of a frequently-accessed queue, +// so a one-off scan passes through the recent queue without flushing the +// working set. That makes it a useful arm to hold alongside LRU, which a scan +// defeats completely. +// 2Q reports neither evictions nor removals and cannot be resized, so it is +// adapted rather than used directly. +func NewTwoQueue[K comparable, V any](size int) (ascache.Policy[K, V], error) { + cache, err := Adapt[K, V](size, func(size int) (PartialCacher[K, V], error) { + built, err := lru.New2Q[K, V](size) + if err != nil { + return nil, fmt.Errorf("build 2q cache: %w", err) + } + + return built, nil + }) + if err != nil { + return nil, err + } + + return ascache.NewCache[K, V](cache, ascache.TwoQueue, size), nil +} + +// NewTTL returns a policy that evicts by expiry as well as by recency, backed +// by hashicorp/golang-lru/v2/expirable. Entries older than ttl are evicted +// regardless of use; a ttl of zero disables expiry, leaving plain LRU +// behaviour. +// +// Note that this policy's hit rate depends on wall-clock time, not only on the +// access pattern. As a shadow it is therefore measuring something the other +// arms are not, which is the point when the workload has genuinely stale data, +// and misleading when it does not. +func NewTTL[K comparable, V any](size int, ttl time.Duration) ascache.Policy[K, V] { + return ascache.NewCache[K, V](NewTTLCache[K, V](size, ttl), ascache.TTL, size) +} + +// NewRandomPolicy returns a random-eviction policy of the given size, ready to +// be used as a bandit arm. +func NewRandomPolicy[K comparable, V any](size int) ascache.Policy[K, V] { + return ascache.NewCache[K, V](NewRandom[K, V](size), ascache.Random, size) +} diff --git a/policies/arc/arc.go b/policies/arc/arc.go new file mode 100644 index 0000000..fad018b --- /dev/null +++ b/policies/arc/arc.go @@ -0,0 +1,50 @@ +// Package arc adapts the Adaptive Replacement Cache to ascache.Policy. +// +// It is a module of its own, separate from the other policy adapters, for one +// reason: ARC is patented by IBM (US 6,996,676, filed 2002). Upstream +// hashicorp/golang-lru made the same split in v2 so that its main module +// carries no patented algorithm, and this package preserves that property for +// as-cache. Importing github.com/sshaplygin/as-cache/policies never pulls ARC +// into a build; only importing this package does. +// +// Whether the patent still restricts anything is a question for the adopter +// and their counsel, not for this comment. The isolation exists so that the +// choice is explicit and never made by accident. +package arc + +import ( + "fmt" + + arclru "github.com/hashicorp/golang-lru/arc/v2" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/policies" +) + +// New returns an ARC cache holding up to size entries, satisfying +// ascache.Cacher. +// +// Upstream ARCCache reports neither evictions nor removals and has no Resize, +// so it is wrapped by policies.Adapt, which supplies all three. See +// policies.AdaptedCache.Resize for what resizing costs an adaptive algorithm. +func New[K comparable, V any](size int) (*policies.AdaptedCache[K, V], error) { + return policies.Adapt[K, V](size, func(size int) (policies.PartialCacher[K, V], error) { + cache, err := arclru.NewARC[K, V](size) + if err != nil { + return nil, fmt.Errorf("build arc cache: %w", err) + } + + return cache, nil + }) +} + +// NewPolicy returns an ARC policy of the given size, ready to be used as a +// bandit arm. +func NewPolicy[K comparable, V any](size int) (ascache.Policy[K, V], error) { + cache, err := New[K, V](size) + if err != nil { + return nil, err + } + + return ascache.NewCache[K, V](cache, ascache.ARC, size), nil +} diff --git a/policies/arc/arc_test.go b/policies/arc/arc_test.go new file mode 100644 index 0000000..22025ed --- /dev/null +++ b/policies/arc/arc_test.go @@ -0,0 +1,264 @@ +package arc_test + +import ( + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/policies/arc" +) + +func newARC(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + + p, err := arc.NewPolicy[string, int](size) + require.NoError(t, err) + + return p +} + +// TestARCConformance mirrors the conformance suite the other policies run, +// since ARC lives in its own module and cannot share that file. +func TestARCConformance(t *testing.T) { + t.Run("stores and retrieves", func(t *testing.T) { + p := newARC(t, 10) + p.Add("a", 1) + + got, ok := p.Get("a") + require.True(t, ok) + assert.Equal(t, 1, got) + }) + + t.Run("reports a miss for an absent key", func(t *testing.T) { + p := newARC(t, 10) + + got, ok := p.Get("nope") + assert.False(t, ok) + assert.Zero(t, got) + }) + + t.Run("overwrites without growing", func(t *testing.T) { + p := newARC(t, 10) + p.Add("a", 1) + p.Add("a", 2) + + got, ok := p.Get("a") + require.True(t, ok) + assert.Equal(t, 2, got) + assert.Equal(t, 1, p.Len()) + }) + + t.Run("never exceeds capacity", func(t *testing.T) { + const size = 10 + p := newARC(t, size) + + for i := 0; i < size*5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + assert.LessOrEqual(t, p.Len(), size) + assert.Equal(t, size, p.Cap()) + }) + + t.Run("Remove reports presence", func(t *testing.T) { + p := newARC(t, 10) + p.Add("a", 1) + + assert.True(t, p.Remove("a"), "removing a present key must report true") + assert.False(t, p.Remove("a"), "removing an absent key must report false") + }) + + t.Run("Add reports eviction only when full", func(t *testing.T) { + p := newARC(t, 2) + + assert.False(t, p.Add("a", 1), "adding into a cache with room must not report an eviction") + assert.False(t, p.Add("b", 2)) + assert.False(t, p.Add("a", 3), "overwriting must not report an eviction") + assert.True(t, p.Add("c", 4), "adding into a full cache must report an eviction") + }) + + t.Run("Purge empties", func(t *testing.T) { + p := newARC(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Purge() + assert.Zero(t, p.Len()) + }) + + t.Run("Resize shrinks and Cap follows", func(t *testing.T) { + p := newARC(t, 20) + for i := 0; i < 20; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(5) + + assert.LessOrEqual(t, p.Len(), 5, "Resize must enforce the new capacity") + assert.Equal(t, 5, p.Cap(), "Cap must follow Resize - the adaptive layer relies on it") + }) + + t.Run("Resize retains entries and leaves survivors intact", func(t *testing.T) { + p := newARC(t, 10) + for i := 0; i < 10; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(3) + + // Which entries survive is not guaranteed - ARC's Keys() is recent + // then frequent, not one recency order, so the rebuilt cache chooses. + // What must hold is that the shrink retains what it can and that no + // survivor comes back corrupted. + assert.Equal(t, 3, p.Len(), "a shrink from 10 to 3 must retain 3 entries, not fewer") + + survivors := 0 + for i := 0; i < 10; i++ { + key := "key-" + strconv.Itoa(i) + if got, ok := p.Peek(key); ok { + assert.Equal(t, i, got, "%s survived the resize with a corrupted value", key) + survivors++ + } + } + assert.Equal(t, 3, survivors, "Len must agree with what is actually retrievable") + }) + + t.Run("Resize grows without losing data", func(t *testing.T) { + p := newARC(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(100) + + assert.Equal(t, 5, p.Len(), "growing must not evict") + assert.Equal(t, 100, p.Cap()) + }) + + t.Run("Resize to zero empties and refuses new entries", func(t *testing.T) { + p := newARC(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(0) + assert.Zero(t, p.Len(), "a zero-capacity policy must hold nothing") + + p.Add("x", 1) + assert.Zero(t, p.Len(), "a zero-capacity policy must not accept entries") + }) + + t.Run("tracks hits and misses", func(t *testing.T) { + p := newARC(t, 10) + p.Add("a", 1) + p.ResetStats() + + p.Get("a") + p.Get("absent") + + stats := p.GetStats() + assert.Equal(t, int64(1), stats.Hits) + assert.Equal(t, int64(1), stats.Misses) + }) + + t.Run("is safe under concurrent use, including resizes", func(t *testing.T) { + p := newARC(t, 100) + + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func(seed int) { + defer wg.Done() + for i := 0; i < 200; i++ { + key := "key-" + strconv.Itoa((seed+i)%150) + p.Add(key, i) + p.Get(key) + _ = p.Len() + if i%50 == 0 { + _ = p.Keys() + } + } + }(g) + } + + // Resize concurrently: it swaps the underlying cache wholesale, which + // is exactly the operation most likely to race with the workers. + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 20; i++ { + p.Resize(50 + i%100) + } + }() + + wg.Wait() + }) + + t.Run("reports the ARC policy type", func(t *testing.T) { + assert.Equal(t, ascache.ARC, newARC(t, 10).GetType()) + assert.Equal(t, "ARC", ascache.ARC.String()) + }) +} + +// TestARCDrivesAnAdaptiveCache checks ARC works as a real bandit arm, and in +// particular that a caller never sees a value that was never stored while the +// cache switches between arms. +func TestARCDrivesAnAdaptiveCache(t *testing.T) { + const size = 500 + + arcPolicy, err := arc.NewPolicy[string, int](size) + require.NoError(t, err) + + // ARC is the only arm here: a second ARC instance would report the same + // PolicyType and the constructor rejects that collision by design. + cache, err := ascache.NewAdaptiveCache( + []ascache.Policy[string, int]{arcPolicy}, + &fixedBandit{pick: ascache.ARC}, + &ascache.Settings{ + EpochDuration: time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: ascache.MigrationWarm, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + for i := 0; i < size; i++ { + cache.Add("key-"+strconv.Itoa(i), i) + } + + deadline := time.Now().Add(100 * time.Millisecond) + for time.Now().Before(deadline) { + for i := 0; i < 100; i++ { + key := "key-" + strconv.Itoa(i) + if got, ok := cache.Get(key); ok { + require.Equal(t, i, got, "key %q returned a value that was never stored", key) + } + } + } + + assert.Equal(t, ascache.ARC, cache.ActivePolicy()) +} + +// TestARCPolicyTypeIsDistinctFromTheOthers guards the enum wiring: ARC must +// not collide with a policy from the sibling module, or the two could not be +// used as arms of the same cache. +func TestARCPolicyTypeIsDistinctFromTheOthers(t *testing.T) { + for _, other := range []ascache.PolicyType{ + ascache.Undefined, ascache.LRU, ascache.LFU, + ascache.TwoQueue, ascache.Random, ascache.TTL, + } { + assert.NotEqual(t, other, ascache.ARC, "ARC must have its own PolicyType") + } +} + +type fixedBandit struct{ pick ascache.PolicyType } + +func (b *fixedBandit) RecordStats(_ ascache.ShadowStats) {} +func (b *fixedBandit) SelectPolicy() ascache.PolicyType { return b.pick } diff --git a/policies/arc/go.mod b/policies/arc/go.mod new file mode 100644 index 0000000..c034e2c --- /dev/null +++ b/policies/arc/go.mod @@ -0,0 +1,27 @@ +module github.com/sshaplygin/as-cache/policies/arc + +go 1.25.2 + +// golang-lru is pinned to v2.0.6 deliberately: see the note in +// ../go.mod. The v2.0.7 release of the base module does not build. +require ( + github.com/hashicorp/golang-lru/arc/v2 v2.0.6 + github.com/sshaplygin/as-cache v0.0.0 + github.com/sshaplygin/as-cache/lfu v0.0.0 // indirect + github.com/sshaplygin/as-cache/policies v0.0.0 +) + +require github.com/stretchr/testify v1.11.1 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.6 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/sshaplygin/as-cache => ../.. + +replace github.com/sshaplygin/as-cache/policies => .. + +replace github.com/sshaplygin/as-cache/lfu => ../../lfu diff --git a/policies/arc/go.sum b/policies/arc/go.sum new file mode 100644 index 0000000..af33fb7 --- /dev/null +++ b/policies/arc/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/golang-lru/arc/v2 v2.0.6 h1:4NU7uP5vSoK6TbaMj3NtY478TTAWLso/vL1gpNrInHg= +github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/policies/conformance_test.go b/policies/conformance_test.go new file mode 100644 index 0000000..d6a2ae4 --- /dev/null +++ b/policies/conformance_test.go @@ -0,0 +1,352 @@ +package policies_test + +import ( + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/policies" +) + +// newPolicy builds a policy of the given capacity for the conformance suite. +type newPolicy func(t *testing.T, size int) ascache.Policy[string, int] + +// policiesUnderTest is every policy this module provides. ARC is absent by +// design: it lives in its own module and runs the same suite there. +var policiesUnderTest = map[string]newPolicy{ + "lru": func(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + p, err := policies.NewLRU[string, int](size) + require.NoError(t, err) + + return p + }, + "lfu": func(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + p, err := policies.NewLFU[string, int](size) + require.NoError(t, err) + + return p + }, + "2q": func(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + p, err := policies.NewTwoQueue[string, int](size) + require.NoError(t, err) + + return p + }, + "ttl": func(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + + // A long TTL keeps expiry out of the way: this suite is about the + // Cacher contract, not about expiry behaviour. + return policies.NewTTL[string, int](size, time.Hour) + }, + "random": func(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + + return policies.NewRandomPolicy[string, int](size) + }, +} + +// TestPolicyConformance runs the Cacher/Policy contract against every policy. +// Anything an AdaptiveCache relies on belongs here, because a policy that +// breaks one of these is a policy the adaptive layer will mis-drive. +func TestPolicyConformance(t *testing.T) { + for name, build := range policiesUnderTest { + t.Run(name, func(t *testing.T) { + runConformance(t, build) + }) + } +} + +func runConformance(t *testing.T, build newPolicy) { + t.Helper() + + t.Run("stores and retrieves", func(t *testing.T) { + p := build(t, 10) + p.Add("a", 1) + + got, ok := p.Get("a") + require.True(t, ok, "a stored key must be retrievable") + assert.Equal(t, 1, got) + }) + + t.Run("reports a miss for an absent key", func(t *testing.T) { + p := build(t, 10) + + got, ok := p.Get("nope") + assert.False(t, ok) + assert.Zero(t, got, "a miss must return the zero value") + }) + + t.Run("overwrites without growing", func(t *testing.T) { + p := build(t, 10) + p.Add("a", 1) + p.Add("a", 2) + + got, ok := p.Get("a") + require.True(t, ok) + assert.Equal(t, 2, got, "re-adding a key must overwrite it") + assert.Equal(t, 1, p.Len(), "re-adding a key must not add an entry") + }) + + t.Run("never exceeds capacity", func(t *testing.T) { + const size = 10 + p := build(t, size) + + for i := 0; i < size*5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + assert.LessOrEqual(t, p.Len(), size, "a policy must not exceed its capacity") + assert.Equal(t, size, p.Cap(), "Cap must report the configured capacity") + }) + + t.Run("Peek does not report a miss for a live key", func(t *testing.T) { + p := build(t, 10) + p.Add("a", 42) + + got, ok := p.Peek("a") + require.True(t, ok) + assert.Equal(t, 42, got) + }) + + t.Run("Contains agrees with Get", func(t *testing.T) { + p := build(t, 10) + p.Add("a", 1) + + assert.True(t, p.Contains("a")) + assert.False(t, p.Contains("b")) + }) + + t.Run("Remove reports presence", func(t *testing.T) { + p := build(t, 10) + p.Add("a", 1) + + assert.True(t, p.Remove("a"), "removing a present key must report true") + assert.False(t, p.Contains("a")) + assert.False(t, p.Remove("a"), "removing an absent key must report false") + }) + + t.Run("Purge empties", func(t *testing.T) { + p := build(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Purge() + assert.Zero(t, p.Len()) + assert.Empty(t, p.Keys()) + }) + + t.Run("Keys and Values agree with Len", func(t *testing.T) { + p := build(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + assert.Len(t, p.Keys(), p.Len()) + assert.Len(t, p.Values(), p.Len()) + }) + + t.Run("Resize shrinks to the new capacity", func(t *testing.T) { + p := build(t, 20) + for i := 0; i < 20; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + require.Positive(t, p.Len(), "expected entries before resizing") + + p.Resize(5) + + assert.LessOrEqual(t, p.Len(), 5, "Resize must enforce the new capacity") + assert.Equal(t, 5, p.Cap(), "Cap must follow Resize - the adaptive layer relies on it") + }) + + t.Run("Resize grows without losing data", func(t *testing.T) { + p := build(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + before := p.Len() + + p.Resize(100) + + assert.Equal(t, before, p.Len(), "growing must not evict") + assert.Equal(t, 100, p.Cap()) + }) + + t.Run("Resize to zero empties", func(t *testing.T) { + p := build(t, 10) + for i := 0; i < 5; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(0) + + assert.Zero(t, p.Len(), "a zero-capacity policy must hold nothing") + + // Adding to a zero-capacity policy must not panic or retain anything. + p.Add("x", 1) + assert.Zero(t, p.Len()) + }) + + t.Run("tracks hits and misses", func(t *testing.T) { + p := build(t, 10) + p.Add("a", 1) + p.ResetStats() + + p.Get("a") + p.Get("absent") + + stats := p.GetStats() + assert.Equal(t, int64(1), stats.Hits) + assert.Equal(t, int64(1), stats.Misses) + + p.ResetStats() + assert.Equal(t, ascache.PolicyStats{}, p.GetStats()) + }) + + t.Run("is safe under concurrent use", func(t *testing.T) { + p := build(t, 100) + + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func(seed int) { + defer wg.Done() + for i := 0; i < 200; i++ { + key := "key-" + strconv.Itoa((seed+i)%150) + p.Add(key, i) + p.Get(key) + p.Contains(key) + _ = p.Len() + if i%20 == 0 { + _ = p.Keys() + } + if i%50 == 0 { + p.Remove(key) + } + } + }(g) + } + wg.Wait() + + assert.LessOrEqual(t, p.Len(), 100) + }) +} + +// TestPolicyTypesAreDistinct guards the wiring: two arms reporting the same +// PolicyType would collide in AdaptiveCache's policy map, and the constructor +// rejects that. +func TestPolicyTypesAreDistinct(t *testing.T) { + seen := map[ascache.PolicyType]string{} + for name, build := range policiesUnderTest { + p := build(t, 10) + policyType := p.GetType() + + if other, dup := seen[policyType]; dup { + assert.Fail(t, "duplicate PolicyType", + "%s and %s both report %s", name, other, policyType) + } + seen[policyType] = name + + assert.NotEqual(t, ascache.Undefined, policyType, + "%s must report a defined PolicyType", name) + } +} + +// TestPoliciesDriveAnAdaptiveCache is the integration check: every policy this +// module provides must work as an arm of a real AdaptiveCache, including +// through a switch. +func TestPoliciesDriveAnAdaptiveCache(t *testing.T) { + const size = 1000 + + lruPolicy, err := policies.NewLRU[string, int](size) + require.NoError(t, err) + twoQ, err := policies.NewTwoQueue[string, int](size) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache( + []ascache.Policy[string, int]{ + lruPolicy, + twoQ, + policies.NewRandomPolicy[string, int](size), + policies.NewTTL[string, int](size, time.Hour), + }, + &alternatingBandit{}, + &ascache.Settings{ + EpochDuration: time.Millisecond, + EvictPartialCapacityFilling: true, + MigrationStrategy: ascache.MigrationWarm, + ShadowSampleRate: 0.05, + MinShadowCapacity: 16, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + for i := 0; i < size; i++ { + cache.Add("key-"+strconv.Itoa(i), i) + } + + // Let several epochs elapse so the arms rotate through active duty. + seenActive := map[ascache.PolicyType]struct{}{} + served := 0 + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + for i := 0; i < 100; i++ { + seenActive[cache.ActivePolicy()] = struct{}{} + key := "key-" + strconv.Itoa(i) + if got, ok := cache.Get(key); ok { + require.Equal(t, i, got, + "key %q returned a value that was never stored - a shadow zero leaked", key) + served++ + } + } + } + + // Without these the loop above proves nothing: an always-missing cache + // satisfies every assertion in it vacuously, and a cache that never + // switches never exercises migration between arms at all. + assert.Positive(t, served, "the cache must actually serve hits for the value check to mean anything") + assert.Greater(t, len(seenActive), 1, + "the bandit must rotate the active policy so migration between arms is exercised, saw %v", seenActive) +} + +// alternatingBandit cycles through the arms so every policy takes a turn as +// the active one. +type alternatingBandit struct { + mu sync.Mutex + n int + arms []ascache.PolicyType +} + +func (b *alternatingBandit) RecordStats(stats ascache.ShadowStats) { + b.mu.Lock() + defer b.mu.Unlock() + + for _, arm := range b.arms { + if arm == stats.Policy { + return + } + } + b.arms = append(b.arms, stats.Policy) +} + +func (b *alternatingBandit) SelectPolicy() ascache.PolicyType { + b.mu.Lock() + defer b.mu.Unlock() + + if len(b.arms) == 0 { + return ascache.Undefined + } + b.n++ + + return b.arms[b.n%len(b.arms)] +} diff --git a/policies/go.mod b/policies/go.mod new file mode 100644 index 0000000..a0a81f4 --- /dev/null +++ b/policies/go.mod @@ -0,0 +1,26 @@ +module github.com/sshaplygin/as-cache/policies + +go 1.25.2 + +// golang-lru is pinned to v2.0.6 deliberately. In v2.0.7 the published module's +// simplelru package imports github.com/hashicorp/golang-lru/v2/simplelru/internal, +// which does not exist in that module (only a top-level internal/ does), so the +// release does not build. Verified against the checksum database, so it is an +// upstream defect rather than a local cache problem. +require ( + github.com/hashicorp/golang-lru/v2 v2.0.6 + github.com/sshaplygin/as-cache v0.0.0 + github.com/stretchr/testify v1.11.1 +) + +require github.com/sshaplygin/as-cache/lfu v0.0.0-00010101000000-000000000000 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/sshaplygin/as-cache => .. + +replace github.com/sshaplygin/as-cache/lfu => ../lfu diff --git a/policies/go.sum b/policies/go.sum new file mode 100644 index 0000000..dc21af3 --- /dev/null +++ b/policies/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/policies/random.go b/policies/random.go new file mode 100644 index 0000000..c845401 --- /dev/null +++ b/policies/random.go @@ -0,0 +1,245 @@ +// Package policies provides ready-made Policy implementations for +// AdaptiveCache, adapting well-known cache libraries to the ascache.Cacher +// interface so they can be used as bandit arms without writing glue. +// +// The ARC policy is deliberately absent: the algorithm is patented by IBM and +// its adapter lives in the separate github.com/sshaplygin/as-cache/policies/arc +// module, so importing this package never pulls a patented implementation into +// a build. +package policies + +import ( + "math/rand/v2" + "sync" + + ascache "github.com/sshaplygin/as-cache" +) + +// RandomCache evicts an arbitrary entry when it is full. +// +// Random eviction is worth having as a bandit arm precisely because it is the +// null hypothesis: it carries no bookkeeping and makes no assumption about the +// workload, so a policy that cannot beat it is not paying for itself. On a +// workload with no reuse structure - a uniform random or a pure scan - it also +// happens to be competitive with far more elaborate policies. +// +// It is safe for concurrent use. +type RandomCache[K comparable, V any] struct { + mu sync.Mutex + data map[K]V + // keys holds every key currently in data, and index maps a key to its slot + // in keys. Together they make "pick a uniformly random key" and "remove a + // key" both O(1): removal swaps the last key into the freed slot. + keys []K + index map[K]int + size int + rng *rand.Rand +} + +// NewRandom returns a random-eviction cache holding up to size entries. +// A size of zero or less means the cache holds nothing. +func NewRandom[K comparable, V any](size int) *RandomCache[K, V] { + if size < 0 { + size = 0 + } + + return &RandomCache[K, V]{ + data: make(map[K]V, size), + keys: make([]K, 0, size), + index: make(map[K]int, size), + size: size, + //nolint:gosec // Eviction choice is not a security decision; a cheap + // non-cryptographic source is the right one here. + rng: rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64())), + } +} + +// trackLocked records a newly inserted key. +func (c *RandomCache[K, V]) trackLocked(key K) { + c.index[key] = len(c.keys) + c.keys = append(c.keys, key) +} + +// untrackLocked removes a key from the tracking structures in O(1) by swapping +// the last key into the vacated slot. +func (c *RandomCache[K, V]) untrackLocked(key K) { + slot, ok := c.index[key] + if !ok { + return + } + + last := len(c.keys) - 1 + if slot != last { + moved := c.keys[last] + c.keys[slot] = moved + c.index[moved] = slot + } + + c.keys = c.keys[:last] + delete(c.index, key) +} + +// removeLocked deletes a key and its tracking entry. +func (c *RandomCache[K, V]) removeLocked(key K) bool { + if _, ok := c.data[key]; !ok { + return false + } + + delete(c.data, key) + c.untrackLocked(key) + + return true +} + +// evictLocked drops random entries until the cache is within capacity, +// returning how many it removed. +func (c *RandomCache[K, V]) evictLocked() int { + evicted := 0 + for len(c.keys) > c.size { + victim := c.keys[c.rng.IntN(len(c.keys))] + c.removeLocked(victim) + evicted++ + } + + return evicted +} + +// Add stores a value, reporting whether storing it evicted another entry. +// +// When the cache is full, room is made before the new entry is inserted, so +// the victim is drawn from the entries that were already resident. Inserting +// first and then evicting would put the caller's own write into the draw and +// discard it with probability 1/(size+1) - accepting a value and losing it +// before the next read, which no other policy here does. +func (c *RandomCache[K, V]) Add(key K, value V) bool { + c.mu.Lock() + defer c.mu.Unlock() + + if _, exists := c.data[key]; exists { + c.data[key] = value + + return false + } + + if c.size <= 0 { + // A cache of zero capacity holds nothing, and nothing was evicted to + // make room, because nothing was stored. + return false + } + + evicted := 0 + for len(c.keys) >= c.size { + victim := c.keys[c.rng.IntN(len(c.keys))] + c.removeLocked(victim) + evicted++ + } + + c.trackLocked(key) + c.data[key] = value + + return evicted > 0 +} + +// Get returns the value for key, if present. +func (c *RandomCache[K, V]) Get(key K) (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + value, ok := c.data[key] + + return value, ok +} + +// Peek returns the value for key without affecting eviction order. For random +// eviction there is no order to affect, so it is identical to Get. +func (c *RandomCache[K, V]) Peek(key K) (V, bool) { + return c.Get(key) +} + +// Contains reports whether key is cached. +func (c *RandomCache[K, V]) Contains(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + _, ok := c.data[key] + + return ok +} + +// Remove deletes key, reporting whether it was present. +func (c *RandomCache[K, V]) Remove(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.removeLocked(key) +} + +// Purge empties the cache. +func (c *RandomCache[K, V]) Purge() { + c.mu.Lock() + defer c.mu.Unlock() + + c.data = make(map[K]V, c.size) + // A new slice rather than a truncation: truncating keeps the backing array + // and every key in it reachable, so a Purge meant to release memory would + // pin the whole keyspace until an equal number of Adds overwrote the slots. + c.keys = make([]K, 0, c.size) + c.index = make(map[K]int, c.size) +} + +// Keys returns the cached keys. The order is arbitrary and not an eviction +// order: this cache has none. +func (c *RandomCache[K, V]) Keys() []K { + c.mu.Lock() + defer c.mu.Unlock() + + keys := make([]K, len(c.keys)) + copy(keys, c.keys) + + return keys +} + +// Values returns the cached values, in the same arbitrary order as Keys. +func (c *RandomCache[K, V]) Values() []V { + c.mu.Lock() + defer c.mu.Unlock() + + values := make([]V, 0, len(c.keys)) + for _, key := range c.keys { + values = append(values, c.data[key]) + } + + return values +} + +// Len returns the number of cached entries. +func (c *RandomCache[K, V]) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + + return len(c.data) +} + +// Resize changes the capacity, evicting at random down to the new size, and +// returns how many entries it evicted. +func (c *RandomCache[K, V]) Resize(size int) int { + c.mu.Lock() + defer c.mu.Unlock() + + if size < 0 { + size = 0 + } + c.size = size + + return c.evictLocked() +} + +// Cap returns the capacity. +func (c *RandomCache[K, V]) Cap() int { + c.mu.Lock() + defer c.mu.Unlock() + + return c.size +} + +var _ ascache.Cacher[string, int] = (*RandomCache[string, int])(nil) diff --git a/policies/regression_test.go b/policies/regression_test.go new file mode 100644 index 0000000..0918acf --- /dev/null +++ b/policies/regression_test.go @@ -0,0 +1,238 @@ +package policies_test + +import ( + "runtime" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/policies" +) + +// TestAddKeepsTheKeyItJustStored is the property every cache owes its caller: +// a successful Add leaves the key retrievable. RandomCache used to draw its +// eviction victim from a pool that already included the incoming key, so a +// write into a full cache was discarded with probability 1/(size+1) - accepted +// and then lost before the very next read. +func TestAddKeepsTheKeyItJustStored(t *testing.T) { + for name, build := range policiesUnderTest { + t.Run(name, func(t *testing.T) { + const size = 4 + const trials = 4000 + + lost := 0 + for i := 0; i < trials; i++ { + p := build(t, size) + for j := 0; j < size; j++ { + p.Add("fill-"+strconv.Itoa(j), j) + } + + p.Add("fresh", 42) + if got, ok := p.Peek("fresh"); !ok || got != 42 { + lost++ + } + } + + assert.Zero(t, lost, + "a successful Add must leave the key present: lost %d of %d writes into a full cache", lost, trials) + }) + } +} + +// TestRandomPurgeReleasesKeys guards a Purge that truncated the key slice +// instead of replacing it, leaving every key reachable through the retained +// backing array and pinning memory the caller purged specifically to free. +func TestRandomPurgeReleasesKeys(t *testing.T) { + const size = 20000 + + cache := policies.NewRandom[string, int](size) + for i := 0; i < size; i++ { + cache.Add(strconv.Itoa(i)+"-------------------------------------------------", i) + } + + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + + cache.Purge() + + runtime.GC() + runtime.GC() + runtime.ReadMemStats(&after) + + require.Zero(t, cache.Len(), "Purge must empty the cache") + + // The keys were the bulk of what was allocated, so Purge must release most + // of it. A truncating Purge released essentially nothing. + assert.Less(t, after.HeapAlloc, before.HeapAlloc, + "Purge must release the key payload, not pin it in a retained backing array") +} + +// TestTTLExpiryNeverServesAZeroValue guards the invariant this library is +// built around. hashicorp's expirable LRU returns (zeroValue, true) from Get +// and Peek for an entry that has expired but not been reaped, which would hand +// a caller a zero value and call it a hit. +func TestTTLExpiryNeverServesAZeroValue(t *testing.T) { + const ttl = 30 * time.Millisecond + p := policies.NewTTL[string, int](100, ttl) + + for i := 1; i <= 20; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + // Wait past the deadline but keep reading throughout, so the check covers + // the window in which an entry is expired but still resident. + deadline := time.Now().Add(4 * ttl) + for time.Now().Before(deadline) { + for i := 1; i <= 20; i++ { + key := "key-" + strconv.Itoa(i) + + if got, ok := p.Get(key); ok { + require.Equal(t, i, got, "Get served a value that was never stored for %q", key) + } + if got, ok := p.Peek(key); ok { + require.Equal(t, i, got, "Peek served a value that was never stored for %q", key) + } + } + } + + assert.Zero(t, p.Len(), "every entry should have expired by now") +} + +// TestTTLValuesAlignWithKeys guards a Values that returned a full-length slice +// padded with zeros once entries expired, so Values no longer corresponded to +// Keys. +func TestTTLValuesAlignWithKeys(t *testing.T) { + const ttl = 30 * time.Millisecond + p := policies.NewTTL[string, int](100, ttl) + + for i := 1; i <= 10; i++ { + p.Add("fresh-"+strconv.Itoa(i), i) + } + time.Sleep(2 * ttl) + for i := 1; i <= 5; i++ { + p.Add("live-"+strconv.Itoa(i), 100+i) + } + + keys, values := p.Keys(), p.Values() + + require.Len(t, values, len(keys), "Values must correspond one-to-one with Keys") + for _, v := range values { + assert.NotZero(t, v, "Values must not report a zero value for an expired entry") + } + assert.Len(t, keys, 5, "only the unexpired entries should be listed") +} + +// TestTTLDoesNotLeakGoroutines guards against the reaper goroutine that +// hashicorp's expirable LRU starts per cache and offers no way to stop, which +// leaked a goroutine and the whole cache for every policy ever constructed. +func TestTTLDoesNotLeakGoroutines(t *testing.T) { + runtime.GC() + before := runtime.NumGoroutine() + + for i := 0; i < 50; i++ { + p := policies.NewTTL[string, int](100, time.Millisecond) + p.Add("a", 1) + } + + runtime.GC() + time.Sleep(50 * time.Millisecond) + after := runtime.NumGoroutine() + + assert.LessOrEqual(t, after, before+2, + "constructing 50 TTL policies must not leave goroutines behind (before %d, after %d)", before, after) +} + +// TestAdaptedResizeSurvivorsAreIntact pins down what a shrinking Resize does +// guarantee. It cannot guarantee WHICH entries survive: Keys() means different +// things per implementation - 2Q returns frequent-then-recent, ARC returns +// recent-then-frequent - so no selection rule is right for both, and the +// rebuilt cache picks its own victims. What must hold is that the cache ends +// up within capacity and every survivor carries the value it was stored with, +// never a zero or another key's value. +func TestAdaptedResizeSurvivorsAreIntact(t *testing.T) { + p, err := policies.NewTwoQueue[string, int](200) + require.NoError(t, err) + + // Promote a small set into 2Q's frequent queue by touching it repeatedly. + hot := make([]string, 10) + for i := range hot { + hot[i] = "hot-" + strconv.Itoa(i) + p.Add(hot[i], i) + } + for round := 0; round < 5; round++ { + for _, key := range hot { + p.Get(key) + } + } + + // Fill the rest with one-off keys that stay in the recent queue. + for i := 0; i < 150; i++ { + p.Add("cold-"+strconv.Itoa(i), i) + } + + p.Resize(20) + + assert.LessOrEqual(t, p.Len(), 20, "Resize must enforce the new capacity") + assert.Equal(t, 20, p.Cap()) + + for i, key := range hot { + if got, ok := p.Peek(key); ok { + assert.Equal(t, i, got, "%s survived the resize with a corrupted value", key) + } + } + for i := 0; i < 150; i++ { + key := "cold-" + strconv.Itoa(i) + if got, ok := p.Peek(key); ok { + assert.Equal(t, i, got, "%s survived the resize with a corrupted value", key) + } + } + + assert.Positive(t, p.Len(), "a shrink to 20 should retain entries, not empty the cache") +} + +// TestAdaptedResizeEnforcesCapacityWhenRebuildFails guards a Resize that +// swallowed a build error, leaving the old capacity in force while Cap() +// reported the new one. hashicorp's 2Q rejects a size of 1, so this is +// reachable rather than hypothetical. +func TestAdaptedResizeEnforcesCapacityWhenRebuildFails(t *testing.T) { + p, err := policies.NewTwoQueue[string, int](50) + require.NoError(t, err) + + for i := 0; i < 50; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + // A 2Q of size 1 cannot be built: its ghost queues round down to zero. + p.Resize(1) + + assert.Equal(t, 1, p.Cap(), "Cap must report the requested capacity") + assert.LessOrEqual(t, p.Len(), 1, + "the requested capacity must actually be enforced, not just reported") + + // And it must keep honouring it. + for i := 0; i < 20; i++ { + p.Add("more-"+strconv.Itoa(i), i) + } + assert.LessOrEqual(t, p.Len(), 1, "capacity must stay enforced after further Adds") +} + +// TestPolicyTypeNamesRoundTrip checks the regenerated stringer output covers +// every policy this repository ships. +func TestPolicyTypeNamesRoundTrip(t *testing.T) { + for policyType, want := range map[ascache.PolicyType]string{ + ascache.LRU: "LRU", + ascache.LFU: "LFU", + ascache.TwoQueue: "TwoQueue", + ascache.ARC: "ARC", + ascache.Random: "Random", + ascache.TTL: "TTL", + ascache.TinyLFU: "TinyLFU", + } { + assert.Equal(t, want, policyType.String()) + } +} diff --git a/policies/tinylfu/go.mod b/policies/tinylfu/go.mod new file mode 100644 index 0000000..d0dc590 --- /dev/null +++ b/policies/tinylfu/go.mod @@ -0,0 +1,17 @@ +module github.com/sshaplygin/as-cache/policies/tinylfu + +go 1.25.2 + +require ( + github.com/maypok86/otter/v2 v2.3.0 + github.com/sshaplygin/as-cache v0.0.0 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/sshaplygin/as-cache => ../.. diff --git a/policies/tinylfu/go.sum b/policies/tinylfu/go.sum new file mode 100644 index 0000000..3662525 --- /dev/null +++ b/policies/tinylfu/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w= +github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/policies/tinylfu/tinylfu.go b/policies/tinylfu/tinylfu.go new file mode 100644 index 0000000..eaae014 --- /dev/null +++ b/policies/tinylfu/tinylfu.go @@ -0,0 +1,207 @@ +// Package tinylfu adapts the W-TinyLFU cache from maypok86/otter to +// ascache.Policy. +// +// It is a module of its own so that otter and its dependencies stay out of any +// build that does not use this arm, matching how the other adapter modules are +// arranged. +// +// W-TinyLFU is the arm worth carrying if the point of an adaptive cache is to +// be better than the state of the art rather than better than LRU. It gates +// admission on a frequency sketch, so a newly requested key must out-score the +// entry it would displace, which makes it strong on skewed workloads and +// resistant to scans. Including it as a candidate is what makes a comparison +// honest: if the bandit keeps choosing it, that is a real result, and if the +// bandit beats it by switching, that is a stronger one. +package tinylfu + +import ( + "fmt" + "math" + + "github.com/maypok86/otter/v2" + + ascache "github.com/sshaplygin/as-cache" +) + +// otter expresses capacity as a uint64 while Cacher uses int. These two +// helpers make every conversion between them explicit and total, so no value +// can wrap: a capacity larger than an int can hold is clamped rather than +// silently becoming negative. +func capacityToInt(capacity uint64) int { + if capacity > math.MaxInt { + return math.MaxInt + } + + return int(capacity) +} + +func capacityToUint64(size int) uint64 { + if size < 0 { + return 0 + } + + return uint64(size) +} + +// Cache adapts otter's W-TinyLFU cache to ascache.Cacher. +// +// otter is natively resizable, so unlike the 2Q and ARC adapters this one +// never rebuilds and never discards the frequency sketch the algorithm has +// built up. +// +// One contract difference is worth knowing: otter reports an approximate size, +// so Len is approximate too. See Len. +type Cache[K comparable, V any] struct { + cache *otter.Cache[K, V] +} + +// New returns a W-TinyLFU cache holding up to size entries. +func New[K comparable, V any](size int) (*Cache[K, V], error) { + cache, err := otter.New(&otter.Options[K, V]{MaximumSize: size}) + if err != nil { + return nil, fmt.Errorf("build w-tinylfu cache: %w", err) + } + + return &Cache[K, V]{cache: cache}, nil +} + +// Add stores a value, reporting whether storing it evicted another entry. +// +// otter reports the previous value rather than an eviction, so an eviction is +// inferred the same way the other adapters do it: storing a key the cache did +// not already hold, while the cache is full, must have displaced something. +// +// The inference is best-effort for a second reason here. W-TinyLFU may reject +// the incoming key outright when the frequency sketch says the resident entry +// is the more valuable one, in which case nothing was evicted and nothing was +// admitted. Callers use this result as a hint, and AdaptiveCache ignores it +// for shadow policies entirely. +func (c *Cache[K, V]) Add(key K, value V) bool { + _, existed := c.cache.GetEntryQuietly(key) + full := c.cache.EstimatedSize() >= capacityToInt(c.cache.GetMaximum()) + c.cache.Set(key, value) + + return !existed && full +} + +// Get returns the value for key, if present, and records the access so the +// frequency sketch sees it. +func (c *Cache[K, V]) Get(key K) (V, bool) { + return c.cache.GetIfPresent(key) +} + +// Peek returns the value for key without recording an access, leaving the +// frequency sketch untouched. +func (c *Cache[K, V]) Peek(key K) (V, bool) { + entry, ok := c.cache.GetEntryQuietly(key) + if !ok { + var zero V + + return zero, false + } + + return entry.Value, true +} + +// Contains reports whether key is cached, without recording an access. +func (c *Cache[K, V]) Contains(key K) bool { + _, ok := c.cache.GetEntryQuietly(key) + + return ok +} + +// Remove deletes key, reporting whether it was present. +func (c *Cache[K, V]) Remove(key K) bool { + _, invalidated := c.cache.Invalidate(key) + + return invalidated +} + +// Purge empties the cache. +func (c *Cache[K, V]) Purge() { + c.cache.InvalidateAll() +} + +// Keys returns the cached keys. otter exposes them as an iterator; they are +// materialised here because Cacher hands callers a slice, and AdaptiveCache +// walks it while migrating data between policies. +// +// The order carries no eviction meaning. Migration only needs the set of live +// keys, but a policy whose Keys order does matter should not be adapted this +// way without checking. +func (c *Cache[K, V]) Keys() []K { + keys := make([]K, 0, c.cache.EstimatedSize()) + for key := range c.cache.Keys() { + keys = append(keys, key) + } + + return keys +} + +// Values returns the cached values, in the same arbitrary order as Keys. +func (c *Cache[K, V]) Values() []V { + values := make([]V, 0, c.cache.EstimatedSize()) + for value := range c.cache.Values() { + values = append(values, value) + } + + return values +} + +// Len returns the number of cached entries. +// +// otter reports an approximate size: it may differ from the true count while +// insertions or deletions are in flight, or while expired entries await +// removal. That is fine for every use AdaptiveCache makes of it except one - +// the capacity gate that holds off policy switching until the cache is full +// compares Len against Cap for exact equality, and an approximate Len can miss +// that equality. Set Settings.EvictPartialCapacityFilling to true when this +// policy is an arm, or accept that switching may start later than it would +// otherwise. +func (c *Cache[K, V]) Len() int { + return c.cache.EstimatedSize() +} + +// Cap returns the capacity. +func (c *Cache[K, V]) Cap() int { + return capacityToInt(c.cache.GetMaximum()) +} + +// Resize changes the capacity to size and returns the number of entries +// evicted to reach it. +// +// otter resizes in place, so the frequency sketch survives - an adapted 2Q or +// ARC policy would have to be rebuilt and would lose everything it had +// learned. Eviction happens asynchronously, so the returned count is what had +// been evicted by the time this call finished and may understate the total. +func (c *Cache[K, V]) Resize(size int) int { + if size < 0 { + size = 0 + } + + before := c.cache.EstimatedSize() + c.cache.SetMaximum(capacityToUint64(size)) + // Force pending maintenance so the eviction the new maximum implies has + // happened before the count is taken. + c.cache.CleanUp() + + evicted := before - c.cache.EstimatedSize() + if evicted < 0 { + return 0 + } + + return evicted +} + +// NewPolicy returns a W-TinyLFU policy of the given size, ready to be used as +// a bandit arm. +func NewPolicy[K comparable, V any](size int) (ascache.Policy[K, V], error) { + cache, err := New[K, V](size) + if err != nil { + return nil, err + } + + return ascache.NewCache[K, V](cache, ascache.TinyLFU, size), nil +} + +var _ ascache.Cacher[string, int] = (*Cache[string, int])(nil) diff --git a/policies/tinylfu/tinylfu_test.go b/policies/tinylfu/tinylfu_test.go new file mode 100644 index 0000000..3f19494 --- /dev/null +++ b/policies/tinylfu/tinylfu_test.go @@ -0,0 +1,279 @@ +package tinylfu_test + +import ( + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ascache "github.com/sshaplygin/as-cache" + "github.com/sshaplygin/as-cache/policies/tinylfu" +) + +func newTinyLFU(t *testing.T, size int) ascache.Policy[string, int] { + t.Helper() + + p, err := tinylfu.NewPolicy[string, int](size) + require.NoError(t, err) + + return p +} + +func TestTinyLFUConformance(t *testing.T) { + t.Run("stores and retrieves", func(t *testing.T) { + p := newTinyLFU(t, 100) + p.Add("a", 1) + + got, ok := p.Get("a") + require.True(t, ok) + assert.Equal(t, 1, got) + }) + + t.Run("reports a miss for an absent key", func(t *testing.T) { + p := newTinyLFU(t, 100) + + got, ok := p.Get("nope") + assert.False(t, ok) + assert.Zero(t, got) + }) + + t.Run("overwrites without growing", func(t *testing.T) { + p := newTinyLFU(t, 100) + p.Add("a", 1) + p.Add("a", 2) + + got, ok := p.Get("a") + require.True(t, ok) + assert.Equal(t, 2, got) + assert.Equal(t, 1, p.Len()) + }) + + t.Run("Peek does not report a miss for a live key", func(t *testing.T) { + p := newTinyLFU(t, 100) + p.Add("a", 42) + + got, ok := p.Peek("a") + require.True(t, ok) + assert.Equal(t, 42, got) + }) + + t.Run("Contains agrees with Peek", func(t *testing.T) { + p := newTinyLFU(t, 100) + p.Add("a", 1) + + assert.True(t, p.Contains("a")) + assert.False(t, p.Contains("b")) + }) + + t.Run("Remove reports presence", func(t *testing.T) { + p := newTinyLFU(t, 100) + p.Add("a", 1) + + assert.True(t, p.Remove("a"), "removing a present key must report true") + assert.False(t, p.Remove("a"), "removing an absent key must report false") + }) + + t.Run("Purge empties", func(t *testing.T) { + p := newTinyLFU(t, 100) + for i := 0; i < 50; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Purge() + assert.Zero(t, p.Len()) + }) + + t.Run("stays within capacity", func(t *testing.T) { + const size = 100 + p := newTinyLFU(t, size) + + for i := 0; i < size*10; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + // otter evicts asynchronously and reports an approximate size, so this + // asserts the bound is respected rather than an exact count. + assert.LessOrEqual(t, p.Len(), size*2, + "a cache of %d must not grow without bound", size) + assert.Equal(t, size, p.Cap()) + }) + + t.Run("Keys and Values are materialised", func(t *testing.T) { + p := newTinyLFU(t, 100) + for i := 0; i < 10; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + assert.NotEmpty(t, p.Keys(), "Keys must materialise otter's iterator") + assert.NotEmpty(t, p.Values(), "Values must materialise otter's iterator") + }) + + t.Run("never serves a value that was never stored", func(t *testing.T) { + p := newTinyLFU(t, 200) + for i := 0; i < 500; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + for i := 0; i < 500; i++ { + key := "key-" + strconv.Itoa(i) + if got, ok := p.Peek(key); ok { + require.Equal(t, i, got, "%s carries a value that was never stored", key) + } + } + }) + + t.Run("Resize shrinks and Cap follows", func(t *testing.T) { + p := newTinyLFU(t, 500) + for i := 0; i < 500; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(50) + + assert.Equal(t, 50, p.Cap(), "Cap must follow Resize - the adaptive layer relies on it") + assert.LessOrEqual(t, p.Len(), 100, "Resize must bring the cache down toward the new capacity") + }) + + t.Run("Resize grows", func(t *testing.T) { + p := newTinyLFU(t, 100) + for i := 0; i < 50; i++ { + p.Add("key-"+strconv.Itoa(i), i) + } + + p.Resize(1000) + assert.Equal(t, 1000, p.Cap()) + }) + + t.Run("tracks hits and misses", func(t *testing.T) { + p := newTinyLFU(t, 100) + p.Add("a", 1) + p.ResetStats() + + p.Get("a") + p.Get("absent") + + stats := p.GetStats() + assert.Equal(t, int64(1), stats.Hits) + assert.Equal(t, int64(1), stats.Misses) + }) + + t.Run("reports the TinyLFU policy type", func(t *testing.T) { + assert.Equal(t, ascache.TinyLFU, newTinyLFU(t, 10).GetType()) + assert.Equal(t, "TinyLFU", ascache.TinyLFU.String()) + }) + + t.Run("is safe under concurrent use, including resizes", func(t *testing.T) { + p := newTinyLFU(t, 200) + + var wg sync.WaitGroup + for g := 0; g < 8; g++ { + wg.Add(1) + go func(seed int) { + defer wg.Done() + for i := 0; i < 300; i++ { + key := "key-" + strconv.Itoa((seed+i)%400) + p.Add(key, i) + p.Get(key) + p.Contains(key) + _ = p.Len() + if i%50 == 0 { + _ = p.Keys() + } + } + }(g) + } + + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 20; i++ { + p.Resize(100 + i*10) + } + }() + + wg.Wait() + }) +} + +// TestTinyLFUKeepsHotKeysUnderScan is the property the arm is carried for: a +// long scan of one-off keys must not flush a small set of repeatedly requested +// ones, which is exactly what defeats plain LRU. +func TestTinyLFUKeepsHotKeysUnderScan(t *testing.T) { + const size = 500 + p := newTinyLFU(t, size) + + hot := make([]string, 50) + for i := range hot { + hot[i] = "hot-" + strconv.Itoa(i) + } + + // Establish the hot set firmly in the frequency sketch. + for round := 0; round < 40; round++ { + for _, key := range hot { + p.Add(key, 1) + p.Get(key) + } + } + + // Now scan far more one-off keys than the cache can hold. + for i := 0; i < size*20; i++ { + p.Add("scan-"+strconv.Itoa(i), i) + } + + retained := 0 + for _, key := range hot { + if p.Contains(key) { + retained++ + } + } + + assert.Greater(t, retained, len(hot)/2, + "W-TinyLFU should keep most of the hot set through a scan, kept %d of %d", retained, len(hot)) +} + +// TestTinyLFUDrivesAnAdaptiveCache checks the arm works inside a real cache, +// including the invariant that a caller never sees a value never stored. +func TestTinyLFUDrivesAnAdaptiveCache(t *testing.T) { + const size = 1000 + + arm, err := tinylfu.NewPolicy[string, int](size) + require.NoError(t, err) + + cache, err := ascache.NewAdaptiveCache( + []ascache.Policy[string, int]{arm}, + &fixedBandit{pick: ascache.TinyLFU}, + &ascache.Settings{ + EpochDuration: time.Millisecond, + // Len is approximate for this policy, so the capacity gate cannot + // be relied on to fire - see Cache.Len. + EvictPartialCapacityFilling: true, + MigrationStrategy: ascache.MigrationWarm, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + + for i := 0; i < size; i++ { + cache.Add("key-"+strconv.Itoa(i), i) + } + + deadline := time.Now().Add(100 * time.Millisecond) + for time.Now().Before(deadline) { + for i := 0; i < 200; i++ { + key := "key-" + strconv.Itoa(i) + if got, ok := cache.Get(key); ok { + require.Equal(t, i, got, "key %q returned a value that was never stored", key) + } + } + } + + assert.Equal(t, ascache.TinyLFU, cache.ActivePolicy()) +} + +type fixedBandit struct{ pick ascache.PolicyType } + +func (b *fixedBandit) RecordStats(_ ascache.ShadowStats) {} +func (b *fixedBandit) SelectPolicy() ascache.PolicyType { return b.pick } diff --git a/policies/ttl.go b/policies/ttl.go new file mode 100644 index 0000000..f10ee81 --- /dev/null +++ b/policies/ttl.go @@ -0,0 +1,257 @@ +package policies + +import ( + "sync" + "time" + + lru "github.com/hashicorp/golang-lru/v2" +) + +// ttlEntry pairs a value with the moment it stops being servable. Keeping the +// deadline alongside the value is what makes expiry exact: the entry and its +// deadline are evicted together, so there is no bookkeeping to fall out of +// step with the cache. +type ttlEntry[V any] struct { + value V + expiresAt time.Time +} + +// TTLCache is an LRU cache whose entries also expire after a fixed duration. +// +// It is built on hashicorp's plain LRU with expiry handled here rather than on +// hashicorp's expirable LRU, for three reasons, each of which was a real +// defect when this was tried the other way round: +// +// - expirable.LRU's Get and Peek return (zeroValue, true) for an entry that +// has expired but not yet been reaped, because the expiry branch takes a +// bare return over an already-true named result. A cache that hands a +// caller a zero value and calls it a hit breaks the one invariant this +// library is built around. +// - expirable.LRU's Values allocates a full-length slice and skips expired +// entries while filling it, so it returns trailing zero values that do not +// line up with Keys. +// - expirable.NewLRU starts a reaper goroutine per cache and offers no way +// to stop it, so every policy ever constructed leaks that goroutine and +// everything the cache retains, for the life of the process. +// +// Expiry here is lazy: an expired entry occupies its slot until it is read, +// overwritten, or evicted by LRU pressure. That trades a little memory for +// exactness and for not owning a goroutine, which is the right trade for a +// policy that may exist only to be measured as a shadow. +// +// It also treats capacity the way the rest of the Cacher implementations do. +// For expirable.LRU a size of zero means *unlimited* - documented as turning +// the LRU mechanism off - whereas every other policy treats zero as holding +// nothing. Since AdaptiveCache resizes shadow policies automatically, a resize +// that reached zero would quietly convert a bounded shadow into an unbounded +// one. Here zero means empty, like everywhere else. +type TTLCache[K comparable, V any] struct { + mu sync.RWMutex + cache *lru.Cache[K, ttlEntry[V]] + size int + ttl time.Duration + // now is time.Now except in tests, which need to move the clock. + now func() time.Time +} + +// NewTTLCache returns a cache holding up to size entries, treating entries +// older than ttl as absent. A ttl of zero or less disables expiry, leaving +// plain LRU behaviour. A size of zero or less holds nothing. +func NewTTLCache[K comparable, V any](size int, ttl time.Duration) *TTLCache[K, V] { + if size < 0 { + size = 0 + } + + // lru.New rejects a non-positive size, and a zero-capacity cache is + // represented by size, not by the underlying cache's capacity. + cache, err := lru.New[K, ttlEntry[V]](max(size, 1)) + if err != nil { + // Unreachable: the size passed is at least 1. + panic("policies: building ttl cache: " + err.Error()) + } + + return &TTLCache[K, V]{ + cache: cache, + size: size, + ttl: ttl, + now: time.Now, + } +} + +// expired reports whether an entry is past its deadline. +func (c *TTLCache[K, V]) expired(e ttlEntry[V]) bool { + return !e.expiresAt.IsZero() && c.now().After(e.expiresAt) +} + +// deadline returns the expiry instant for an entry stored now, or the zero +// time when expiry is disabled. +func (c *TTLCache[K, V]) deadline() time.Time { + if c.ttl <= 0 { + return time.Time{} + } + + return c.now().Add(c.ttl) +} + +// Add stores a value, reporting whether storing it evicted another entry. +func (c *TTLCache[K, V]) Add(key K, value V) bool { + c.mu.Lock() + defer c.mu.Unlock() + + if c.size <= 0 { + return false + } + + return c.cache.Add(key, ttlEntry[V]{value: value, expiresAt: c.deadline()}) +} + +// Get returns the value for key if it is present and unexpired, recording the +// access. An expired entry is reported as a miss and dropped. +func (c *TTLCache[K, V]) Get(key K) (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.cache.Get(key) + if !ok { + var zero V + + return zero, false + } + + if c.expired(entry) { + c.cache.Remove(key) + var zero V + + return zero, false + } + + return entry.value, true +} + +// Peek returns the value for key if it is present and unexpired, without +// recording an access or dropping the expired entry. +func (c *TTLCache[K, V]) Peek(key K) (V, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + entry, ok := c.cache.Peek(key) + if !ok || c.expired(entry) { + var zero V + + return zero, false + } + + return entry.value, true +} + +// Contains reports whether key is cached and unexpired, without recording an +// access. +func (c *TTLCache[K, V]) Contains(key K) bool { + c.mu.RLock() + defer c.mu.RUnlock() + + entry, ok := c.cache.Peek(key) + + return ok && !c.expired(entry) +} + +// Remove deletes key, reporting whether it was present. An entry that has +// expired but not yet been reclaimed is reported as absent. +func (c *TTLCache[K, V]) Remove(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + entry, ok := c.cache.Peek(key) + c.cache.Remove(key) + + return ok && !c.expired(entry) +} + +// Purge empties the cache. +func (c *TTLCache[K, V]) Purge() { + c.mu.Lock() + defer c.mu.Unlock() + + c.cache.Purge() +} + +// Keys returns the unexpired cached keys, oldest first. +func (c *TTLCache[K, V]) Keys() []K { + c.mu.RLock() + defer c.mu.RUnlock() + + all := c.cache.Keys() + keys := make([]K, 0, len(all)) + for _, key := range all { + if entry, ok := c.cache.Peek(key); ok && !c.expired(entry) { + keys = append(keys, key) + } + } + + return keys +} + +// Values returns the unexpired cached values, in the same order as Keys. +func (c *TTLCache[K, V]) Values() []V { + c.mu.RLock() + defer c.mu.RUnlock() + + all := c.cache.Keys() + values := make([]V, 0, len(all)) + for _, key := range all { + if entry, ok := c.cache.Peek(key); ok && !c.expired(entry) { + values = append(values, entry.value) + } + } + + return values +} + +// Len returns the number of unexpired cached entries. +func (c *TTLCache[K, V]) Len() int { + c.mu.RLock() + defer c.mu.RUnlock() + + if c.ttl <= 0 { + return c.cache.Len() + } + + live := 0 + for _, key := range c.cache.Keys() { + if entry, ok := c.cache.Peek(key); ok && !c.expired(entry) { + live++ + } + } + + return live +} + +// Cap returns the capacity. +func (c *TTLCache[K, V]) Cap() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.size +} + +// Resize changes the capacity to size and returns the number of entries +// evicted to reach it. A size of zero or less empties the cache and holds +// nothing. +func (c *TTLCache[K, V]) Resize(size int) int { + c.mu.Lock() + defer c.mu.Unlock() + + if size < 0 { + size = 0 + } + c.size = size + + if size == 0 { + evicted := c.cache.Len() + c.cache.Purge() + + return evicted + } + + return c.cache.Resize(size) +} diff --git a/policytype_string.go b/policytype_string.go index be12f54..0cd1602 100644 --- a/policytype_string.go +++ b/policytype_string.go @@ -11,11 +11,16 @@ func _() { _ = x[Undefined-0] _ = x[LRU-1] _ = x[LFU-2] + _ = x[TwoQueue-3] + _ = x[ARC-4] + _ = x[Random-5] + _ = x[TTL-6] + _ = x[TinyLFU-7] } -const _PolicyType_name = "UndefinedLRULFU" +const _PolicyType_name = "UndefinedLRULFUTwoQueueARCRandomTTLTinyLFU" -var _PolicyType_index = [...]uint8{0, 9, 12, 15} +var _PolicyType_index = [...]uint8{0, 9, 12, 15, 23, 26, 32, 35, 42} func (i PolicyType) String() string { idx := int(i) - 0 diff --git a/sampling.go b/sampling.go new file mode 100644 index 0000000..c168b09 --- /dev/null +++ b/sampling.go @@ -0,0 +1,129 @@ +package ascache + +import ( + "hash/maphash" + "math" +) + +// maxUint64AsFloat is 2^64 as a float64. Converting a float64 that is greater +// than or equal to it back to uint64 is undefined in Go, so rates at or above 1 +// are handled separately rather than scaled. +const maxUint64AsFloat = float64(1 << 64) + +// keySampler decides whether a key belongs to the deterministic subset of the +// keyspace that shadow policies track. Sampling lets a shadow estimate its hit +// rate from a small fraction of traffic instead of mirroring every operation. +// +// The decision is a pure function of the key and the seed, so a given key is +// either always sampled or never sampled for the lifetime of the sampler. That +// matters twice over: a sampled shadow sees a coherent access pattern for the +// keys it does track (rather than a random scatter that would destroy any +// notion of reuse), and every shadow sharing one sampler measures the same +// sub-workload, which is what makes their hit rates comparable to each other. +// +// The seed is drawn per cache rather than fixed, so the sampled subset differs +// between processes and cannot be predicted or targeted by a caller. +// +// Sampled counts are never scaled back up to full-traffic magnitude before +// reaching the bandit. Scaling would restore magnitude while inventing +// confidence, handing a Beta posterior twenty times the evidence that was +// actually collected. Instead every arm, the active policy included, is +// measured over this same sampled substream, so the arms carry equal and +// honest evidence and remain directly comparable. +type keySampler[K comparable] struct { + seed maphash.Seed + // threshold is the exclusive upper bound on a key's hash for it to be in + // the sample. It is only meaningful when sampling is true. + threshold uint64 + // sampling reports whether any filtering happens at all. It is false when + // the rate is 1 (or above), in which case every key is in the sample. + sampling bool + rate float64 +} + +// newKeySampler returns a sampler admitting approximately rate of the keyspace. +// A rate of 1 or above admits every key and performs no hashing. +func newKeySampler[K comparable](rate float64) *keySampler[K] { + s := &keySampler[K]{ + seed: maphash.MakeSeed(), + rate: rate, + } + + if rate >= 1 { + s.rate = 1 + return s + } + + s.sampling = true + s.threshold = uint64(math.Max(0, rate) * maxUint64AsFloat) + + return s +} + +// sampled reports whether key is part of the tracked sample. +func (s *keySampler[K]) sampled(key K) bool { + if !s.sampling { + return true + } + + return maphash.Comparable(s.seed, key) < s.threshold +} + +// scaledCapacity returns the miniature capacity corresponding to sampling rate +// of a cache of the given size, holding the identity capacity/size == rate. +// +// Unlike shadowCapacity it applies no floor. The floor exists to stop a cache +// from being built with a miniature too small to measure, and it works by +// raising the sample rate to match. After construction the rate can no longer +// move, so applying the floor alone would leave shadows running at a capacity +// larger than their share of the traffic - and a shadow of capacity C fed an +// r-sampled stream simulates a cache of C/r. Every shadow would then simulate a +// larger cache than the active policy actually is and report a better hit rate +// for that reason alone, which is a systematic bias against whichever policy is +// active. A miniature that is merely small is noisy; one that is inconsistent +// with its rate is wrong, so the identity wins. +func scaledCapacity(size int, rate float64) int { + if size <= 0 || rate >= 1 { + return size + } + + capacity := int(math.Ceil(rate * float64(size))) + if capacity < 1 { + capacity = 1 + } + if capacity > size { + capacity = size + } + + return capacity +} + +// shadowCapacity returns the capacity a shadow policy should run at to +// simulate a full-size cache of nominalCap over the sampled substream, and the +// effective rate that capacity corresponds to. +// +// A cache of capacity rate*N fed an rate-sampled stream approximates a cache +// of capacity N fed the full stream, so the capacity has to shrink with the +// rate for the estimate to mean anything. A floor guards the degenerate end: +// a five-entry miniature measures noise, so when rate*nominalCap falls below +// minCapacity the rate itself is raised (not just the capacity) to keep the +// simulation identity intact, up to the point where sampling disables itself. +func shadowCapacity(nominalCap int, rate float64, minCapacity int) (capacity int, effectiveRate float64) { + if nominalCap <= 0 || rate >= 1 { + return nominalCap, 1 + } + + capacity = int(math.Ceil(rate * float64(nominalCap))) + effectiveRate = rate + + if capacity < minCapacity { + capacity = minCapacity + effectiveRate = float64(minCapacity) / float64(nominalCap) + } + + if capacity >= nominalCap { + return nominalCap, 1 + } + + return capacity, effectiveRate +} diff --git a/sampling_test.go b/sampling_test.go new file mode 100644 index 0000000..116b29a --- /dev/null +++ b/sampling_test.go @@ -0,0 +1,147 @@ +package ascache + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestKeySampler_FullRateAdmitsEverything(t *testing.T) { + for _, rate := range []float64{1, 1.5, 2} { + s := newKeySampler[string](rate) + require.False(t, s.sampling, "rate %v must disable filtering", rate) + + for i := 0; i < 1000; i++ { + assert.True(t, s.sampled("key-"+strconv.Itoa(i)), + "rate %v must admit every key", rate) + } + } +} + +func TestKeySampler_Deterministic(t *testing.T) { + s := newKeySampler[string](0.05) + + for i := 0; i < 1000; i++ { + key := "key-" + strconv.Itoa(i) + want := s.sampled(key) + for r := 0; r < 5; r++ { + assert.Equal(t, want, s.sampled(key), + "sampling decision for %q must be stable", key) + } + } +} + +func TestKeySampler_RateIsApproximatelyHonoured(t *testing.T) { + const keys = 200000 + + for _, rate := range []float64{0.01, 0.05, 0.25} { + s := newKeySampler[string](rate) + + sampled := 0 + for i := 0; i < keys; i++ { + if s.sampled("key-" + strconv.Itoa(i)) { + sampled++ + } + } + + got := float64(sampled) / float64(keys) + // Generous tolerance: this asserts the sampler is not wildly off, + // not that a hash is perfectly uniform. + assert.InDelta(t, rate, got, rate*0.15, + "rate %v: sampled %d of %d keys (%.4f)", rate, sampled, keys, got) + } +} + +func TestKeySampler_ZeroRateAdmitsNothing(t *testing.T) { + s := newKeySampler[string](0) + + for i := 0; i < 1000; i++ { + require.False(t, s.sampled("key-"+strconv.Itoa(i)), + "a zero rate must admit no keys") + } +} + +func TestKeySampler_WorksForNonStringKeys(t *testing.T) { + type composite struct { + A int + B string + } + + si := newKeySampler[int](0.5) + sc := newKeySampler[composite](0.5) + + intSampled, compositeSampled := 0, 0 + for i := 0; i < 10000; i++ { + if si.sampled(i) { + intSampled++ + } + if sc.sampled(composite{A: i, B: strconv.Itoa(i)}) { + compositeSampled++ + } + } + + assert.InDelta(t, 0.5, float64(intSampled)/10000, 0.05, "int keys") + assert.InDelta(t, 0.5, float64(compositeSampled)/10000, 0.05, "struct keys") +} + +func TestShadowCapacity(t *testing.T) { + tests := []struct { + name string + nominal int + rate float64 + minCapacity int + wantCap int + wantRate float64 + }{ + { + name: "rate 1 disables sampling", + nominal: 10000, rate: 1, minCapacity: 256, + wantCap: 10000, wantRate: 1, + }, + { + name: "capacity shrinks with the rate", + nominal: 100000, rate: 0.05, minCapacity: 256, + wantCap: 5000, wantRate: 0.05, + }, + { + name: "floor raises the rate, not just the capacity", + nominal: 1000, rate: 0.01, minCapacity: 256, + wantCap: 256, wantRate: 0.256, + }, + { + name: "a cache smaller than the floor disables sampling", + nominal: 100, rate: 0.05, minCapacity: 256, + wantCap: 100, wantRate: 1, + }, + { + name: "zero capacity is left alone", + nominal: 0, rate: 0.05, minCapacity: 256, + wantCap: 0, wantRate: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCap, gotRate := shadowCapacity(tt.nominal, tt.rate, tt.minCapacity) + assert.Equal(t, tt.wantCap, gotCap, "capacity") + assert.InDelta(t, tt.wantRate, gotRate, 1e-9, "effective rate") + }) + } +} + +// TestShadowCapacity_PreservesSimulationIdentity checks the property the +// miniature relies on: its capacity is the same fraction of the full cache +// that the sample is of the keyspace. If the two drift apart the shadow is no +// longer simulating the cache it claims to. +func TestShadowCapacity_PreservesSimulationIdentity(t *testing.T) { + for _, nominal := range []int{1000, 10000, 100000} { + for _, rate := range []float64{0.01, 0.05, 0.2} { + gotCap, gotRate := shadowCapacity(nominal, rate, 256) + + assert.InDelta(t, gotRate, float64(gotCap)/float64(nominal), 0.001, + "nominal=%d rate=%v: capacity fraction must match the effective rate", nominal, rate) + } + } +} diff --git a/scripts/fetch-traces.sh b/scripts/fetch-traces.sh new file mode 100755 index 0000000..1a333b0 --- /dev/null +++ b/scripts/fetch-traces.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Fetch real cache traces for the evidence harness. +# +# Traces are NOT committed to this repository: they are large, and most carry +# licences that do not grant redistribution. This downloads them to a local +# directory and prints the environment variable the harness reads. +# +# Usage: +# ./scripts/fetch-traces.sh [target-dir] # default: ./traces (gitignored) +# AS_CACHE_TRACES=$(pwd)/traces make evidence +set -euo pipefail + +TRACES="${1:-$(pwd)/traces}" +mkdir -p "$TRACES" + +fetch() { + local name="$1" url="$2" + if [ -s "$TRACES/$name" ]; then + echo " have $name" + return + fi + echo " get $name" + # --fail so an HTML error page is never mistaken for trace data. + curl -fSL --retry 3 -o "$TRACES/$name" "$url" +} + +echo "Fetching traces into $TRACES" + +# --- Twitter Twemcache, CC BY 4.0 ------------------------------------------- +# A real production in-memory key-value workload, which is this library's +# actual target domain rather than block I/O. ~1M requests, string keys. +# Cite: Yang, Yue & Rashmi, "A Large Scale Analysis of Hundreds of In-memory +# Cache Clusters at Twitter", OSDI '20. +fetch twitter_cluster052.csv \ + https://raw.githubusercontent.com/twitter/cache-trace/master/samples/2020Mar/cluster052 + +# --- LIRS research traces --------------------------------------------------- +# Tiny and deliberately adversarial. `loop` is a cyclic scan that defeats LRU +# outright, which is the clearest demonstration of why policy choice matters. +# No explicit licence: benchmark against them and cite, but do not vendor. +# Cite: Jiang & Zhang, "LIRS", SIGMETRICS '02. +LIRS=https://raw.githubusercontent.com/ben-manes/caffeine/master/simulator/src/main/resources/com/github/benmanes/caffeine/cache/simulator/parser/lirs +for f in loop 2_pools multi2; do + fetch "lirs_$f.trace.gz" "$LIRS/$f.trace.gz" +done + +# --- ARC paper traces ------------------------------------------------------- +# The traces the ARC paper reported on, so numbers here are comparable with the +# literature. Each record expands into blockCount consecutive accesses - see +# LoadARCTrace. The canonical IBM host is long dead; these are mirrored in +# otter's benchmark suite. +# Cite: Megiddo & Modha, "ARC", FAST '03. +ARC=https://raw.githubusercontent.com/maypok86/otter/main/benchmarks/simulator/trace/arc +for f in p3 oltp; do + fetch "arc_$f.gz" "$ARC/$f.gz" +done + +echo +echo "Done. Run the evidence harness with:" +echo " AS_CACHE_TRACES=$TRACES make evidence" diff --git a/settings.go b/settings.go new file mode 100644 index 0000000..fe14213 --- /dev/null +++ b/settings.go @@ -0,0 +1,148 @@ +package ascache + +import ( + "context" + "fmt" + "time" +) + +// Settings configures the behaviour of AdaptiveCache. +type Settings struct { + EpochDuration time.Duration + // EvictPartialCapacityFilling allows policy switching even when the cache + // is not yet full. + EvictPartialCapacityFilling bool + // MigrationStrategy determines how data is moved when the active policy + // changes. Defaults to MigrationCold (zero value). + MigrationStrategy MigrationStrategy + + // MinHitRateImprovement is the hit-rate advantage, as an absolute + // difference in [0,1], that the bandit's selection must hold over the + // active policy in the epoch just measured before the switch is applied. + // It damps oscillation between policies that perform almost identically. + // Zero (the default) applies every selection the bandit makes. + MinHitRateImprovement float64 + + // SwitchCooldownEpochs is the number of epochs that must elapse after a + // policy switch before another switch is allowed, counted from the last + // switch or from cache creation if none has happened yet. Zero (the + // default) allows a switch on every epoch. + SwitchCooldownEpochs int64 + + // MinEpochRequests is the number of requests (hits plus misses) both the + // active policy and the candidate must have observed in the measured + // epoch before a switch is allowed, so the cache does not react to a + // handful of samples. Zero (the default) imposes no minimum. + // + // The requests counted are the ones the bandit sees, which under + // ShadowSampleRate means sampled requests: at a rate of 0.05 a threshold + // of 100 is reached after roughly 2000 real requests. + MinEpochRequests int64 + + // ShadowSampleRate is the fraction of the keyspace, in (0,1], that shadow + // policies track. Shadows exist only to estimate a hit rate, and a hit + // rate can be estimated from a sample: at 0.05 a shadow skips 95% of the + // operations it would otherwise mirror, which is where the bulk of the + // adaptive layer's overhead goes. + // + // Shadows shrink with the rate so each remains a faithful miniature of a + // full-size cache, and every shadow samples the same keys so their hit + // rates stay comparable. The active policy still serves every key; only + // the measurement is sampled, and it is sampled for the active policy too + // so that all arms carry equally weighted evidence. + // + // Zero (the default) means 1: shadows mirror every key, which is the + // behaviour of earlier versions. + ShadowSampleRate float64 + + // ObserveOnly runs the cache as a measurement instrument: every policy is + // still measured each epoch and reported to the bandit, but the active + // policy never changes and no migration ever happens. + // + // This is the zero-risk way to adopt the library. The cache behaves + // exactly like the single policy you gave it first, while Advice() answers + // the question that is otherwise expensive to ask: would a different + // eviction policy serve this traffic better, and by how much. Once the + // answer is in, either switch to that policy directly or turn this off and + // let the bandit do it. + ObserveOnly bool + + // MinShadowCapacity is the floor on a shadow's miniature capacity. A + // miniature of a handful of entries measures noise rather than a policy, + // so when the sample rate would shrink a shadow below this floor the + // effective rate is raised instead, up to the point where sampling + // disables itself entirely. Zero (the default) applies + // DefaultMinShadowCapacity. + MinShadowCapacity int +} + +// DefaultMinShadowCapacity is the miniature capacity floor applied when +// Settings.MinShadowCapacity is zero. +const DefaultMinShadowCapacity = 256 + +// NewAdaptiveCache validates its inputs and starts the background epoch +// goroutine. Callers must call Close to stop that goroutine. +func NewAdaptiveCache[K comparable, V any]( + policies []Policy[K, V], + bandit Bandit, + settings *Settings, +) (*AdaptiveCache[K, V], error) { + if len(policies) == 0 { + return nil, ErrEmptyPolicies + } + if settings == nil { + return nil, ErrNilSettings + } + if bandit == nil { + // Observing needs no strategy: nothing is ever selected. Requiring a + // bandit for the zero-risk adoption path would be friction for no + // reason, since implementing one is the fiddliest part of using this + // library. + if !settings.ObserveOnly { + return nil, ErrNilBandit + } + bandit = observerBandit{} + } + if settings.EpochDuration <= 0 { + return nil, fmt.Errorf("%w: got %s", ErrInvalidEpochDuration, settings.EpochDuration) + } + + availablePolicies := make(map[PolicyType]Policy[K, V], len(policies)) + for _, policy := range policies { + if policy == nil { + return nil, ErrNilPolicy + } + if _, exists := availablePolicies[policy.GetType()]; exists { + return nil, fmt.Errorf("%w: %s", ErrDuplicatePolicy, policy.GetType()) + } + availablePolicies[policy.GetType()] = policy + } + + ctx, cancel := context.WithCancel(context.Background()) + + ac := &AdaptiveCache[K, V]{ + policies: availablePolicies, + activePolicy: policies[0].GetType(), + bandit: bandit, + epochTicker: time.NewTicker(settings.EpochDuration), + ctx: ctx, + cancel: cancel, + settings: settings, + } + + sampleRate := settings.ShadowSampleRate + if sampleRate <= 0 { + sampleRate = 1 + } + minShadowCap := settings.MinShadowCapacity + if minShadowCap <= 0 { + minShadowCap = DefaultMinShadowCapacity + } + ac.minShadowCap = minShadowCap + ac.initShadowDutyLocked(sampleRate, minShadowCap) + + ac.wg.Add(1) + go ac.runAdaptiveSelect() + + return ac, nil +} diff --git a/shadow.go b/shadow.go new file mode 100644 index 0000000..4f9eab1 --- /dev/null +++ b/shadow.go @@ -0,0 +1,153 @@ +package ascache + +// demoteLocked puts a policy that has just stopped being active onto shadow +// duty: it releases the policy's hold on real values and shrinks it to the +// miniature capacity it simulates at. +// +// Values are dropped rather than kept because a demoted policy no longer +// serves anyone. Its keys still matter - they are the eviction bookkeeping +// that makes its hit-rate estimate meaningful - so entries are rewritten to +// the zero value instead of being purged. Rewriting in Keys() order preserves +// the ordering the policy maintains: for a recency policy the oldest-to-newest +// walk re-establishes the same recency order, and for a frequency policy every +// surviving key gains exactly one access, which leaves the relative ordering +// untouched. +// +// Keys outside the sample are removed outright, so what remains is the +// substream every other shadow is measuring. +// +// It must be called while the write lock is held, and only after the new state +// has been published, so a reader holding a stale view cannot observe a value +// being dropped and mistake the zero for real data. +func (c *AdaptiveCache[K, V]) demoteLocked(policyType PolicyType) { + policy, ok := c.policies[policyType] + if !ok { + return + } + + var zero V + for _, key := range policy.Keys() { + if c.sampler.sampled(key) { + policy.Add(key, zero) + continue + } + policy.Remove(key) + } + + if capacity := c.shadowCap[policyType]; capacity > 0 { + policy.Resize(capacity) + } + + // Whatever this policy measured in its previous role was measured at a + // different capacity, and over all traffic rather than the sample. Carrying + // those counts into its first shadow epoch would misreport it to the bandit. + policy.ResetStats() +} + +// promoteLockedCapacity restores a policy to its full nominal capacity as it +// takes over active duty. The caller purges it afterwards - a policy arriving +// from shadow duty holds only zero values - so no real data is resized away. +// +// It must be called while the write lock is held. +func (c *AdaptiveCache[K, V]) promoteLockedCapacity(policyType PolicyType) { + policy, ok := c.policies[policyType] + if !ok { + return + } + + if capacity := c.nominalCap[policyType]; capacity > 0 && policy.Cap() != capacity { + policy.Resize(capacity) + } +} + +// switchLocked applies a policy change end to end: it restores the incoming +// policy to full capacity, migrates data according to the configured strategy, +// makes it active, and puts the outgoing policy onto shadow duty. +// +// The order of those steps is load-bearing, and the rule generalises: +// +// Every mutation of a policy must happen while that policy is not the +// active one. +// +// So the incoming policy is resized and migrated into before it is made +// active, and the outgoing policy has its values dropped only after it has +// stopped being active. Reversing either half would let a caller observe a +// policy mid-rewrite - most damagingly, read a dropped value and take the zero +// for real data. The rule is what keeps that impossible, and it is what any +// future move to lock-free reads would rest on: a reader can only ever hold a +// policy that is not being mutated. +// +// The capacity is restored before migrateData runs for the same reason it is +// restored at all: a warm migration must copy into a full-size policy rather +// than a miniature that would evict most of what it is handed. +// +// Demotion of the outgoing policy is deferred when a gradual window opens, +// because that window serves promotions out of the outgoing policy's real +// values; closeMigrationLocked performs it once the window closes. +// +// It must be called while the write lock is held. +func (c *AdaptiveCache[K, V]) switchLocked(from, to PolicyType) { + // Abandon any window still open from a previous switch, demoting its + // source now that nothing will promote out of it again. + c.closeMigrationLocked() + + c.promoteLockedCapacity(to) + c.migrateData(from, to) + c.activePolicy = to + + // Both policies just changed role, so what they measured in the previous + // one no longer describes them. Advice compares them from here. + delete(c.tenureStats, from) + delete(c.tenureStats, to) + + if !c.migrating { + c.demoteLocked(from) + } +} + +// closeMigrationLocked ends a gradual migration window and puts the source +// policy onto shadow duty, the demotion that was deferred while the window +// still needed the source's real values. +// +// It must be called while the write lock is held. +func (c *AdaptiveCache[K, V]) closeMigrationLocked() { + source, wasMigrating := c.migrateFrom, c.migrating + c.clearMigrationState() + + if wasMigrating && source != Undefined && source != c.activePolicy { + c.demoteLocked(source) + } +} + +// initShadowDutyLocked records each policy's nominal capacity, computes the +// miniature capacity it runs at while shadowing, and puts every policy except +// the initially active one onto shadow duty. It runs once, during +// construction, before the cache is reachable by any caller. +func (c *AdaptiveCache[K, V]) initShadowDutyLocked(rate float64, minCapacity int) { + c.nominalCap = make(map[PolicyType]int, len(c.policies)) + c.shadowCap = make(map[PolicyType]int, len(c.policies)) + + // The sample must be identical for every shadow or their hit rates are not + // comparable, so one effective rate is derived from the smallest policy: + // that is the capacity most at risk of shrinking into noise. + minNominal := 0 + for policyType, policy := range c.policies { + capacity := policy.Cap() + c.nominalCap[policyType] = capacity + if capacity > 0 && (minNominal == 0 || capacity < minNominal) { + minNominal = capacity + } + } + + _, effectiveRate := shadowCapacity(minNominal, rate, minCapacity) + c.sampler = newKeySampler[K](effectiveRate) + + for policyType := range c.policies { + capacity, _ := shadowCapacity(c.nominalCap[policyType], effectiveRate, minCapacity) + c.shadowCap[policyType] = capacity + + if policyType != c.activePolicy { + c.demoteLocked(policyType) + } + } +} diff --git a/shadow_test.go b/shadow_test.go new file mode 100644 index 0000000..d53f80a --- /dev/null +++ b/shadow_test.go @@ -0,0 +1,315 @@ +package ascache + +import ( + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// makeSampledCache builds a two-policy cache with sampling enabled at the +// given rate. Capacities are large enough that the miniature floor does not +// silently disable sampling. +func makeSampledCache(t *testing.T, rate float64, strategy MigrationStrategy) ( + *AdaptiveCache[string, int], + *mockPolicy[string, int], + *mockPolicy[string, int], +) { + t.Helper() + + lru := newMockPolicy[string, int](LRU, 100000) + lfu := newMockPolicy[string, int](LFU, 100000) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, + &mockBandit{next: LRU}, + &Settings{ + EpochDuration: 24 * time.Hour, + EvictPartialCapacityFilling: true, + MigrationStrategy: strategy, + ShadowSampleRate: rate, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + return ac, lru, lfu +} + +// --------------------------------------------------------------------------- +// Sampling: shadows track a fraction of the keyspace, the active policy all of it +// --------------------------------------------------------------------------- + +func TestSampling_ShadowTracksOnlyTheSample(t *testing.T) { + const rate = 0.05 + const keys = 5000 + + ac, lru, lfu := makeSampledCache(t, rate, MigrationCold) + + for i := 0; i < keys; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + + assert.Equal(t, keys, lru.Len(), "the active policy must hold every key") + + shadowLen := lfu.Len() + assert.InDelta(t, rate*keys, float64(shadowLen), rate*keys*0.25, + "the shadow should hold roughly %v of the keys, held %d", rate, shadowLen) + assert.Less(t, shadowLen, keys/10, "the shadow must be dramatically smaller than the active policy") +} + +func TestSampling_ActiveServesEveryKey(t *testing.T) { + const keys = 2000 + + ac, _, _ := makeSampledCache(t, 0.05, MigrationCold) + + for i := 0; i < keys; i++ { + ac.Add("key-"+strconv.Itoa(i), i*7) + } + + for i := 0; i < keys; i++ { + got, ok := ac.Get("key-" + strconv.Itoa(i)) + require.True(t, ok, "key-%d must be served regardless of sampling", i) + require.Equal(t, i*7, got, "key-%d value mismatch", i) + } +} + +// TestSampling_ArmsAreJudgedOnEqualEvidence checks the property that makes +// sampled measurement sound: the active policy is reported to the bandit over +// the same sampled substream as the shadows, so no arm carries more evidence +// than another. +func TestSampling_ArmsAreJudgedOnEqualEvidence(t *testing.T) { + const keys = 4000 + + lru := newMockPolicy[string, int](LRU, 100000) + lfu := newMockPolicy[string, int](LFU, 100000) + bandit := &recordingBandit{next: LRU} + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, + bandit, + &Settings{ + EpochDuration: 24 * time.Hour, + EvictPartialCapacityFilling: true, + ShadowSampleRate: 0.05, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + for i := 0; i < keys; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + for i := 0; i < keys; i++ { + ac.Get("key-" + strconv.Itoa(i)) + } + + _ = ac.tryChangePolicy() + + byPolicy := map[PolicyType]ShadowStats{} + for _, r := range bandit.getRecords() { + byPolicy[r.Policy] = r + } + + activeTotal := byPolicy[LRU].Hits + byPolicy[LRU].Misses + shadowTotal := byPolicy[LFU].Hits + byPolicy[LFU].Misses + + require.NotZero(t, activeTotal, "the active arm must report something") + assert.Equal(t, activeTotal, shadowTotal, + "both arms must be measured over the identical sampled substream") + assert.Less(t, activeTotal, int64(keys/10), + "the reported evidence must be the sample, not the full traffic") + + // Stats(), unlike the bandit report, covers everything the cache served. + stats := ac.Stats() + assert.Equal(t, int64(keys), stats.Hits+stats.Misses, + "Stats must report real traffic, not the sample") +} + +func TestSampling_DisabledByDefault(t *testing.T) { + const keys = 500 + + ac, lru, lfu := makeCache4(t, MigrationCold) + + for i := 0; i < keys; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + + assert.Equal(t, keys, lru.Len(), "active policy holds every key") + assert.Equal(t, keys, lfu.Len(), + "with the default settings the shadow must mirror every key, as before") +} + +// TestSampling_SmallCacheDisablesSampling checks the degenerate end: a cache +// too small to host a meaningful miniature falls back to mirroring rather than +// measuring noise. +func TestSampling_SmallCacheDisablesSampling(t *testing.T) { + lru := newMockPolicy[string, int](LRU, 50) + lfu := newMockPolicy[string, int](LFU, 50) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, + &mockBandit{next: LRU}, + &Settings{ + EpochDuration: 24 * time.Hour, + EvictPartialCapacityFilling: true, + ShadowSampleRate: 0.01, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + assert.False(t, ac.sampler.sampling, + "a 50-entry cache cannot host a useful miniature, so sampling must disable itself") + + for i := 0; i < 40; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + assert.Equal(t, 40, lfu.Len(), "with sampling disabled the shadow mirrors every key") +} + +// --------------------------------------------------------------------------- +// Demotion: values are dropped, bookkeeping is kept +// --------------------------------------------------------------------------- + +func TestDemotion_DropsValuesButKeepsKeys(t *testing.T) { + ac, lru, _ := makeCache4(t, MigrationCold) + + const keys = 200 + for i := 0; i < keys; i++ { + ac.Add("key-"+strconv.Itoa(i), i+1) + } + require.Equal(t, keys, lru.Len()) + + triggerSwitch(ac, LFU) + + assert.Equal(t, keys, lru.Len(), + "a demoted policy must keep its keys: they are its eviction bookkeeping") + + lru.mu.Lock() + defer lru.mu.Unlock() + for key, value := range lru.data { + require.Zero(t, value, + "a demoted policy must not keep holding real values (key %q)", key) + } +} + +// TestDemotion_NeverLeaksAZeroToACaller is the invariant everything else +// protects: dropping a demoted policy's values must never make a caller see a +// zero as if it were cached data, under any migration strategy. +func TestDemotion_NeverLeaksAZeroToACaller(t *testing.T) { + strategies := map[string]MigrationStrategy{ + "cold": MigrationCold, + "warm": MigrationWarm, + "gradual": MigrationGradual, + } + + for name, strategy := range strategies { + t.Run(name, func(t *testing.T) { + ac, _, _ := makeCache4(t, strategy) + + const keys = 100 + for i := 1; i <= keys; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + + triggerSwitch(ac, LFU) + + for i := 1; i <= keys; i++ { + key := "key-" + strconv.Itoa(i) + got, ok := ac.Get(key) + if !ok { + // A miss is always a legitimate cache answer. + continue + } + require.Equal(t, i, got, + "%s: %q returned a value that was never stored - a dropped value leaked", name, key) + } + }) + } +} + +func TestPromotion_RestoresFullCapacity(t *testing.T) { + ac, lru, lfu := makeSampledCache(t, 0.05, MigrationCold) + + nominal := ac.nominalCap[LFU] + require.Positive(t, nominal) + require.Less(t, lfu.Cap(), nominal, "a shadow must start at miniature capacity") + + triggerSwitch(ac, LFU) + + assert.Equal(t, nominal, lfu.Cap(), "a promoted policy must be restored to full capacity") + assert.Less(t, lru.Cap(), nominal, "the demoted policy must shrink to miniature capacity") +} + +func TestDemotion_DeferredDuringGradualWindow(t *testing.T) { + ac, lru, _ := makeCache4(t, MigrationGradual) + + for i := 1; i <= 20; i++ { + ac.Add("key-"+strconv.Itoa(i), i) + } + + triggerSwitch(ac, LFU) + require.True(t, ac.migrating, "expected a gradual window to open") + + // While the window is open the source must still hold real values, or + // promotion would hand callers zeros. + lru.mu.Lock() + nonZero := 0 + for _, v := range lru.data { + if v != 0 { + nonZero++ + } + } + lru.mu.Unlock() + assert.Positive(t, nonZero, "the gradual source must keep its values while the window is open") + + ac.Purge() + + ac.mu.RLock() + migrating := ac.migrating + ac.mu.RUnlock() + assert.False(t, migrating, "Purge must close the window") +} + +// makeCache4 is makeCache with capacities large enough that the miniature +// capacity floor does not disable sampling, and sampling left at the default. +func makeCache4(t *testing.T, strategy MigrationStrategy) ( + *AdaptiveCache[string, int], + *mockPolicy[string, int], + *mockPolicy[string, int], +) { + t.Helper() + + lru := newMockPolicy[string, int](LRU, 100000) + lfu := newMockPolicy[string, int](LFU, 100000) + + ac, err := NewAdaptiveCache( + []Policy[string, int]{lru, lfu}, + &mockBandit{next: LRU}, + &Settings{ + EpochDuration: 24 * time.Hour, + EvictPartialCapacityFilling: true, + MigrationStrategy: strategy, + }, + ) + require.NoError(t, err) + t.Cleanup(func() { _ = ac.Close() }) + + return ac, lru, lfu +} + +// --------------------------------------------------------------------------- +// Resize keeps shadows miniature +// --------------------------------------------------------------------------- + +func TestResize_KeepsShadowsMiniature(t *testing.T) { + ac, lru, lfu := makeSampledCache(t, 0.05, MigrationCold) + + ac.Resize(200000) + + assert.Equal(t, 200000, lru.Cap(), "the active policy takes the requested capacity") + assert.Equal(t, 10000, lfu.Cap(), "a shadow takes the miniature capacity for that size") +} diff --git a/stability.go b/stability.go new file mode 100644 index 0000000..12accaf --- /dev/null +++ b/stability.go @@ -0,0 +1,60 @@ +package ascache + +// hitRate returns the fraction of requests that were hits, or 0 when no +// requests were observed. +func hitRate(s PolicyStats) float64 { + total := s.Hits + s.Misses + if total == 0 { + return 0 + } + + return float64(s.Hits) / float64(total) +} + +// switchGated reports whether the stability settings are configured to gate +// switches at all. When none are set, AdaptiveCache applies every bandit +// selection, which is the behaviour of a zero-valued Settings. +func (s *Settings) switchGated() bool { + return s.MinHitRateImprovement > 0 || s.SwitchCooldownEpochs > 0 || s.MinEpochRequests > 0 +} + +// allowSwitchLocked reports whether the bandit's selection of candidate should +// actually be applied, given the stability settings and the stats measured in +// the epoch that just ended. A rejected switch leaves the active policy in +// place; the bandit still keeps the posterior it learned this epoch, so a +// genuinely better policy wins again on a later epoch. +// +// It must be called while the write lock is held, immediately after +// selectPolicyLocked, which populates epochStats. +func (c *AdaptiveCache[K, V]) allowSwitchLocked(candidate PolicyType) bool { + if !c.settings.switchGated() { + return true + } + + if c.settings.SwitchCooldownEpochs > 0 && + c.epochID-c.lastSwitchEpoch < c.settings.SwitchCooldownEpochs { + return false + } + + active, okActive := c.epochStats[c.activePolicy] + cand, okCandidate := c.epochStats[candidate] + if !okActive || !okCandidate { + // The epoch produced no comparable measurement (see the + // EvictPartialCapacityFilling gate in selectPolicyLocked). Hold the + // current policy rather than switch on no evidence. + return false + } + + if c.settings.MinEpochRequests > 0 && + (active.Hits+active.Misses < c.settings.MinEpochRequests || + cand.Hits+cand.Misses < c.settings.MinEpochRequests) { + return false + } + + if c.settings.MinHitRateImprovement > 0 && + hitRate(cand)-hitRate(active) < c.settings.MinHitRateImprovement { + return false + } + + return true +} diff --git a/wrapper.go b/wrapper.go index 940db90..5b89c64 100644 --- a/wrapper.go +++ b/wrapper.go @@ -2,6 +2,7 @@ package ascache import ( "strings" + "sync/atomic" ) func NewCache[K comparable, V any]( @@ -9,33 +10,50 @@ func NewCache[K comparable, V any]( policy PolicyType, size int, ) *CacheWrapper[K, V] { - return &CacheWrapper[K, V]{ + w := &CacheWrapper[K, V]{ Cacher: cache, policy: policy, - size: size, - stats: PolicyStats{}, } + w.size.Store(int64(size)) + + return w } type CacheWrapper[K comparable, V any] struct { Cacher[K, V] - size int + // size tracks the wrapped cache's capacity. Resize updates it, and Cap + // may be read concurrently with a resize, so it is atomic. + size atomic.Int64 policy PolicyType - stats PolicyStats + // hits and misses are updated from Get, which callers may invoke + // concurrently (AdaptiveCache.Get holds only a read lock), so they must be + // mutated atomically. + hits atomic.Int64 + misses atomic.Int64 } func (c *CacheWrapper[K, V]) Get(key K) (value V, ok bool) { value, ok = c.Cacher.Get(key) if ok { - c.stats.Hits++ + c.hits.Add(1) } else { - c.stats.Misses++ + c.misses.Add(1) } return } func (c *CacheWrapper[K, V]) Cap() int { - return c.size + return int(c.size.Load()) +} + +// Resize changes the wrapped cache's capacity and keeps Cap in step with it. +// The embedded Cacher's Resize would otherwise be promoted directly, leaving +// Cap reporting the capacity the wrapper was built with forever. +func (c *CacheWrapper[K, V]) Resize(size int) int { + evicted := c.Cacher.Resize(size) + c.size.Store(int64(size)) + + return evicted } func (c *CacheWrapper[K, V]) Name() string { @@ -47,9 +65,13 @@ func (c *CacheWrapper[K, V]) GetType() PolicyType { } func (c *CacheWrapper[K, V]) GetStats() PolicyStats { - return c.stats + return PolicyStats{ + Hits: c.hits.Load(), + Misses: c.misses.Load(), + } } func (c *CacheWrapper[K, V]) ResetStats() { - c.stats = PolicyStats{} + c.hits.Store(0) + c.misses.Store(0) }