From 651dec3b164bf21b11e47f8b433fc18b898c92bf Mon Sep 17 00:00:00 2001 From: Shreyas Kalyan Date: Fri, 18 Sep 2026 17:26:02 -0400 Subject: [PATCH] census: bucket never-read evictions by created age A never-read entry evicted minutes after writing means the cache could not hold a write until its first read - the worst eviction outcome. It was counted only in the aggregate never-read counter (no age dimension) and the created bands (mixed with read entries), so a young never-read eviction rendered as a healthy-looking last_read mix. New age_kind=never_read series on evicted_bytes_by_age, bucketed by created age. created stays the superset; last_read stays the pure live-harm signal. Co-authored-by: Cursor --- cache/disk/census.go | 17 +++++++++++------ cache/disk/census_test.go | 8 ++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cache/disk/census.go b/cache/disk/census.go index d940184..1707157 100644 --- a/cache/disk/census.go +++ b/cache/disk/census.go @@ -137,7 +137,7 @@ func newCensusRecorder() *censusRecorder { evictedBytesByAge: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "bazel_remote_disk_cache_evicted_bytes_by_age", - Help: "Bytes evicted from the disk cache, bucketed by age at eviction. age_kind is last_read (time since the entry was last read: evicted still-live bytes are the harm signal) or created (time since the entry was written: write-once bytes dying young are the garbage signal).", + Help: "Bytes evicted from the disk cache, bucketed by age at eviction. age_kind is last_read (time since the entry was last read: evicted still-live bytes are the harm signal), never_read (bucketed by created age, never-read entries only: young bands mean writes evicted before their first read), or created (time since the entry was written, every eviction: the superset of the other two).", }, []string{"age_kind", "age_bucket", "tool"}), accessIntervalByAge: prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -271,13 +271,18 @@ func (r *censusRecorder) OnEvict(key string, value lruItem) { sz := float64(value.sizeOnDisk) // Never-read entries have no last read: they are counted in the - // never-read counter and the created-age buckets only, so the - // last_read series stays a pure harm signal (bytes someone actually - // used, evicted anyway). This also gives thrash alerts built-in - // restart tolerance: boot-scanned entries look never-read until first - // touched, so a roll cannot fabricate still-live evictions. + // never-read counter, a never_read age series (bucketed by CREATED + // age — a never-read entry evicted minutes after writing means the + // cache could not hold a write until its first read, and burying it + // in the aggregate created bands made that failure look healthy), + // and the created-age buckets. The last_read series stays a pure + // harm signal (bytes someone actually used, evicted anyway). This + // also gives thrash alerts built-in restart tolerance: boot-scanned + // entries look never-read until first touched, so a roll cannot + // fabricate still-live evictions. if neverRead { r.evictedNeverReadBytes.WithLabelValues(tool).Add(sz) + r.evictedBytesByAge.WithLabelValues("never_read", censusBucketLabels[createdIdx], tool).Add(sz) } else { r.evictedBytesByAge.WithLabelValues("last_read", censusBucketLabels[lastReadIdx], tool).Add(sz) } diff --git a/cache/disk/census_test.go b/cache/disk/census_test.go index 51caa79..0d0858c 100644 --- a/cache/disk/census_test.go +++ b/cache/disk/census_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/buchgr/bazel-remote/v2/cache" + + "github.com/prometheus/client_golang/prometheus/testutil" ) func TestCensusBucketIdx(t *testing.T) { @@ -153,6 +155,12 @@ func TestCensusAccounting(t *testing.T) { if rows3[0].EvictedNeverReadBytes != 200 { t.Errorf("EvictedNeverReadBytes = %d, want 200", rows3[0].EvictedNeverReadBytes) } + // The never_read Prometheus series buckets by CREATED age (14h -> 24h + // band): a young never-read eviction must be visible as such, not + // buried in the aggregate created bands. + if got := testutil.ToFloat64(rec.evictedBytesByAge.WithLabelValues("never_read", "24h", "go")); got != 200 { + t.Errorf("never_read 24h band = %v, want 200", got) + } // B's last read is its creation stamp, 14h before eviction: not live. if rows3[0].EvictedLiveBytes != 0 { t.Errorf("EvictedLiveBytes = %d, want 0", rows3[0].EvictedLiveBytes)