l1: default-route unresolvable S3 backend selectors instead of rejecting

cache/s3proxy/multi.go

@@ -27,11 +27,12 @@ import (
 
 var (
 	// backendUnknown is belt-and-braces coverage for interceptor/config
-	// drift and stays flat in practice: the gRPC interceptor rejects
-	// unknown selectors at the boundary, and the HTTP listener carries no
-	// selector at all. Do NOT alert on it — the interceptor rejection
-	// counters (bazel_remote_s3_backend_selector_rejected_total{reason})
-	// are the signal.
+	// drift and stays flat in practice: the gRPC interceptor resolves every
+	// selector to a configured key (unresolvable ones route to the default
+	// backend), and the HTTP listener carries no selector at all. Do NOT
+	// alert on it — the interceptor's defaulting counter
+	// (bazel_remote_s3_backend_selector_defaulted_total{reason}) is the
+	// drift signal.
 	backendUnknown = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "bazel_remote_s3_backend_unknown_total",
 		Help: "Requests carrying a backend selector not present in the configured backends map (refused; indicates an interceptor/config mismatch).",

cache/trust_rejection.go

@@ -22,9 +22,13 @@ const TrustRejectionErrorDomain = "cache.blacksmith.sh"
 // cause (missing/duplicate/unknown/invalid) travels in ErrorInfo.Metadata
 // under "cause".
 const (
-	// RejectionReasonS3BackendSelector marks rejections from the S3
+	// RejectionReasonS3BackendSelector marked rejections from the S3
 	// backend-selector trust interceptor (missing, duplicate, or
-	// non-allowlisted x-blacksmith-s3-endpoint metadata).
+	// non-allowlisted x-blacksmith-s3-endpoint metadata). Current L1s no
+	// longer mint it — unresolvable selectors route to the default backend
+	// instead of rejecting — but the constant stays: L1s predating the
+	// change still send it, and upstream clients must keep degrading it to
+	// a metered miss.
 	RejectionReasonS3BackendSelector = "S3_BACKEND_SELECTOR_REJECTED"
 
 	// RejectionReasonStoragePrefix marks rejections from the storage-prefix

config/s3.go

@@ -140,20 +140,46 @@ type S3BackendConfig struct {
 	Default         bool     `yaml:"default"`
 }
 
-// AllowedBackends returns, for each valid backend selector, the bucket set
-// the fail-closed gRPC interceptor accepts for it: the entry's default
-// bucket plus its extra_buckets. Only valid to call after validateConfig has
-// passed (bucket resolution cannot fail then).
-func (s3c *S3CloudStorageConfig) AllowedBackends() (map[string]map[string]bool, error) {
-	allowed := make(map[string]map[string]bool, len(s3c.Backends))
+// S3BackendRoutingEntry is one backends-map entry as the gRPC routing
+// interceptor needs it: the entry's resolved default bucket (used when a
+// request carries no usable bucket) and the full allowed set (default plus
+// extra_buckets).
+type S3BackendRoutingEntry struct {
+	DefaultBucket string
+	Buckets       map[string]bool
+}
+
+// RoutingBackends resolves the backends map for the gRPC routing
+// interceptor: for each selector key, its default bucket and allowed bucket
+// set; plus the map's designated default key, which serves every request
+// whose selector is missing or not in the map. Exactly one entry must be
+// marked default (the same invariant s3proxy.NewMulti enforces). Only valid
+// to call after validateConfig has passed (bucket resolution cannot fail
+// then).
+func (s3c *S3CloudStorageConfig) RoutingBackends() (defaultKey string, entries map[string]S3BackendRoutingEntry, err error) {
+	entries = make(map[string]S3BackendRoutingEntry, len(s3c.Backends))
 	for key := range s3c.Backends {
 		buckets, err := s3c.allowedBucketsForBackend(key)
 		if err != nil {
-			return nil, err
+			return "", nil, err
 		}
-		allowed[key] = buckets
+		backend := s3c.Backends[key]
+		bucket := backend.Bucket
+		if bucket == "" {
+			bucket = s3c.Bucket
+		}
+		entries[key] = S3BackendRoutingEntry{DefaultBucket: bucket, Buckets: buckets}
+		if backend.Default {
+			if defaultKey != "" {
+				return "", nil, fmt.Errorf("multiple s3.backends entries marked as default")
+			}
+			defaultKey = key
+		}
+	}
+	if defaultKey == "" {
+		return "", nil, fmt.Errorf("no s3.backends entry marked as default")
 	}
-	return allowed, nil
+	return defaultKey, entries, nil
 }
 
 // allowedBucketsForBackend resolves one backends-map entry's allowed bucket

config/s3_test.go

@@ -48,27 +48,37 @@ func TestS3BackendsMapConfig(t *testing.T) {
 		t.Fatalf("ConnRecycleInterval = %v, want 1m", s3.ConnRecycleInterval)
 	}
 
-	// The allowlist pairs each selector with its bucket set: the entry's
-	// (possibly inherited) default bucket plus any extra_buckets.
-	allowed, err := s3.AllowedBackends()
+	// The routing table pairs each selector with its default bucket and
+	// bucket set (the entry's — possibly inherited — default bucket plus any
+	// extra_buckets), and names the map's default key.
+	defaultKey, entries, err := s3.RoutingBackends()
 	if err != nil {
 		t.Fatal(err)
 	}
-	wantBuckets := map[string][]string{
-		"http://minio-a.example.com:9000":  {"shared-bucket"},
-		"https://minio-b.example.com:9000": {"bucket-b", "bucket-b-pre-rename"},
+	if defaultKey != "http://minio-a.example.com:9000" {
+		t.Fatalf("default key = %q, want backend a", defaultKey)
 	}
-	for key, buckets := range wantBuckets {
-		got, ok := allowed[key]
+	want := map[string]struct {
+		defaultBucket string
+		buckets       []string
+	}{
+		"http://minio-a.example.com:9000":  {"shared-bucket", []string{"shared-bucket"}},
+		"https://minio-b.example.com:9000": {"bucket-b", []string{"bucket-b", "bucket-b-pre-rename"}},
+	}
+	for key, w := range want {
+		got, ok := entries[key]
 		if !ok {
-			t.Fatalf("expected %q in allowed backends %v", key, allowed)
+			t.Fatalf("expected %q in routing backends %v", key, entries)
+		}
+		if got.DefaultBucket != w.defaultBucket {
+			t.Fatalf("backend %q default bucket = %q, want %q", key, got.DefaultBucket, w.defaultBucket)
 		}
-		if len(got) != len(buckets) {
-			t.Fatalf("backend %q allowed buckets = %v, want %v", key, got, buckets)
+		if len(got.Buckets) != len(w.buckets) {
+			t.Fatalf("backend %q allowed buckets = %v, want %v", key, got.Buckets, w.buckets)
 		}
-		for _, bucket := range buckets {
-			if !got[bucket] {
-				t.Fatalf("backend %q allowed buckets = %v, missing %q", key, got, bucket)
+		for _, bucket := range w.buckets {
+			if !got.Buckets[bucket] {
+				t.Fatalf("backend %q allowed buckets = %v, missing %q", key, got.Buckets, bucket)
 			}
 		}
 	}

main.go

@@ -497,21 +497,35 @@ func startGrpcServer(c *config.Config, grpcServer **grpc.Server,
 		unaryInterceptors = append(unaryInterceptors, server.GRPCStoragePrefixUnaryServerInterceptor(authSecret))
 	}
 
-	// Multi-backend S3 mode: every cache RPC must carry exactly one
-	// allowlisted (endpoint, bucket) pair (the tenant's pinned backing-store
-	// endpoint and bucket, forwarded by the trusted upstream) or it is
-	// rejected fail-closed — same trust model as the storage prefix above.
-	// Health and capabilities RPCs are exempt. Only installed when a
-	// backends map is configured; single-backend deployments ignore both
-	// metadata keys.
+	// Multi-backend S3 mode: cache RPCs carry the tenant's pinned
+	// backing-store (endpoint, bucket) pair as gRPC metadata, forwarded by
+	// the trusted upstream. Requests resolve against the backends map;
+	// anything unresolvable (missing/unknown selector or bucket) routes to
+	// the map's default backend / the entry's default bucket, metered per
+	// cause — the L1 owns which backends exist, the upstream needs no
+	// endpoint knowledge of its own to be served. Tenant isolation is the
+	// storage-prefix interceptor's job (fail-closed above); the selector
+	// only picks the shard. Health and capabilities RPCs are exempt. Only
+	// installed when a backends map is configured; single-backend
+	// deployments ignore both metadata keys.
 	if c.S3CloudStorage != nil && len(c.S3CloudStorage.Backends) > 0 {
-		allowed, err := c.S3CloudStorage.AllowedBackends()
+		defaultKey, entries, err := c.S3CloudStorage.RoutingBackends()
 		if err != nil {
 			return err
 		}
-		log.Printf("Routing S3 operations by forwarded (endpoint, bucket) gRPC metadata, fail-closed (%d allowlisted backends)", len(allowed))
-		streamInterceptors = append(streamInterceptors, server.GRPCS3BackendStreamServerInterceptor(allowed))
-		unaryInterceptors = append(unaryInterceptors, server.GRPCS3BackendUnaryServerInterceptor(allowed))
+		routing := server.S3BackendRouting{
+			DefaultKey: defaultKey,
+			Backends:   make(map[string]server.S3BackendRoutingEntry, len(entries)),
+		}
+		for key, entry := range entries {
+			routing.Backends[key] = server.S3BackendRoutingEntry{
+				DefaultBucket: entry.DefaultBucket,
+				Buckets:       entry.Buckets,
+			}
+		}
+		log.Printf("Routing S3 operations by forwarded (endpoint, bucket) gRPC metadata (%d backends, default %q; unresolvable selectors route to the default)", len(entries), defaultKey)
+		streamInterceptors = append(streamInterceptors, server.GRPCS3BackendStreamServerInterceptor(routing))
+		unaryInterceptors = append(unaryInterceptors, server.GRPCS3BackendUnaryServerInterceptor(routing))
 	}
 
 	if c.TLSConfig != nil {