From 58b78f8c8cab9d7e5b0456639c71d38d9d0598f9 Mon Sep 17 00:00:00 2001
From: Arnold Castro <arnoldcastro@gmail.com>
Date: Fri, 3 Jul 2026 09:55:37 -0400
Subject: [PATCH] fix(deployment): add resource requests/limits, arena cap, and
 spread tile server replicas

- Production overlay: requests cpu 100m + memory 768Mi, memory limit
  1536Mi, baselined from July 2026 production measurements (issue
  #315 investigation; cpu steady state measured 52-62 millicores).
  Without a request the pods are BestEffort: the kubelet evicts them
  first under node memory pressure and the scheduler keeps placing
  replacements on the starved node, which produced a ~90 pod eviction
  storm on July 2. With a memory limit, a leaking pod is OOM restarted
  cleanly in place instead of taking the whole node down. No cpu
  limit on purpose: throttling slows renders even on idle nodes,
  stacking up concurrent requests and raising the memory spikes this
  change exists to contain.
- Production overlay: MALLOC_ARENA_MAX=2 (verified unset in prod;
  glibc default is 8 arenas per cpu = 16 here). The render path
  churns 300-450MB native alloc/free cycles; many arenas fragment
  under that churn and RSS ratchets up over days. Capping to 2 is
  the standard mitigation and doubles as the production test of the
  ratchet hypothesis: if the slow creep flattens, mechanism found.
- Base: soft podAntiAffinity so the two replicas prefer different
  nodes and one bad node cannot take out every tile server at once.
- The existing nodeAffinity pin to microservices-node-pool is kept;
  all 5 pool nodes have headroom for the new requests.
- The V8 heap is already capped (--max-old-space-size=1024 in the
  start script since 2021); 1024Mi heap + ~500Mi native headroom is
  the budget behind the 1536Mi limit.

For Greenstand/treetracker-infrastructure#315.
---
 deployment/base/deployment.yaml               | 11 ++++++++
 .../overlays/production/deployment.yaml       | 26 +++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/deployment/base/deployment.yaml b/deployment/base/deployment.yaml
index a8510e3e22..4fac0e45c5 100644
--- a/deployment/base/deployment.yaml
+++ b/deployment/base/deployment.yaml
@@ -23,6 +23,17 @@ spec:
                 operator: In
                 values:
                 - microservices-node-pool
+        # prefer spreading replicas across nodes, so one bad node cannot
+        # take out every tile server at once (soft rule: still schedules
+        # if only one node has room)
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app: treetracker-tile-server
+              topologyKey: kubernetes.io/hostname
       containers:
         - name: treetracker-tile-server
           image: greenstand/treetracker-map-tile-server:VERSION
diff --git a/deployment/overlays/production/deployment.yaml b/deployment/overlays/production/deployment.yaml
index 26bce4b1f8..4ebab6364d 100644
--- a/deployment/overlays/production/deployment.yaml
+++ b/deployment/overlays/production/deployment.yaml
@@ -10,7 +10,33 @@ spec:
     spec:
       containers:
         - name: treetracker-tile-server
+          # memory numbers baselined from production measurements
+          # (treetracker-infrastructure issue #315, July 3 2026):
+          # steady state 510-650Mi per pod; transient render spikes up to
+          # +458Mi that Kubernetes should allow (burst headroom), and a slow
+          # leak that the limit converts from node-killing evictions into
+          # clean in-place pod restarts. Re-baseline after the cache fix.
+          # cpu: steady state measured at 52-62 millicores (July 3 2026);
+          # request 2x steady for honest scheduling and contention weight.
+          # Deliberately NO cpu limit: limits throttle even on idle nodes,
+          # slow renders stack up concurrent requests and that raises the
+          # memory spikes this file exists to contain.
+          resources:
+            requests:
+              cpu: 100m
+              memory: 768Mi
+            limits:
+              memory: 1536Mi
           env:
+            # cap glibc malloc arenas (default is 8 per cpu = 16 here).
+            # The render path constantly allocates and frees 300-450MB of
+            # native memory; with 16 arenas that churn fragments and RSS
+            # ratchets up over days without any live memory growing.
+            # Verified unset in prod on 2026-07-03. If the slow memory
+            # creep flattens after this deploys, issue #315's RSS ratchet
+            # mechanism is confirmed.
+            - name: MALLOC_ARENA_MAX
+              value: "2"
             - name: PORT
               value: "3000"
             - name: PG_POOL_SIZE