Serve the cache from local disk and sync with the NFS archive

.github/workflows/docker.yml

@@ -61,11 +61,17 @@ jobs:
         CACHE_STALE_TIME: 6M
         DNS_RESOLVER: 8.8.8.8
       run: |
+        # A fake archive with one entry: the container must restore it into
+        # the local cache in the background and report that in /status.
+        mkdir -p test-archive/owlery/0/00
+        printf 'KEY: GET/probe\nHTTP/1.1 200 OK\n\nprobe\n' > test-archive/owlery/0/00/00000000000000000000000000000001
+        chmod -R a+rwX test-archive   # the sync scripts run as the nginx user
         docker run -d --name test-container \
           -e UPSTREAM_SERVER \
           -e CACHE_MAX_SIZE \
           -e CACHE_STALE_TIME \
           -e DNS_RESOLVER \
+          -v $(pwd)/test-archive:/cache \
           test-image
         sleep 5
         if docker ps | grep -q test-container; then
@@ -75,8 +81,13 @@ jobs:
           docker logs test-container
           exit 1
         fi
+        docker logs test-container | grep -q 'Scheduled cache backup: crontab' || { echo "backup not scheduled"; docker logs test-container; exit 1; }
+        docker exec test-container test -f /var/cache/nginx/owlery/0/00/00000000000000000000000000000001 || { echo "archive entry not restored"; docker logs test-container; exit 1; }
+        docker exec test-container wget -q -O - http://localhost/status | grep -q '"restore": {' || { echo "/status lacks archive.restore"; docker exec test-container wget -q -O - http://localhost/status; exit 1; }
+        docker exec test-container cache-backup.sh --full | grep -q 'backup: done' || { echo "manual backup failed"; exit 1; }
         docker stop test-container || true
         docker rm test-container
+        sudo rm -rf test-archive
 
     - name: Build Docker image
       run: docker build . --file Dockerfile --tag virtualflybrain/owl_cache:${{ steps.meta.outputs.tag }}

Dockerfile

@@ -10,6 +10,10 @@ ENV DNS_RESOLVER=8.8.8.8
 ENV WORKER_PROCESSES=auto
 ENV WORKER_CONNECTIONS=4096
 ENV WORKER_RLIMIT_NOFILE=65535
+ENV CACHE_ARCHIVE_DIR=/cache
+ENV CACHE_BACKUP_SCHEDULE=daily
+ENV CACHE_BACKUP_TIME=03:00
+ENV CACHE_BACKUP_JITTER_MINUTES=120
 
 ARG NGINX_CONF=nginx.conf.template
 COPY $NGINX_CONF /etc/nginx/nginx.conf.template
@@ -18,20 +22,36 @@ COPY health-monitor.sh /usr/local/bin/health-monitor.sh
 COPY test/ip-maps-test.sh /usr/local/bin/ip-maps-test.sh
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 COPY purge-cached-404s.sh /usr/local/bin/purge-cached-404s.sh
+COPY cache-lib.sh /usr/local/bin/cache-lib.sh
+COPY cache-restore.sh /usr/local/bin/cache-restore.sh
+COPY cache-backup.sh /usr/local/bin/cache-backup.sh
+COPY test/cache-sync-test.sh /usr/local/bin/cache-sync-test.sh
 
 RUN mkdir -p /var/cache/nginx/owlery /logs/hacks && \
     touch /logs/blocked.txt /logs/whitelist.txt /etc/nginx/blocked-ips.map /etc/nginx/whitelisted-ips.map /etc/nginx/whitelisted-cidrs.map && \
     chown -R nginx:nginx /var/cache/nginx /logs && \
-    chmod +x /usr/local/bin/health-monitor.sh /usr/local/bin/docker-entrypoint.sh /usr/local/bin/purge-cached-404s.sh /usr/local/bin/ip-maps-test.sh && \
-    apk add --no-cache gettext
+    chmod +x /usr/local/bin/health-monitor.sh /usr/local/bin/docker-entrypoint.sh /usr/local/bin/purge-cached-404s.sh /usr/local/bin/ip-maps-test.sh \
+             /usr/local/bin/cache-restore.sh /usr/local/bin/cache-backup.sh /usr/local/bin/cache-sync-test.sh && \
+    # rsync moves cache entries between the shared archive and local disk;
+    # GNU findutils gives `find -printf`, which lists millions of entries with
+    # their mtime and size in one walk (BusyBox find would need a stat per file).
+    # su-exec drops the sync scripts to the nginx user (root-squashed NFS).
+    apk add --no-cache gettext rsync findutils su-exec
 
 # Fail the build rather than the deployment: a whitelist entry that the map
 # compiler quietly discards is invisible until someone notices a cache bypass
 # not happening, so the list-compilation logic is unit tested here.
 RUN IP_MAPS_LIB=/usr/local/bin/ip-maps.sh /usr/local/bin/ip-maps-test.sh
 
+# Same reasoning for the archive sync: a filter regression would silently copy
+# in-flight temp files (served as truncated responses) or skip real entries.
+RUN CACHE_RUN_AS= CACHE_LIB=/usr/local/bin/cache-lib.sh \
+    CACHE_RESTORE_SCRIPT=/usr/local/bin/cache-restore.sh \
+    CACHE_BACKUP_SCRIPT=/usr/local/bin/cache-backup.sh \
+    /usr/local/bin/cache-sync-test.sh
+
 EXPOSE 80 8080
 
-VOLUME ["/var/cache/nginx", "/logs"]
+VOLUME ["/var/cache/nginx", "/cache", "/logs"]
 
 CMD ["/usr/local/bin/docker-entrypoint.sh"]

README.md

@@ -32,14 +32,17 @@ services:
       - "80:80"
       - "8080:8080"
     volumes:
-      - /cache:/var/cache/nginx
+      - /data/owl-cache:/var/cache/nginx   # local node disk: what NGINX serves from
+      - /cache:/cache                      # shared NFS archive: restored from at start, backed up into daily
       - /logs:/logs
     environment:
       - UPSTREAM_SERVER=owl:8080  # For production with owl service
-      - CACHE_MAX_SIZE=1t         # 1TB cache size for high-traffic deployments
+      - CACHE_MAX_SIZE=1t         # bound on the LOCAL cache; size it to the node disk
       - DNS_RESOLVER=169.254.169.250  # Rancher internal DNS (check /etc/resolv.conf)
 ```
 
+See [Cache archive](#cache-archive) for why the cache is split across two volumes.
+
 ### Health Check
 
 ```bash
@@ -99,7 +102,7 @@ Example response:
 - `HEALTH_LOG_INTERVAL`: Seconds between periodic upstream health log lines when state is unchanged (default: `300`)
 - `AUTO_BLOCK_SCANNERS`: Automatically append probe-source IPs from `/logs/hacks/probes.log` to `/logs/blocked.txt` and live-reload NGINX maps (default: `true`)
 - `FORCE_CACHE_REFRESH_ON_REQUEST`: When `true`, each incoming request bypasses the cache and fetches fresh content from upstream, updating the cache on demand instead of serving cached entries.
-- `WORKER_PROCESSES`: Number of NGINX worker processes (default: `auto`). `auto` spawns one worker per host CPU core, but it reads the host's online core count and **ignores the container's cgroup CPU quota** — on a shared/Rancher host pin this to the CPU reservation (e.g. `2`) so you don't over-spawn workers that can't run in parallel. A single container with a coherent local cache is the only safe way to share one cache directory across workers; do not point multiple containers at the same cache volume.
+- `WORKER_PROCESSES`: Number of NGINX worker processes (default: `auto`). `auto` spawns one worker per host CPU core, but it reads the host's online core count and **ignores the container's cgroup CPU quota** — on a shared/Rancher host pin this to the CPU reservation (e.g. `2`) so you don't over-spawn workers that can't run in parallel. A single container with a coherent local cache is the only safe way to share one cache directory across workers; do not point multiple containers at the same `/var/cache/nginx` volume — share the archive at `/cache` instead (see [Cache archive](#cache-archive)).
 - `WORKER_CONNECTIONS`: Max simultaneous connections per worker (default: `4096`). Effective client concurrency is roughly `WORKER_PROCESSES × WORKER_CONNECTIONS`, halved on cache MISS since each client connection also opens an upstream connection.
 - `WORKER_RLIMIT_NOFILE`: Per-worker open-file-descriptor ceiling (default: `65535`). Each client connection plus every open cached file uses a descriptor, so the OS default of 1024 throttles a busy cache. Must stay within the container's hard `nofile` ulimit — NGINX logs a warning and caps to the runtime limit if this is higher.
 
@@ -157,6 +160,75 @@ The proxy adds helpful headers to responses:
 - `X-Cache-Status`: `HIT`, `MISS`, `EXPIRED`, `STALE`, `UPDATING`, or `REVALIDATED`
 - `X-Cache-Key`: The cache key used for the request
 
+### Cache archive
+
+NGINX serves the cache from `/var/cache/nginx` on local node disk. Serving
+directly from an NFS volume costs an `open()`/`stat()` round trip per hit and
+a metadata walk over millions of entries for the cache manager, and NGINX
+cannot share one cache directory between instances anyway (the index lives in
+each instance's shared memory). The shared NFS volume mounted at `/cache`
+is therefore an **archive**, not the live cache: every instance restores from
+it at start and backs up into it on a schedule.
+
+Both directions follow one rule. NGINX names each entry by the MD5 of its
+cache key and replaces entries atomically, so the union of several instances'
+caches is itself a valid cache. Files are only ever copied with
+`rsync --update` (skip when the destination is newer) and nothing is ever
+deleted, so the archive is the union of everything any instance has cached,
+with the newest version of each entry winning. The comparison is by mtime,
+so the nodes and the NAS must agree on time (NTP); `--modify-window=2`
+absorbs filesystem timestamp granularity, not clock skew.
+
+**Restore** (`cache-restore.sh`, started by the entrypoint in the background)
+walks the archive once, sorts entries newest first, and copies them in
+batches while NGINX is already serving. NGINX serves a cache file that
+appears on disk after it has started (verified against 1.26: a lookup that
+misses the in-memory index still opens and validates the file), so startup
+never waits for the copy. Requests whose entry has not landed yet are
+ordinary misses. Progress is reported under `archive.restore` in `/status`.
+A restore of ~1 TB / millions of files takes hours; an instance is fully
+warm when `archive.restore.state` is `done`. If the local volume persists
+across restarts, the `.restored` marker makes later starts skip the restore
+(`CACHE_RESTORE=always` forces it; `off` disables it).
+
+**Backup** (`cache-backup.sh`) lists entries written since the previous run
+from the local disk (`find -newer`; the NFS side is never walked) and copies
+them into the archive. `crond` inside the container runs it daily at
+`CACHE_BACKUP_TIME` (or weekly on `CACHE_BACKUP_WEEKDAY`). When a service is
+scaled to several containers they all share one environment, so each adds a
+deterministic per-host offset of up to `CACHE_BACKUP_JITTER_MINUTES` to spread
+the NAS load; a `mkdir`-based lock on the archive serialises any that still
+overlap (a lock older than `CACHE_LOCK_STALE_MINUTES` is treated as
+abandoned). Run it by hand at any time:
+
+```bash
+docker exec owlery-cache cache-backup.sh          # entries new since the last run
+docker exec owlery-cache cache-backup.sh --full   # consider every local entry
+docker exec owlery-cache cache-restore.sh --force # re-pull the archive now
+curl -s http://localhost/status | jq .archive     # progress of both
+```
+
+Loss window: a container that dies loses whatever it cached since its last
+backup (at most one day on the default schedule); everything older is in the
+archive and comes back on the next restore.
+
+Warm-up note: the `X-Force-Refresh` warm-up tool only refreshes the instance
+that the load balancer routes it to. With several instances, warm one, run
+`cache-backup.sh` on it, then let the others pick the entries up on their
+next restore (or point the warm-up at each instance in turn).
+
+Variables (all optional):
+
+- `CACHE_ARCHIVE_DIR` (`/cache`), `CACHE_LOCAL_DIR` (`/var/cache/nginx`): the two roots; both hold an `owlery/` tree.
+- `CACHE_RESTORE`: `auto` (default; skip if `.restored` exists), `always`, `off`.
+- `CACHE_RESTORE_BWLIMIT`, `CACHE_BACKUP_BWLIMIT`: rsync `--bwlimit` in KiB/s (default unlimited).
+- `CACHE_RESTORE_MAX_BYTES`: stop the restore after this many bytes of the newest entries (default: whole archive).
+- `CACHE_RESTORE_BATCH`, `CACHE_BACKUP_BATCH`: entries per rsync invocation (default 5000).
+- `CACHE_BACKUP_SCHEDULE`: `daily` (default), `weekly`, `off`.
+- `CACHE_BACKUP_TIME` (`03:00`, container local time), `CACHE_BACKUP_WEEKDAY` (`0` = Sunday).
+- `CACHE_BACKUP_JITTER_MINUTES` (`120`); `CACHE_BACKUP_CRON`: a verbatim 5-field crontab spec that overrides the above and gets no jitter.
+- `CACHE_BACKUP_LOCK_WAIT` (`120` min), `CACHE_LOCK_STALE_MINUTES` (`360`).
+
 ### Selective 404 cache eviction
 
 404 responses are not cached going forward, but a long-lived cache may still
@@ -175,7 +247,9 @@ The script identifies entries by matching the response status line
 happens to contain the text "HTTP/1.1 404" elsewhere in the body are not
 affected. Files removed from disk are simply treated as `MISS` on the next
 request — no nginx reload required. The cache directory is taken from
-`$CACHE_DIR` (default `/var/cache/nginx/owlery`).
+`$CACHE_DIR` (default `/var/cache/nginx/owlery`). Add `--archive` to walk the
+shared archive as well; without it the next restore brings the purged entries
+back.
 
 ## Performance
 
@@ -224,9 +298,9 @@ request — no nginx reload required. The cache directory is taken from
 # Pull image
 docker pull virtualflybrain/owl_cache:latest
 
-# Create cache directory
-mkdir -p /cache
-chown -R 101:101 /cache
+# Local cache on node disk, and the shared archive (NFS) it syncs with
+mkdir -p /data/owl-cache /cache
+chown -R 101:101 /data/owl-cache /cache
 
 # Create persistent logs + blocklist file
 mkdir -p /logs/hacks
@@ -245,6 +319,7 @@ curl -I http://localhost/health
 - `Dockerfile`: Image build instructions
 - `nginx.conf.template`: NGINX configuration template
 - `docker-compose.yml`: Example deployment configuration
+- `cache-lib.sh`, `cache-restore.sh`, `cache-backup.sh`: archive ↔ local cache sync (see [Cache archive](#cache-archive)); `test/cache-sync-test.sh` runs at image build
 - `.github/workflows/docker.yml`: GitHub Actions CI/CD pipeline
 
 ## CI/CD

cache-backup.sh

@@ -0,0 +1,136 @@
+#!/bin/sh
+# Stash the local nginx cache into the shared archive.
+#
+# Only entries written since the previous backup are considered. The list is
+# built from the LOCAL disk (`find -newer <marker>`), which is cheap; the
+# archive on NFS is never walked, only written to. rsync --update then copies
+# each entry unless the archive already holds a newer copy, so several
+# instances backing up into the same archive produce a union in which the
+# newest version of every entry wins. Nothing is ever deleted from the
+# archive; use purge-cached-404s.sh --archive to remove poisoned entries.
+#
+# Scheduled by docker-entrypoint.sh through crond (see CACHE_BACKUP_SCHEDULE)
+# and runnable by hand:
+#   docker exec owlery-cache cache-backup.sh          # incremental
+#   docker exec owlery-cache cache-backup.sh --full   # ignore the marker
+#
+# Environment:
+#   CACHE_ARCHIVE_DIR          /cache
+#   CACHE_LOCAL_DIR            /var/cache/nginx
+#   CACHE_BACKUP_BWLIMIT       rsync --bwlimit in KiB/s; 0 = unlimited
+#   CACHE_BACKUP_BATCH         files per rsync invocation (default 5000)
+#   CACHE_BACKUP_LOCK_WAIT     minutes to wait for another instance's backup
+#                              to finish before giving up (default 120)
+#   CACHE_LOCK_STALE_MINUTES   lock older than this is taken over (default 360)
+
+set -eu
+
+# Run as the nginx user, never root: the archive is NFS and typically
+# root-squashed, and every file we create must be owned by the user nginx
+# runs as. CACHE_RUN_AS="" keeps the current user (tests).
+CACHE_RUN_AS="${CACHE_RUN_AS-nginx}"
+if [ -n "$CACHE_RUN_AS" ] && [ "$(id -u)" = "0" ] && command -v su-exec >/dev/null 2>&1; then
+    exec su-exec "$CACHE_RUN_AS" "$0" "$@"
+fi
+
+CACHE_LIB="${CACHE_LIB:-/usr/local/bin/cache-lib.sh}"
+# shellcheck source=cache-lib.sh
+. "$CACHE_LIB"
+
+CACHE_BACKUP_BWLIMIT="${CACHE_BACKUP_BWLIMIT:-0}"
+CACHE_BACKUP_BATCH="${CACHE_BACKUP_BATCH:-5000}"
+CACHE_BACKUP_LOCK_WAIT="${CACHE_BACKUP_LOCK_WAIT:-120}"
+
+STATE_FILE="$CACHE_STATE_DIR/cache-backup.json"
+MARKER="$CACHE_LOCAL_DIR/.last-backup"
+SRC="$CACHE_LOCAL_DIR/$CACHE_SUBDIR"
+DST="$CACHE_ARCHIVE_DIR/$CACHE_SUBDIR"
+
+full=0
+[ "${1:-}" = "--full" ] && full=1
+
+started="$(date +%s)"
+files_done=0
+bytes_done=0
+finish() {
+    cache_write_state "$STATE_FILE" "$1" "$files_done" "$bytes_done" "$started" "$(date +%s)" "$2"
+    cache_log "backup: $1 -- $2"
+}
+
+if [ ! -d "$CACHE_ARCHIVE_DIR" ]; then
+    finish skipped "archive $CACHE_ARCHIVE_DIR not mounted"; exit 0
+fi
+if [ ! -d "$SRC" ]; then
+    finish skipped "local cache $SRC does not exist yet"; exit 0
+fi
+if [ "$(cd "$SRC" && pwd -P)" = "$(mkdir -p "$DST" && cd "$DST" && pwd -P)" ]; then
+    finish skipped "archive and local cache are the same directory; nothing to back up"; exit 0
+fi
+
+work="$(mktemp -d /tmp/cache-backup.XXXXXX)"
+trap 'rm -rf "$work"' EXIT
+
+# Stamp the new marker before listing, so entries written while the backup
+# runs are picked up next time rather than falling between two runs.
+new_marker="$work/marker"
+# Back-dated a couple of seconds: find -newer is a strict comparison, and a
+# file written in the same second as the marker would otherwise be skipped.
+touch -t "$(date -d "@$(( started - 2 ))" +%Y%m%d%H%M.%S 2>/dev/null || date +%Y%m%d%H%M.%S)" "$new_marker"
+
+if [ "$full" -eq 1 ] || [ ! -f "$MARKER" ]; then
+    find "$SRC" -type f -printf '%s %P\n'
+else
+    find "$SRC" -type f -newer "$MARKER" -printf '%s %P\n'
+fi 2>/dev/null | awk '
+    { path=$2; for (i=3; i<=NF; i++) path=path " " $i }
+    path ~ /^([0-9a-f]\/[0-9a-f][0-9a-f]\/)?[0-9a-f]{32}$/ { print $1, path }
+' > "$work/entries.lst"
+
+total_files="$(wc -l < "$work/entries.lst" | tr -d ' ')"
+total_bytes="$(awk '{ s += $1 } END { print s + 0 }' "$work/entries.lst")"
+
+if [ "$total_files" -eq 0 ]; then
+    cp -p "$new_marker" "$MARKER"
+    finish "done" "nothing new since last backup"; exit 0
+fi
+
+cache_log "backup: $total_files entries ($total_bytes bytes) changed since last backup; waiting for archive lock"
+cache_write_state "$STATE_FILE" waiting 0 0 "$started" "" "waiting for lock ($total_files files pending)"
+
+if ! cache_lock_acquire "$CACHE_BACKUP_LOCK_WAIT"; then
+    finish skipped "another instance held the archive lock for more than ${CACHE_BACKUP_LOCK_WAIT} min; will retry next run"
+    exit 0
+fi
+trap 'cache_lock_release; rm -rf "$work"' EXIT
+
+cache_write_state "$STATE_FILE" running 0 0 "$started" "" "0/$total_files files"
+mkdir -p "$DST"
+# Batch files in listing order (awk rather than split: BusyBox builds differ
+# in whether split is present, and this keeps the newest-first order).
+awk -v n="$CACHE_BACKUP_BATCH" -v dir="$work" '{
+    f = sprintf("%s/batch.%08d", dir, int((NR - 1) / n)); print > f
+    if (NR % n == 0) close(f)
+}' "$work/entries.lst"
+errors=0
+for batch in "$work"/batch.*; do
+    [ -f "$batch" ] || continue
+    cut -d' ' -f2- "$batch" > "$batch.paths"
+    n="$(wc -l < "$batch" | tr -d ' ')"
+    if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_BACKUP_BWLIMIT"; then
+        # Exit 24 (vanished source file) is normal: the cache manager evicts
+        # entries under max_size while we run. Anything else is counted.
+        errors=$(( errors + 1 ))
+        cache_log "backup: rsync reported errors on batch $(basename "$batch"); continuing"
+    fi
+    files_done=$(( files_done + n ))
+    bytes_done=$(( bytes_done + $(awk '{ s += $1 } END { print s + 0 }' "$batch") ))
+    rm -f "$batch" "$batch.paths"
+    cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "$files_done/$total_files files"
+done
+
+cp -p "$new_marker" "$MARKER"
+if [ "$errors" -gt 0 ]; then
+    finish "done" "$files_done entries synchronised to $DST ($errors batches reported rsync errors)"
+else
+    finish "done" "$files_done entries synchronised to $DST"
+fi