diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cb236f0..716b9f5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -61,11 +61,17 @@ jobs: CACHE_STALE_TIME: 6M DNS_RESOLVER: 8.8.8.8 run: | + # A fake archive with one entry: the container must restore it into + # the local cache in the background and report that in /status. + mkdir -p test-archive/owlery/0/00 + printf 'KEY: GET/probe\nHTTP/1.1 200 OK\n\nprobe\n' > test-archive/owlery/0/00/00000000000000000000000000000001 + chmod -R a+rwX test-archive # the sync scripts run as the nginx user docker run -d --name test-container \ -e UPSTREAM_SERVER \ -e CACHE_MAX_SIZE \ -e CACHE_STALE_TIME \ -e DNS_RESOLVER \ + -v $(pwd)/test-archive:/cache \ test-image sleep 5 if docker ps | grep -q test-container; then @@ -75,8 +81,13 @@ jobs: docker logs test-container exit 1 fi + docker logs test-container | grep -q 'Scheduled cache backup: crontab' || { echo "backup not scheduled"; docker logs test-container; exit 1; } + docker exec test-container test -f /var/cache/nginx/owlery/0/00/00000000000000000000000000000001 || { echo "archive entry not restored"; docker logs test-container; exit 1; } + docker exec test-container wget -q -O - http://localhost/status | grep -q '"restore": {' || { echo "/status lacks archive.restore"; docker exec test-container wget -q -O - http://localhost/status; exit 1; } + docker exec test-container cache-backup.sh --full | grep -q 'backup: done' || { echo "manual backup failed"; exit 1; } docker stop test-container || true docker rm test-container + sudo rm -rf test-archive - name: Build Docker image run: docker build . --file Dockerfile --tag virtualflybrain/owl_cache:${{ steps.meta.outputs.tag }} diff --git a/Dockerfile b/Dockerfile index 3803518..b3fdd2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,10 @@ ENV DNS_RESOLVER=8.8.8.8 ENV WORKER_PROCESSES=auto ENV WORKER_CONNECTIONS=4096 ENV WORKER_RLIMIT_NOFILE=65535 +ENV CACHE_ARCHIVE_DIR=/cache +ENV CACHE_BACKUP_SCHEDULE=daily +ENV CACHE_BACKUP_TIME=03:00 +ENV CACHE_BACKUP_JITTER_MINUTES=120 ARG NGINX_CONF=nginx.conf.template COPY $NGINX_CONF /etc/nginx/nginx.conf.template @@ -18,20 +22,36 @@ COPY health-monitor.sh /usr/local/bin/health-monitor.sh COPY test/ip-maps-test.sh /usr/local/bin/ip-maps-test.sh COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh COPY purge-cached-404s.sh /usr/local/bin/purge-cached-404s.sh +COPY cache-lib.sh /usr/local/bin/cache-lib.sh +COPY cache-restore.sh /usr/local/bin/cache-restore.sh +COPY cache-backup.sh /usr/local/bin/cache-backup.sh +COPY test/cache-sync-test.sh /usr/local/bin/cache-sync-test.sh RUN mkdir -p /var/cache/nginx/owlery /logs/hacks && \ touch /logs/blocked.txt /logs/whitelist.txt /etc/nginx/blocked-ips.map /etc/nginx/whitelisted-ips.map /etc/nginx/whitelisted-cidrs.map && \ chown -R nginx:nginx /var/cache/nginx /logs && \ - chmod +x /usr/local/bin/health-monitor.sh /usr/local/bin/docker-entrypoint.sh /usr/local/bin/purge-cached-404s.sh /usr/local/bin/ip-maps-test.sh && \ - apk add --no-cache gettext + chmod +x /usr/local/bin/health-monitor.sh /usr/local/bin/docker-entrypoint.sh /usr/local/bin/purge-cached-404s.sh /usr/local/bin/ip-maps-test.sh \ + /usr/local/bin/cache-restore.sh /usr/local/bin/cache-backup.sh /usr/local/bin/cache-sync-test.sh && \ + # rsync moves cache entries between the shared archive and local disk; + # GNU findutils gives `find -printf`, which lists millions of entries with + # their mtime and size in one walk (BusyBox find would need a stat per file). + # su-exec drops the sync scripts to the nginx user (root-squashed NFS). + apk add --no-cache gettext rsync findutils su-exec # Fail the build rather than the deployment: a whitelist entry that the map # compiler quietly discards is invisible until someone notices a cache bypass # not happening, so the list-compilation logic is unit tested here. RUN IP_MAPS_LIB=/usr/local/bin/ip-maps.sh /usr/local/bin/ip-maps-test.sh +# Same reasoning for the archive sync: a filter regression would silently copy +# in-flight temp files (served as truncated responses) or skip real entries. +RUN CACHE_RUN_AS= CACHE_LIB=/usr/local/bin/cache-lib.sh \ + CACHE_RESTORE_SCRIPT=/usr/local/bin/cache-restore.sh \ + CACHE_BACKUP_SCRIPT=/usr/local/bin/cache-backup.sh \ + /usr/local/bin/cache-sync-test.sh + EXPOSE 80 8080 -VOLUME ["/var/cache/nginx", "/logs"] +VOLUME ["/var/cache/nginx", "/cache", "/logs"] CMD ["/usr/local/bin/docker-entrypoint.sh"] diff --git a/README.md b/README.md index cddf4c2..8e22e02 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,17 @@ services: - "80:80" - "8080:8080" volumes: - - /cache:/var/cache/nginx + - /data/owl-cache:/var/cache/nginx # local node disk: what NGINX serves from + - /cache:/cache # shared NFS archive: restored from at start, backed up into daily - /logs:/logs environment: - UPSTREAM_SERVER=owl:8080 # For production with owl service - - CACHE_MAX_SIZE=1t # 1TB cache size for high-traffic deployments + - CACHE_MAX_SIZE=1t # bound on the LOCAL cache; size it to the node disk - DNS_RESOLVER=169.254.169.250 # Rancher internal DNS (check /etc/resolv.conf) ``` +See [Cache archive](#cache-archive) for why the cache is split across two volumes. + ### Health Check ```bash @@ -99,7 +102,7 @@ Example response: - `HEALTH_LOG_INTERVAL`: Seconds between periodic upstream health log lines when state is unchanged (default: `300`) - `AUTO_BLOCK_SCANNERS`: Automatically append probe-source IPs from `/logs/hacks/probes.log` to `/logs/blocked.txt` and live-reload NGINX maps (default: `true`) - `FORCE_CACHE_REFRESH_ON_REQUEST`: When `true`, each incoming request bypasses the cache and fetches fresh content from upstream, updating the cache on demand instead of serving cached entries. -- `WORKER_PROCESSES`: Number of NGINX worker processes (default: `auto`). `auto` spawns one worker per host CPU core, but it reads the host's online core count and **ignores the container's cgroup CPU quota** — on a shared/Rancher host pin this to the CPU reservation (e.g. `2`) so you don't over-spawn workers that can't run in parallel. A single container with a coherent local cache is the only safe way to share one cache directory across workers; do not point multiple containers at the same cache volume. +- `WORKER_PROCESSES`: Number of NGINX worker processes (default: `auto`). `auto` spawns one worker per host CPU core, but it reads the host's online core count and **ignores the container's cgroup CPU quota** — on a shared/Rancher host pin this to the CPU reservation (e.g. `2`) so you don't over-spawn workers that can't run in parallel. A single container with a coherent local cache is the only safe way to share one cache directory across workers; do not point multiple containers at the same `/var/cache/nginx` volume — share the archive at `/cache` instead (see [Cache archive](#cache-archive)). - `WORKER_CONNECTIONS`: Max simultaneous connections per worker (default: `4096`). Effective client concurrency is roughly `WORKER_PROCESSES × WORKER_CONNECTIONS`, halved on cache MISS since each client connection also opens an upstream connection. - `WORKER_RLIMIT_NOFILE`: Per-worker open-file-descriptor ceiling (default: `65535`). Each client connection plus every open cached file uses a descriptor, so the OS default of 1024 throttles a busy cache. Must stay within the container's hard `nofile` ulimit — NGINX logs a warning and caps to the runtime limit if this is higher. @@ -157,6 +160,75 @@ The proxy adds helpful headers to responses: - `X-Cache-Status`: `HIT`, `MISS`, `EXPIRED`, `STALE`, `UPDATING`, or `REVALIDATED` - `X-Cache-Key`: The cache key used for the request +### Cache archive + +NGINX serves the cache from `/var/cache/nginx` on local node disk. Serving +directly from an NFS volume costs an `open()`/`stat()` round trip per hit and +a metadata walk over millions of entries for the cache manager, and NGINX +cannot share one cache directory between instances anyway (the index lives in +each instance's shared memory). The shared NFS volume mounted at `/cache` +is therefore an **archive**, not the live cache: every instance restores from +it at start and backs up into it on a schedule. + +Both directions follow one rule. NGINX names each entry by the MD5 of its +cache key and replaces entries atomically, so the union of several instances' +caches is itself a valid cache. Files are only ever copied with +`rsync --update` (skip when the destination is newer) and nothing is ever +deleted, so the archive is the union of everything any instance has cached, +with the newest version of each entry winning. The comparison is by mtime, +so the nodes and the NAS must agree on time (NTP); `--modify-window=2` +absorbs filesystem timestamp granularity, not clock skew. + +**Restore** (`cache-restore.sh`, started by the entrypoint in the background) +walks the archive once, sorts entries newest first, and copies them in +batches while NGINX is already serving. NGINX serves a cache file that +appears on disk after it has started (verified against 1.26: a lookup that +misses the in-memory index still opens and validates the file), so startup +never waits for the copy. Requests whose entry has not landed yet are +ordinary misses. Progress is reported under `archive.restore` in `/status`. +A restore of ~1 TB / millions of files takes hours; an instance is fully +warm when `archive.restore.state` is `done`. If the local volume persists +across restarts, the `.restored` marker makes later starts skip the restore +(`CACHE_RESTORE=always` forces it; `off` disables it). + +**Backup** (`cache-backup.sh`) lists entries written since the previous run +from the local disk (`find -newer`; the NFS side is never walked) and copies +them into the archive. `crond` inside the container runs it daily at +`CACHE_BACKUP_TIME` (or weekly on `CACHE_BACKUP_WEEKDAY`). When a service is +scaled to several containers they all share one environment, so each adds a +deterministic per-host offset of up to `CACHE_BACKUP_JITTER_MINUTES` to spread +the NAS load; a `mkdir`-based lock on the archive serialises any that still +overlap (a lock older than `CACHE_LOCK_STALE_MINUTES` is treated as +abandoned). Run it by hand at any time: + +```bash +docker exec owlery-cache cache-backup.sh # entries new since the last run +docker exec owlery-cache cache-backup.sh --full # consider every local entry +docker exec owlery-cache cache-restore.sh --force # re-pull the archive now +curl -s http://localhost/status | jq .archive # progress of both +``` + +Loss window: a container that dies loses whatever it cached since its last +backup (at most one day on the default schedule); everything older is in the +archive and comes back on the next restore. + +Warm-up note: the `X-Force-Refresh` warm-up tool only refreshes the instance +that the load balancer routes it to. With several instances, warm one, run +`cache-backup.sh` on it, then let the others pick the entries up on their +next restore (or point the warm-up at each instance in turn). + +Variables (all optional): + +- `CACHE_ARCHIVE_DIR` (`/cache`), `CACHE_LOCAL_DIR` (`/var/cache/nginx`): the two roots; both hold an `owlery/` tree. +- `CACHE_RESTORE`: `auto` (default; skip if `.restored` exists), `always`, `off`. +- `CACHE_RESTORE_BWLIMIT`, `CACHE_BACKUP_BWLIMIT`: rsync `--bwlimit` in KiB/s (default unlimited). +- `CACHE_RESTORE_MAX_BYTES`: stop the restore after this many bytes of the newest entries (default: whole archive). +- `CACHE_RESTORE_BATCH`, `CACHE_BACKUP_BATCH`: entries per rsync invocation (default 5000). +- `CACHE_BACKUP_SCHEDULE`: `daily` (default), `weekly`, `off`. +- `CACHE_BACKUP_TIME` (`03:00`, container local time), `CACHE_BACKUP_WEEKDAY` (`0` = Sunday). +- `CACHE_BACKUP_JITTER_MINUTES` (`120`); `CACHE_BACKUP_CRON`: a verbatim 5-field crontab spec that overrides the above and gets no jitter. +- `CACHE_BACKUP_LOCK_WAIT` (`120` min), `CACHE_LOCK_STALE_MINUTES` (`360`). + ### Selective 404 cache eviction 404 responses are not cached going forward, but a long-lived cache may still @@ -175,7 +247,9 @@ The script identifies entries by matching the response status line happens to contain the text "HTTP/1.1 404" elsewhere in the body are not affected. Files removed from disk are simply treated as `MISS` on the next request — no nginx reload required. The cache directory is taken from -`$CACHE_DIR` (default `/var/cache/nginx/owlery`). +`$CACHE_DIR` (default `/var/cache/nginx/owlery`). Add `--archive` to walk the +shared archive as well; without it the next restore brings the purged entries +back. ## Performance @@ -224,9 +298,9 @@ request — no nginx reload required. The cache directory is taken from # Pull image docker pull virtualflybrain/owl_cache:latest -# Create cache directory -mkdir -p /cache -chown -R 101:101 /cache +# Local cache on node disk, and the shared archive (NFS) it syncs with +mkdir -p /data/owl-cache /cache +chown -R 101:101 /data/owl-cache /cache # Create persistent logs + blocklist file mkdir -p /logs/hacks @@ -245,6 +319,7 @@ curl -I http://localhost/health - `Dockerfile`: Image build instructions - `nginx.conf.template`: NGINX configuration template - `docker-compose.yml`: Example deployment configuration +- `cache-lib.sh`, `cache-restore.sh`, `cache-backup.sh`: archive ↔ local cache sync (see [Cache archive](#cache-archive)); `test/cache-sync-test.sh` runs at image build - `.github/workflows/docker.yml`: GitHub Actions CI/CD pipeline ## CI/CD diff --git a/cache-backup.sh b/cache-backup.sh new file mode 100644 index 0000000..df7cf88 --- /dev/null +++ b/cache-backup.sh @@ -0,0 +1,136 @@ +#!/bin/sh +# Stash the local nginx cache into the shared archive. +# +# Only entries written since the previous backup are considered. The list is +# built from the LOCAL disk (`find -newer `), which is cheap; the +# archive on NFS is never walked, only written to. rsync --update then copies +# each entry unless the archive already holds a newer copy, so several +# instances backing up into the same archive produce a union in which the +# newest version of every entry wins. Nothing is ever deleted from the +# archive; use purge-cached-404s.sh --archive to remove poisoned entries. +# +# Scheduled by docker-entrypoint.sh through crond (see CACHE_BACKUP_SCHEDULE) +# and runnable by hand: +# docker exec owlery-cache cache-backup.sh # incremental +# docker exec owlery-cache cache-backup.sh --full # ignore the marker +# +# Environment: +# CACHE_ARCHIVE_DIR /cache +# CACHE_LOCAL_DIR /var/cache/nginx +# CACHE_BACKUP_BWLIMIT rsync --bwlimit in KiB/s; 0 = unlimited +# CACHE_BACKUP_BATCH files per rsync invocation (default 5000) +# CACHE_BACKUP_LOCK_WAIT minutes to wait for another instance's backup +# to finish before giving up (default 120) +# CACHE_LOCK_STALE_MINUTES lock older than this is taken over (default 360) + +set -eu + +# Run as the nginx user, never root: the archive is NFS and typically +# root-squashed, and every file we create must be owned by the user nginx +# runs as. CACHE_RUN_AS="" keeps the current user (tests). +CACHE_RUN_AS="${CACHE_RUN_AS-nginx}" +if [ -n "$CACHE_RUN_AS" ] && [ "$(id -u)" = "0" ] && command -v su-exec >/dev/null 2>&1; then + exec su-exec "$CACHE_RUN_AS" "$0" "$@" +fi + +CACHE_LIB="${CACHE_LIB:-/usr/local/bin/cache-lib.sh}" +# shellcheck source=cache-lib.sh +. "$CACHE_LIB" + +CACHE_BACKUP_BWLIMIT="${CACHE_BACKUP_BWLIMIT:-0}" +CACHE_BACKUP_BATCH="${CACHE_BACKUP_BATCH:-5000}" +CACHE_BACKUP_LOCK_WAIT="${CACHE_BACKUP_LOCK_WAIT:-120}" + +STATE_FILE="$CACHE_STATE_DIR/cache-backup.json" +MARKER="$CACHE_LOCAL_DIR/.last-backup" +SRC="$CACHE_LOCAL_DIR/$CACHE_SUBDIR" +DST="$CACHE_ARCHIVE_DIR/$CACHE_SUBDIR" + +full=0 +[ "${1:-}" = "--full" ] && full=1 + +started="$(date +%s)" +files_done=0 +bytes_done=0 +finish() { + cache_write_state "$STATE_FILE" "$1" "$files_done" "$bytes_done" "$started" "$(date +%s)" "$2" + cache_log "backup: $1 -- $2" +} + +if [ ! -d "$CACHE_ARCHIVE_DIR" ]; then + finish skipped "archive $CACHE_ARCHIVE_DIR not mounted"; exit 0 +fi +if [ ! -d "$SRC" ]; then + finish skipped "local cache $SRC does not exist yet"; exit 0 +fi +if [ "$(cd "$SRC" && pwd -P)" = "$(mkdir -p "$DST" && cd "$DST" && pwd -P)" ]; then + finish skipped "archive and local cache are the same directory; nothing to back up"; exit 0 +fi + +work="$(mktemp -d /tmp/cache-backup.XXXXXX)" +trap 'rm -rf "$work"' EXIT + +# Stamp the new marker before listing, so entries written while the backup +# runs are picked up next time rather than falling between two runs. +new_marker="$work/marker" +# Back-dated a couple of seconds: find -newer is a strict comparison, and a +# file written in the same second as the marker would otherwise be skipped. +touch -t "$(date -d "@$(( started - 2 ))" +%Y%m%d%H%M.%S 2>/dev/null || date +%Y%m%d%H%M.%S)" "$new_marker" + +if [ "$full" -eq 1 ] || [ ! -f "$MARKER" ]; then + find "$SRC" -type f -printf '%s %P\n' +else + find "$SRC" -type f -newer "$MARKER" -printf '%s %P\n' +fi 2>/dev/null | awk ' + { path=$2; for (i=3; i<=NF; i++) path=path " " $i } + path ~ /^([0-9a-f]\/[0-9a-f][0-9a-f]\/)?[0-9a-f]{32}$/ { print $1, path } +' > "$work/entries.lst" + +total_files="$(wc -l < "$work/entries.lst" | tr -d ' ')" +total_bytes="$(awk '{ s += $1 } END { print s + 0 }' "$work/entries.lst")" + +if [ "$total_files" -eq 0 ]; then + cp -p "$new_marker" "$MARKER" + finish "done" "nothing new since last backup"; exit 0 +fi + +cache_log "backup: $total_files entries ($total_bytes bytes) changed since last backup; waiting for archive lock" +cache_write_state "$STATE_FILE" waiting 0 0 "$started" "" "waiting for lock ($total_files files pending)" + +if ! cache_lock_acquire "$CACHE_BACKUP_LOCK_WAIT"; then + finish skipped "another instance held the archive lock for more than ${CACHE_BACKUP_LOCK_WAIT} min; will retry next run" + exit 0 +fi +trap 'cache_lock_release; rm -rf "$work"' EXIT + +cache_write_state "$STATE_FILE" running 0 0 "$started" "" "0/$total_files files" +mkdir -p "$DST" +# Batch files in listing order (awk rather than split: BusyBox builds differ +# in whether split is present, and this keeps the newest-first order). +awk -v n="$CACHE_BACKUP_BATCH" -v dir="$work" '{ + f = sprintf("%s/batch.%08d", dir, int((NR - 1) / n)); print > f + if (NR % n == 0) close(f) +}' "$work/entries.lst" +errors=0 +for batch in "$work"/batch.*; do + [ -f "$batch" ] || continue + cut -d' ' -f2- "$batch" > "$batch.paths" + n="$(wc -l < "$batch" | tr -d ' ')" + if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_BACKUP_BWLIMIT"; then + # Exit 24 (vanished source file) is normal: the cache manager evicts + # entries under max_size while we run. Anything else is counted. + errors=$(( errors + 1 )) + cache_log "backup: rsync reported errors on batch $(basename "$batch"); continuing" + fi + files_done=$(( files_done + n )) + bytes_done=$(( bytes_done + $(awk '{ s += $1 } END { print s + 0 }' "$batch") )) + rm -f "$batch" "$batch.paths" + cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "$files_done/$total_files files" +done + +cp -p "$new_marker" "$MARKER" +if [ "$errors" -gt 0 ]; then + finish "done" "$files_done entries synchronised to $DST ($errors batches reported rsync errors)" +else + finish "done" "$files_done entries synchronised to $DST" +fi diff --git a/cache-lib.sh b/cache-lib.sh new file mode 100644 index 0000000..b425bc2 --- /dev/null +++ b/cache-lib.sh @@ -0,0 +1,166 @@ +#!/bin/sh +# Shared helpers for cache-restore.sh and cache-backup.sh. +# +# The archive (`/cache`, an NFS volume shared by every instance) and the local +# cache (`/var/cache/nginx`, node disk that nginx actually serves from) are +# synchronised in both directions with the same rule: a file only moves if it +# is newer than the copy at the destination (`rsync --update`). nginx names +# each entry by the MD5 of its cache key and replaces entries atomically, so +# the union of several instances' caches is itself a valid cache and "newest +# wins" is the only merge rule needed. Nothing here ever deletes. +# +# POSIX sh only: the runtime image is BusyBox ash. + +CACHE_ARCHIVE_DIR="${CACHE_ARCHIVE_DIR:-/cache}" +CACHE_LOCAL_DIR="${CACHE_LOCAL_DIR:-/var/cache/nginx}" +# Subdirectory (under both roots) that holds the nginx cache tree. +CACHE_SUBDIR="${CACHE_SUBDIR:-owlery}" +CACHE_STATE_DIR="${CACHE_STATE_DIR:-/var/run/nginx}" +# rsync --modify-window: NFS and local filesystems can disagree by a second +# on mtime; without this a byte-identical file would be recopied every run. +CACHE_MODIFY_WINDOW="${CACHE_MODIFY_WINDOW:-2}" +# Lock directory on the archive so concurrent instances serialise their +# backups. flock(2) is unreliable on NFS; mkdir(2) is atomic there. +CACHE_LOCK_DIR="${CACHE_LOCK_DIR:-$CACHE_ARCHIVE_DIR/.owl-cache-backup.lock}" +CACHE_LOCK_STALE_MINUTES="${CACHE_LOCK_STALE_MINUTES:-360}" + +cache_log() { + printf '%s %s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$*" +} + +cache_json_escape() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +# Is the argument a real nginx cache entry path (relative to the tree root)? +# Entries are <32 hex>, optionally under levels=1:2 directories. Anything +# else in the tree is an in-flight temp file or an rsync partial and must not +# be copied: a half-written body would be served as a truncated response. +cache_entry_filter() { + grep -E '^([0-9a-f]/[0-9a-f]{2}/)?[0-9a-f]{32}$' +} + +# Deterministic per-instance offset in [0, N) minutes derived from the +# hostname (the container id under Rancher/Docker), so several replicas of +# one service, which all receive the same environment, do not fire at the +# same minute. A restarted container keeps its slot. +cache_jitter_minutes() { + span="${1:-0}" + case "$span" in + ''|*[!0-9]*) span=0 ;; + esac + [ "$span" -le 0 ] && { printf '0'; return; } + sum="$(hostname 2>/dev/null | cksum | cut -d' ' -f1)" + [ -z "$sum" ] && sum=0 + printf '%s' "$(( sum % span ))" +} + +# Turn CACHE_BACKUP_SCHEDULE / CACHE_BACKUP_TIME / CACHE_BACKUP_WEEKDAY / +# CACHE_BACKUP_JITTER_MINUTES (or a verbatim CACHE_BACKUP_CRON) into one +# 5-field crontab spec on stdout, or nothing when backups are off. +cache_backup_cron_spec() { + if [ -n "${CACHE_BACKUP_CRON:-}" ]; then + printf '%s' "$CACHE_BACKUP_CRON" + return + fi + case "$(printf '%s' "${CACHE_BACKUP_SCHEDULE:-daily}" | tr '[:upper:]' '[:lower:]')" in + off|0|false|no|manual|none) return ;; + weekly) dow="${CACHE_BACKUP_WEEKDAY:-0}" ;; + *) dow='*' ;; + esac + case "$dow" in + '*') ;; + ''|*[!0-9]*) dow=0 ;; + *) dow=$(( dow % 7 )) ;; + esac + time="${CACHE_BACKUP_TIME:-03:00}" + hour="${time%%:*}" + minute="${time#*:}" + case "$hour$minute" in + ''|*[!0-9]*) hour=3; minute=0 ;; + esac + # Strip leading zeros so "08" is not read as octal. + hour=$(( $(printf '%s' "$hour" | sed 's/^0*//; s/^$/0/') )) + minute=$(( $(printf '%s' "$minute" | sed 's/^0*//; s/^$/0/') )) + total=$(( hour * 60 + minute + $(cache_jitter_minutes "${CACHE_BACKUP_JITTER_MINUTES:-0}") )) + if [ "$total" -ge 1440 ]; then + total=$(( total - 1440 )) + # Jitter pushed the slot past midnight: shift the weekday along too. + [ "$dow" != '*' ] && dow=$(( (dow + 1) % 7 )) + fi + printf '%s %s * * %s' "$(( total % 60 ))" "$(( total / 60 ))" "$dow" +} + +# Write a small JSON state file atomically; health-monitor.sh embeds it in +# /status. Arguments: +# +cache_write_state() { + file="$1"; state="$2"; files="$3"; bytes="$4"; started="$5"; finished="$6"; message="$7" + mkdir -p "$(dirname "$file")" + now="$(date +%s)" + if [ -n "$finished" ]; then + elapsed=$(( finished - started )) + finished_json="\"$(date -u -d "@$finished" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u +"%Y-%m-%dT%H:%M:%SZ")\"" + else + elapsed=$(( now - started )) + finished_json=null + fi + started_iso="$(date -u -d "@$started" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u +"%Y-%m-%dT%H:%M:%SZ")" + cat > "$file.tmp" </dev/null || echo unknown)")", + "files": $files, + "bytes": $bytes, + "started_at": "$started_iso", + "finished_at": $finished_json, + "elapsed_seconds": $elapsed, + "message": "$(cache_json_escape "$message")" +} +EOF + mv "$file.tmp" "$file" +} + +# Acquire the archive lock. Returns 0 when held, 1 when it could not be +# obtained within . A lock older than CACHE_LOCK_STALE_MINUTES +# is treated as abandoned (a container killed mid-backup) and taken over. +cache_lock_acquire() { + wait_minutes="${1:-0}" + deadline=$(( $(date +%s) + wait_minutes * 60 )) + while :; do + if mkdir "$CACHE_LOCK_DIR" 2>/dev/null; then + printf '%s %s\n' "$(hostname 2>/dev/null)" "$(date +%s)" > "$CACHE_LOCK_DIR/owner" 2>/dev/null || true + return 0 + fi + lock_epoch="$(stat -c %Y "$CACHE_LOCK_DIR" 2>/dev/null || echo 0)" + age_minutes=$(( ( $(date +%s) - lock_epoch ) / 60 )) + if [ "$age_minutes" -ge "$CACHE_LOCK_STALE_MINUTES" ]; then + cache_log "backup lock at $CACHE_LOCK_DIR is ${age_minutes} min old (stale after ${CACHE_LOCK_STALE_MINUTES}); taking it over" + rm -rf "$CACHE_LOCK_DIR" 2>/dev/null || true + continue + fi + if [ "$(date +%s)" -ge "$deadline" ]; then + return 1 + fi + sleep 60 + done +} + +cache_lock_release() { + rm -rf "$CACHE_LOCK_DIR" 2>/dev/null || true +} + +# Run rsync over a batch file of relative paths from to . +# Whole-file copies (no delta computation: entries are immutable blobs), +# newest wins, never delete, atomic per file via rsync's temp+rename. +cache_rsync_batch() { + src="$1"; dst="$2"; list="$3"; bwlimit="${4:-0}" + # --files-from implies --relative, so `a/bc/` lands at the same + # levels path under and the intermediate directories are created. + set -- -a --whole-file --update --modify-window="$CACHE_MODIFY_WINDOW" \ + --files-from="$list" --quiet + if [ "$bwlimit" != "0" ] && [ -n "$bwlimit" ]; then + set -- "$@" --bwlimit="$bwlimit" + fi + rsync "$@" "$src/" "$dst/" +} diff --git a/cache-restore.sh b/cache-restore.sh new file mode 100644 index 0000000..8abdc1a --- /dev/null +++ b/cache-restore.sh @@ -0,0 +1,151 @@ +#!/bin/sh +# Warm the local nginx cache from the shared archive, newest entries first, +# while nginx is already serving. +# +# nginx serves a cache file that appears on disk after startup: a lookup that +# misses the shared-memory index still opens the file, validates its header, +# serves it and registers it (verified against nginx 1.26). So there is no +# need to block startup on a copy of the archive -- which at ~1 TB and +# millions of files would take hours. Instead docker-entrypoint.sh starts +# nginx immediately and runs this script in the background. Requests whose +# entry has not landed yet are ordinary misses served by Owlery; once the +# file lands, later requests are hits. +# +# Ordering: the archive is walked once to list every entry with its mtime, +# sorted newest first, and copied in batches. Newest entries are the ones +# most likely to be requested again, and a bounded restore +# (CACHE_RESTORE_MAX_BYTES) keeps the freshest part of the archive. +# +# Merge rule: rsync --update, so an entry that nginx has already fetched fresh +# on this instance is never overwritten by an older archive copy. Re-running +# the script is idempotent; a container restart mid-restore simply resumes. +# +# Environment: +# CACHE_RESTORE auto (default) | always | off +# auto skips when the local marker from a previous +# completed restore exists (persistent local disk). +# CACHE_ARCHIVE_DIR /cache +# CACHE_LOCAL_DIR /var/cache/nginx +# CACHE_RESTORE_BWLIMIT rsync --bwlimit in KiB/s; 0 = unlimited (default) +# CACHE_RESTORE_BATCH files per rsync invocation (default 5000) +# CACHE_RESTORE_MAX_BYTES stop after this many bytes (0 = whole archive) +# CACHE_RESTORE_DELAY seconds to wait before starting (default 0) +# +# Usage: cache-restore.sh # honours CACHE_RESTORE +# cache-restore.sh --force # same as CACHE_RESTORE=always + +set -eu + +# Run as the nginx user, never root: the archive is NFS and typically +# root-squashed, and every file we create must be owned by the user nginx +# runs as. CACHE_RUN_AS="" keeps the current user (tests). +CACHE_RUN_AS="${CACHE_RUN_AS-nginx}" +if [ -n "$CACHE_RUN_AS" ] && [ "$(id -u)" = "0" ] && command -v su-exec >/dev/null 2>&1; then + exec su-exec "$CACHE_RUN_AS" "$0" "$@" +fi + +CACHE_LIB="${CACHE_LIB:-/usr/local/bin/cache-lib.sh}" +# shellcheck source=cache-lib.sh +. "$CACHE_LIB" + +CACHE_RESTORE="${CACHE_RESTORE:-auto}" +CACHE_RESTORE_BWLIMIT="${CACHE_RESTORE_BWLIMIT:-0}" +CACHE_RESTORE_BATCH="${CACHE_RESTORE_BATCH:-5000}" +CACHE_RESTORE_MAX_BYTES="${CACHE_RESTORE_MAX_BYTES:-0}" +CACHE_RESTORE_DELAY="${CACHE_RESTORE_DELAY:-0}" + +STATE_FILE="$CACHE_STATE_DIR/cache-restore.json" +MARKER="$CACHE_LOCAL_DIR/.restored" +SRC="$CACHE_ARCHIVE_DIR/$CACHE_SUBDIR" +DST="$CACHE_LOCAL_DIR/$CACHE_SUBDIR" + +[ "${1:-}" = "--force" ] && CACHE_RESTORE=always + +started="$(date +%s)" +finish() { + cache_write_state "$STATE_FILE" "$1" "${files_done:-0}" "${bytes_done:-0}" "$started" "$(date +%s)" "$2" + cache_log "restore: $1 -- $2" +} + +case "$(printf '%s' "$CACHE_RESTORE" | tr '[:upper:]' '[:lower:]')" in + off|0|false|no) + finish skipped "CACHE_RESTORE=off"; exit 0 ;; + always|1|true|yes|force) ;; + *) + if [ -f "$MARKER" ]; then + finish skipped "local cache already restored ($(cat "$MARKER" 2>/dev/null)); set CACHE_RESTORE=always to repeat" + exit 0 + fi ;; +esac + +if [ ! -d "$SRC" ]; then + finish skipped "archive $SRC not present"; exit 0 +fi +if [ "$(cd "$SRC" && pwd -P)" = "$(mkdir -p "$DST" && cd "$DST" && pwd -P)" ]; then + finish skipped "archive and local cache are the same directory; nothing to restore"; exit 0 +fi + +if [ "$CACHE_RESTORE_DELAY" -gt 0 ] 2>/dev/null; then + sleep "$CACHE_RESTORE_DELAY" +fi + +work="$(mktemp -d /tmp/cache-restore.XXXXXX)" +trap 'rm -rf "$work"' EXIT + +files_done=0 +bytes_done=0 +cache_write_state "$STATE_FILE" listing 0 0 "$started" "" "walking archive $SRC" +cache_log "restore: listing $SRC (this walks the whole archive once)" + +# One walk of the archive: " ", newest first. +# find -printf needs GNU findutils (installed in the image); BusyBox find +# has no -printf and would need one stat(2) per file on top of the walk. +find "$SRC" -type f -printf '%T@ %s %P\n' 2>/dev/null \ + | awk '{ path=$3; for (i=4; i<=NF; i++) path=path " " $i; print $1, $2, path }' \ + | sort -k1,1nr > "$work/all.lst" + +# Keep only real cache entries and apply the optional byte bound. +awk -v max="$CACHE_RESTORE_MAX_BYTES" ' + { path=$3; for (i=4; i<=NF; i++) path=path " " $i } + path ~ /^([0-9a-f]\/[0-9a-f][0-9a-f]\/)?[0-9a-f]{32}$/ { + if (max > 0 && total + $2 > max) exit + total += $2; print $2, path + }' "$work/all.lst" > "$work/entries.lst" + +total_files="$(wc -l < "$work/entries.lst" | tr -d ' ')" +total_bytes="$(awk '{ s += $1 } END { print s + 0 }' "$work/entries.lst")" +cache_log "restore: $total_files entries ($total_bytes bytes) to consider, newest first, batches of $CACHE_RESTORE_BATCH" +cache_write_state "$STATE_FILE" running 0 0 "$started" "" "0/$total_files files" + +if [ "$total_files" -eq 0 ]; then + finish "done" "archive is empty"; printf '%s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ") 0 files" > "$MARKER"; exit 0 +fi + +mkdir -p "$DST" +# Batch files in listing order (awk rather than split: BusyBox builds differ +# in whether split is present, and this keeps the newest-first order). +awk -v n="$CACHE_RESTORE_BATCH" -v dir="$work" '{ + f = sprintf("%s/batch.%08d", dir, int((NR - 1) / n)); print > f + if (NR % n == 0) close(f) +}' "$work/entries.lst" + +for batch in "$work"/batch.*; do + [ -f "$batch" ] || continue + cut -d' ' -f2- "$batch" > "$batch.paths" + n="$(wc -l < "$batch" | tr -d ' ')" + if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_RESTORE_BWLIMIT"; then + # Entries evicted from the archive between listing and copy show up as + # vanished files (rsync exit 24); anything else is worth surfacing but + # must not abandon the remaining batches. + cache_log "restore: rsync reported errors on batch $(basename "$batch"); continuing" + fi + files_done=$(( files_done + n )) + # Bytes are accounted from the listing, not from rsync, so this is the + # size of the entries considered so far (already-current files included). + bytes_done=$(( bytes_done + $(awk '{ s += $1 } END { print s + 0 }' "$batch") )) + rm -f "$batch" "$batch.paths" + cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "$files_done/$total_files files" +done + +printf '%s %s files\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$files_done" > "$MARKER" +finish "done" "$files_done/$total_files entries synchronised from $SRC" diff --git a/docker-compose.yml b/docker-compose.yml index e767533..d2f484e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,11 @@ services: environment: - UPSTREAM_SERVER=owl:8080 volumes: - - /cache:/var/cache/nginx + # Local node disk: what nginx serves from. Size CACHE_MAX_SIZE to it. + - /data/owl-cache:/var/cache/nginx + # Shared NFS archive: restored from in the background at start, backed + # up into daily at 03:00 (+ per-container jitter). See README. + - /cache:/cache - /logs:/logs networks: - owlery_network diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index a8eeb61..59684fd 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -70,6 +70,40 @@ case "$(printf '%s' "${FORCE_CACHE_REFRESH_ON_REQUEST:-false}" | tr '[:upper:]' ;; esac +# Cache archive handling. nginx serves from CACHE_LOCAL_DIR (node disk); +# CACHE_ARCHIVE_DIR is the shared NFS volume every instance restores from at +# start and backs up into on a schedule. See cache-restore.sh / cache-backup.sh +# and README "Cache archive". +export CACHE_ARCHIVE_DIR="${CACHE_ARCHIVE_DIR:-/cache}" +export CACHE_LOCAL_DIR="${CACHE_LOCAL_DIR:-/var/cache/nginx}" +# daily | weekly | off, plus CACHE_BACKUP_TIME (HH:MM, container local time) +# and CACHE_BACKUP_WEEKDAY (0-6, Sunday=0, weekly only). Replicas add a +# deterministic per-host offset of up to CACHE_BACKUP_JITTER_MINUTES so they do +# not all hit the NAS in the same minute. CACHE_BACKUP_CRON, if set, is used +# verbatim (5-field crontab spec) and receives no jitter. +export CACHE_BACKUP_SCHEDULE="${CACHE_BACKUP_SCHEDULE:-daily}" +export CACHE_BACKUP_TIME="${CACHE_BACKUP_TIME:-03:00}" +export CACHE_BACKUP_WEEKDAY="${CACHE_BACKUP_WEEKDAY:-0}" +export CACHE_BACKUP_JITTER_MINUTES="${CACHE_BACKUP_JITTER_MINUTES:-120}" + +CACHE_LIB="${CACHE_LIB:-/usr/local/bin/cache-lib.sh}" +# shellcheck source=cache-lib.sh +. "$CACHE_LIB" + +start_backup_scheduler() { + spec="$(cache_backup_cron_spec)" + if [ -z "$spec" ]; then + echo "Scheduled cache backup disabled (CACHE_BACKUP_SCHEDULE=$CACHE_BACKUP_SCHEDULE); run cache-backup.sh by hand" + return + fi + mkdir -p /etc/crontabs + # Container stdout is fd 1 of PID 1, so the backup log lands in `docker logs`. + # crond runs the job as root; cache-backup.sh drops to nginx itself. + printf '%s /usr/local/bin/cache-backup.sh >> /proc/1/fd/1 2>&1\n' "$spec" > /etc/crontabs/root + echo "Scheduled cache backup: crontab '$spec' (base ${CACHE_BACKUP_SCHEDULE} ${CACHE_BACKUP_TIME}, jitter up to ${CACHE_BACKUP_JITTER_MINUTES} min)" + crond -b -l 8 -L /dev/stdout +} + prepare_log_paths generate_ip_map "$BLOCKLIST_SOURCE" "$BLOCKLIST_MAP" "blocked" generate_whitelist_maps "$WHITELIST_SOURCE" "$WHITELIST_MAP" "$WHITELIST_CIDR_MAP" @@ -78,5 +112,16 @@ envsubst '${UPSTREAM_SERVER} ${CACHE_MAX_SIZE} ${CACHE_STALE_TIME} ${DNS_RESOLVE < /etc/nginx/nginx.conf.template \ > /etc/nginx/nginx.conf +# The sync scripts run as nginx (see CACHE_RUN_AS): they need to write their +# markers at the cache root and their state files next to status.json. Only +# the top-level directories are chowned -- a recursive chown over millions of +# cache entries would take longer than the restore. +mkdir -p /var/run/nginx "$CACHE_LOCAL_DIR/owlery" +chown nginx:nginx /var/run/nginx "$CACHE_LOCAL_DIR" "$CACHE_LOCAL_DIR/owlery" 2>/dev/null || true + /usr/local/bin/health-monitor.sh & +# Warm the local cache from the archive in the background; nginx serves +# whatever has landed and treats the rest as ordinary misses meanwhile. +/usr/local/bin/cache-restore.sh & +start_backup_scheduler exec nginx -g 'daemon off;' diff --git a/health-monitor.sh b/health-monitor.sh index 8a01a9d..34c7624 100644 --- a/health-monitor.sh +++ b/health-monitor.sh @@ -55,6 +55,19 @@ upstream_healthy=false last_upstream_state= last_health_log_epoch=0 +CACHE_RESTORE_STATE=${CACHE_RESTORE_STATE:-$STATUS_DIR/cache-restore.json} +CACHE_BACKUP_STATE=${CACHE_BACKUP_STATE:-$STATUS_DIR/cache-backup.json} + +# Embed a state file written by cache-restore.sh / cache-backup.sh, or null +# when that job has not run in this container yet. +json_state_or_null() { + if [ -s "$1" ]; then + sed 's/^/ /' "$1" + else + printf 'null' + fi +} + json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' } @@ -368,6 +381,10 @@ write_status_file() { "reading": $(json_number_or_null "$reading_connections"), "writing": $(json_number_or_null "$writing_connections"), "waiting": $(json_number_or_null "$waiting_connections") + }, + "archive": { + "restore": $(json_state_or_null "$CACHE_RESTORE_STATE"), + "backup": $(json_state_or_null "$CACHE_BACKUP_STATE") } } EOF diff --git a/purge-cached-404s.sh b/purge-cached-404s.sh index ecd953a..769abfc 100755 --- a/purge-cached-404s.sh +++ b/purge-cached-404s.sh @@ -13,14 +13,20 @@ # MISS on the next request -- no nginx reload required. # # Usage: -# purge-cached-404s.sh # dry run; list candidate files -# purge-cached-404s.sh --apply # delete the matching files +# purge-cached-404s.sh # dry run; list candidate files +# purge-cached-404s.sh --apply # delete the matching files +# purge-cached-404s.sh --archive [--apply] # also walk the shared archive # -# Honours $CACHE_DIR (default: /var/cache/nginx/owlery). +# Honours $CACHE_DIR (default: /var/cache/nginx/owlery). --archive adds +# $CACHE_ARCHIVE_DIR/owlery (default /cache/owlery): a purge that only +# touches the local cache is undone by the next cache-restore.sh. Walking +# the archive greps every entry over NFS, so expect it to take hours. set -eu CACHE_DIR="${CACHE_DIR:-/var/cache/nginx/owlery}" +ARCHIVE_DIR="${CACHE_ARCHIVE_DIR:-/cache}/${CACHE_SUBDIR:-owlery}" +WITH_ARCHIVE=0 # Anchor to the start of a line. In an nginx cache file the response # status line is always preceded by the newline that terminates the # preceding "KEY: ..." line, so this matches the real status line and @@ -28,10 +34,14 @@ CACHE_DIR="${CACHE_DIR:-/var/cache/nginx/owlery}" PATTERN='^HTTP/1\.[01] 404 ' APPLY=0 -case "${1:-}" in +for arg in "$@"; do +case "$arg" in --apply|-y) APPLY=1 ;; + --archive) + WITH_ARCHIVE=1 + ;; --help|-h) sed -n 's/^# \{0,1\}//p' "$0" | sed -n '1,20p' exit 0 @@ -39,10 +49,21 @@ case "${1:-}" in "") ;; *) - printf 'purge-cached-404s: unknown argument: %s\n' "$1" >&2 + printf 'purge-cached-404s: unknown argument: %s\n' "$arg" >&2 exit 2 ;; esac +done + +DIRS="$CACHE_DIR" +if [ "$WITH_ARCHIVE" = "1" ]; then + if [ -d "$ARCHIVE_DIR" ]; then + DIRS="$CACHE_DIR $ARCHIVE_DIR" + else + printf 'purge-cached-404s: archive directory not found: %s\n' "$ARCHIVE_DIR" >&2 + exit 1 + fi +fi if [ ! -d "$CACHE_DIR" ]; then printf 'purge-cached-404s: cache directory not found: %s\n' "$CACHE_DIR" >&2 @@ -56,7 +77,8 @@ trap 'rm -f "$CANDIDATES"' EXIT # -m 1 stop reading each file after the first match. Use the short # form: this image runs Alpine/BusyBox grep, which does not accept # the GNU long option --max-count=1. -grep -rla -m 1 "$PATTERN" "$CACHE_DIR" > "$CANDIDATES" 2>/dev/null || true +# shellcheck disable=SC2086 +grep -rla -m 1 "$PATTERN" $DIRS > "$CANDIDATES" 2>/dev/null || true found=$(wc -l < "$CANDIDATES" | tr -d ' ') @@ -65,9 +87,9 @@ if [ "$APPLY" = "1" ]; then xargs -r rm -f -- < "$CANDIDATES" fi printf 'purge-cached-404s: deleted %d cached 404 entries from %s\n' \ - "$found" "$CACHE_DIR" + "$found" "$DIRS" else cat "$CANDIDATES" printf '\npurge-cached-404s: %d candidate file(s) in %s. Re-run with --apply to delete.\n' \ - "$found" "$CACHE_DIR" >&2 + "$found" "$DIRS" >&2 fi diff --git a/test/cache-sync-test.sh b/test/cache-sync-test.sh new file mode 100644 index 0000000..08fce29 --- /dev/null +++ b/test/cache-sync-test.sh @@ -0,0 +1,167 @@ +#!/bin/sh +# Tests for cache-lib.sh, cache-restore.sh and cache-backup.sh on a fake +# cache tree. Run directly (`sh test/cache-sync-test.sh`) or let the Docker +# build run it. Needs rsync and GNU find (-printf), as the image does. +# +# POSIX sh only, to match BusyBox ash in the runtime image. + +set -eu + +here="$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)" +CACHE_LIB="${CACHE_LIB:-$here/cache-lib.sh}" +CACHE_RESTORE_SCRIPT="${CACHE_RESTORE_SCRIPT:-$here/cache-restore.sh}" +CACHE_BACKUP_SCRIPT="${CACHE_BACKUP_SCRIPT:-$here/cache-backup.sh}" +export CACHE_LIB + +WORK="$(mktemp -d /tmp/cache-sync-test.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +export CACHE_ARCHIVE_DIR="$WORK/archive" +export CACHE_LOCAL_DIR="$WORK/local" +export CACHE_STATE_DIR="$WORK/state" +export CACHE_LOCK_DIR="$WORK/archive/.lock" +export CACHE_RESTORE_BATCH=2 +export CACHE_BACKUP_BATCH=2 +export CACHE_BACKUP_LOCK_WAIT=0 +ARCHIVE="$CACHE_ARCHIVE_DIR/owlery" +LOCAL="$CACHE_LOCAL_DIR/owlery" + +failures=0 +checks=0 +ok() { checks=$((checks + 1)); printf ' ok %s\n' "$1"; } +fail() { checks=$((checks + 1)); failures=$((failures + 1)); printf ' FAIL %s\n' "$1"; } +assert_eq() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1 (expected '$3', got '$2')"; fi; } +assert_file() { if [ -f "$2" ]; then ok "$1"; else fail "$1 (missing $2)"; fi; } +assert_no_file() { if [ ! -e "$2" ]; then ok "$1"; else fail "$1 (unexpected $2)"; fi; } + +# entry +entry() { + d="$1/${2%"${2#?}"}/$(printf '%s' "$2" | cut -c2-3)" + mkdir -p "$d" + printf 'KEY: %s\nHTTP/1.1 200 OK\n\n%s\n' "$2" "$3" > "$d/$2" + touch -d "@$(( $(date +%s) - $4 ))" "$d/$2" +} +h() { printf '%032d' "$1"; } + +# shellcheck source=../cache-lib.sh disable=SC1090 +. "$CACHE_LIB" + +echo "cache_entry_filter" +assert_eq "accepts levels path" "$(printf '%s\n' "a/bc/$(h 1)" | cache_entry_filter)" "a/bc/$(h 1)" +assert_eq "accepts flat entry" "$(printf '%s\n' "$(h 2)" | cache_entry_filter)" "$(h 2)" +assert_eq "rejects nginx temp file" "$(printf '%s\n' "a/bc/$(h 3).0000000012" | cache_entry_filter)" "" +assert_eq "rejects rsync partial" "$(printf '%s\n' "a/bc/.$(h 3).Xk3Zq" | cache_entry_filter)" "" +assert_eq "rejects markers" "$(printf '.restored\n.last-backup\n' | cache_entry_filter)" "" + +echo "cache_jitter_minutes" +j1="$(cache_jitter_minutes 120)"; j2="$(cache_jitter_minutes 120)" +assert_eq "deterministic for this host" "$j1" "$j2" +[ "$j1" -ge 0 ] && [ "$j1" -lt 120 ] && ok "within [0,120)" || fail "jitter $j1 out of range" +assert_eq "zero span gives zero" "$(cache_jitter_minutes 0)" "0" +assert_eq "garbage span gives zero" "$(cache_jitter_minutes abc)" "0" + +echo "cache_backup_cron_spec" +spec="$(CACHE_BACKUP_SCHEDULE=daily CACHE_BACKUP_TIME=03:00 CACHE_BACKUP_JITTER_MINUTES=0 cache_backup_cron_spec)" +assert_eq "daily 03:00 no jitter" "$spec" "0 3 * * *" +spec="$(CACHE_BACKUP_SCHEDULE=weekly CACHE_BACKUP_WEEKDAY=6 CACHE_BACKUP_TIME=23:30 CACHE_BACKUP_JITTER_MINUTES=0 cache_backup_cron_spec)" +assert_eq "weekly Saturday 23:30" "$spec" "30 23 * * 6" +spec="$(CACHE_BACKUP_SCHEDULE=daily CACHE_BACKUP_TIME=08:05 CACHE_BACKUP_JITTER_MINUTES=0 cache_backup_cron_spec)" +assert_eq "leading zeros are not octal" "$spec" "5 8 * * *" +spec="$(CACHE_BACKUP_SCHEDULE=off cache_backup_cron_spec)" +assert_eq "off yields empty" "$spec" "" +spec="$(CACHE_BACKUP_CRON='15 4 * * 2' CACHE_BACKUP_SCHEDULE=daily cache_backup_cron_spec)" +assert_eq "verbatim cron override" "$spec" "15 4 * * 2" +spec="$(CACHE_BACKUP_SCHEDULE=daily CACHE_BACKUP_TIME=03:00 CACHE_BACKUP_JITTER_MINUTES=120 cache_backup_cron_spec)" +m="${spec%% *}"; rest="${spec#* }"; hh="${rest%% *}" +total=$(( hh * 60 + m )) +[ "$total" -ge 180 ] && [ "$total" -lt 300 ] && ok "jittered slot lands in 03:00-04:59 ($spec)" || fail "jittered slot $spec outside window" +# Force the wrap: jitter fixed by hostname, so pick a base time near midnight. +spec="$(CACHE_BACKUP_SCHEDULE=weekly CACHE_BACKUP_WEEKDAY=6 CACHE_BACKUP_TIME=23:59 CACHE_BACKUP_JITTER_MINUTES=120 cache_backup_cron_spec)" +case "$spec" in + *" * * 6") [ "$j1" -eq 0 ] && ok "no wrap when jitter is 0" || fail "expected wrap to Sunday, got '$spec'" ;; + *" * * 0") ok "wrap past midnight moves weekday to Sunday ($spec)" ;; + *) fail "unexpected spec '$spec'" ;; +esac + +echo "cache-restore.sh" +entry "$ARCHIVE" "$(h 1)" "old-archive" 3600 +entry "$ARCHIVE" "$(h 2)" "archive" 60 +entry "$ARCHIVE" "$(h 3)" "archive" 10 +entry "$ARCHIVE" "$(h 4)" "archive" 5 +mkdir -p "$ARCHIVE/a/bc"; printf 'partial' > "$ARCHIVE/a/bc/$(h 9).0000000042" +# Local already holds a fresher copy of entry 1 (nginx fetched it after the archive did). +entry "$LOCAL" "$(h 1)" "fresh-local" 0 +sh "$CACHE_RESTORE_SCRIPT" > "$WORK/restore.log" 2>&1 || fail "restore exited non-zero: $(cat "$WORK/restore.log")" +assert_file "restores entry 2" "$LOCAL/0/00/$(h 2)" +assert_file "restores entry 4" "$LOCAL/0/00/$(h 4)" +assert_eq "--update keeps the newer local entry 1" "$(tail -n 1 "$LOCAL/0/00/$(h 1)")" "fresh-local" +assert_no_file "temp file is not restored" "$LOCAL/a/bc/$(h 9).0000000042" +assert_file "marker written" "$CACHE_LOCAL_DIR/.restored" +assert_eq "state is done" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "done" +assert_eq "state counts 4 entries" "$(sed -n 's/.*"files": \([0-9]*\).*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "4" +grep -q "4 entries" "$WORK/restore.log" && ok "log reports entries" || fail "log: $(cat "$WORK/restore.log")" + +entry "$ARCHIVE" "$(h 5)" "later" 1 +sh "$CACHE_RESTORE_SCRIPT" > "$WORK/restore2.log" 2>&1 +assert_no_file "auto mode skips when marker exists" "$LOCAL/0/00/$(h 5)" +assert_eq "skipped state" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "skipped" +sh "$CACHE_RESTORE_SCRIPT" --force > "$WORK/restore3.log" 2>&1 +assert_file "--force restores again" "$LOCAL/0/00/$(h 5)" + +rm -rf "$LOCAL" "$CACHE_LOCAL_DIR/.restored" +CACHE_RESTORE_MAX_BYTES=$(( $(stat -c %s "$ARCHIVE/0/00/$(h 5)") + $(stat -c %s "$ARCHIVE/0/00/$(h 4)") + 1 )) sh "$CACHE_RESTORE_SCRIPT" > "$WORK/restore4.log" 2>&1 +assert_file "bounded restore takes newest (5)" "$LOCAL/0/00/$(h 5)" +assert_file "bounded restore takes next newest (4)" "$LOCAL/0/00/$(h 4)" +assert_no_file "bounded restore stops before older entries (2)" "$LOCAL/0/00/$(h 2)" + +CACHE_RESTORE=off sh "$CACHE_RESTORE_SCRIPT" > /dev/null 2>&1 +assert_eq "CACHE_RESTORE=off skips" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "skipped" + +echo "cache-backup.sh" +rm -rf "$CACHE_ARCHIVE_DIR" "$CACHE_LOCAL_DIR" +entry "$LOCAL" "$(h 1)" "local-new" 0 +entry "$LOCAL" "$(h 2)" "local-old" 3600 +entry "$ARCHIVE" "$(h 2)" "archive-newer" 60 +mkdir -p "$LOCAL/0/00"; printf 'partial' > "$LOCAL/0/00/$(h 7).0000000001" +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup.log" 2>&1 || fail "backup exited non-zero: $(cat "$WORK/backup.log")" +assert_file "new local entry reaches archive" "$ARCHIVE/0/00/$(h 1)" +assert_eq "--update keeps newer archive copy of 2" "$(tail -n 1 "$ARCHIVE/0/00/$(h 2)")" "archive-newer" +assert_no_file "temp file is not backed up" "$ARCHIVE/0/00/$(h 7).0000000001" +assert_file "backup marker written" "$CACHE_LOCAL_DIR/.last-backup" +assert_no_file "lock released" "$CACHE_LOCK_DIR" +assert_eq "state is done" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-backup.json")" "done" + +# The marker is back-dated 2 s (see cache-backup.sh), so the run straight +# after re-lists entries written just before the previous one (harmless: +# --update skips them). Once the marker has moved past them, nothing is listed. +sleep 3 +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup2a.log" 2>&1 +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup2.log" 2>&1 +grep -q "nothing new" "$WORK/backup2.log" && ok "steady state finds nothing new" || fail "steady state: $(cat "$WORK/backup2.log")" +entry "$LOCAL" "$(h 3)" "after-marker" 0 +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup3.log" 2>&1 +assert_file "incremental run copies only the new entry" "$ARCHIVE/0/00/$(h 3)" +grep -q "1 entries" "$WORK/backup3.log" && ok "incremental run lists 1 entry" || fail "incremental: $(cat "$WORK/backup3.log")" + +# Lock held by a live peer: give up after CACHE_BACKUP_LOCK_WAIT=0 minutes. +mkdir -p "$CACHE_LOCK_DIR" +entry "$LOCAL" "$(h 4)" "blocked" 0 +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup4.log" 2>&1 +assert_no_file "held lock skips the run" "$ARCHIVE/0/00/$(h 4)" +assert_eq "skipped state" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-backup.json")" "skipped" +# Stale lock is taken over. +touch -d "@$(( $(date +%s) - 7 * 3600 ))" "$CACHE_LOCK_DIR" +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup5.log" 2>&1 +assert_file "stale lock is taken over" "$ARCHIVE/0/00/$(h 4)" +assert_no_file "lock released after takeover" "$CACHE_LOCK_DIR" + +# --full ignores the marker. +sh "$CACHE_BACKUP_SCRIPT" --full > "$WORK/backup6.log" 2>&1 +grep -q "4 entries" "$WORK/backup6.log" && ok "--full considers every entry" || fail "--full: $(cat "$WORK/backup6.log")" + +rm -rf "$CACHE_ARCHIVE_DIR" +sh "$CACHE_BACKUP_SCRIPT" > "$WORK/backup7.log" 2>&1 +grep -q "not mounted" "$WORK/backup7.log" && ok "missing archive is a clean skip" || fail "missing archive: $(cat "$WORK/backup7.log")" + +printf '\n%d checks, %d failures\n' "$checks" "$failures" +[ "$failures" -eq 0 ]