Restore before nginx starts; keep rsync partials out of the cache tree

README.md

@@ -179,17 +179,24 @@ with the newest version of each entry winning. The comparison is by mtime,
 so the nodes and the NAS must agree on time (NTP); `--modify-window=2`
 absorbs filesystem timestamp granularity, not clock skew.
 
-**Restore** (`cache-restore.sh`, started by the entrypoint in the background)
-walks the archive once, sorts entries newest first, and copies them in
-batches while NGINX is already serving. NGINX serves a cache file that
-appears on disk after it has started (verified against 1.26: a lookup that
-misses the in-memory index still opens and validates the file), so startup
-never waits for the copy. Requests whose entry has not landed yet are
-ordinary misses. Progress is reported under `archive.restore` in `/status`.
-A restore of ~1 TB / millions of files takes hours; an instance is fully
-warm when `archive.restore.state` is `done`. If the local volume persists
-across restarts, the `.restored` marker makes later starts skip the restore
-(`CACHE_RESTORE=always` forces it; `off` disables it).
+**Restore** (`cache-restore.sh`) walks the archive once, sorts entries
+newest first, and copies them in batches. By default
+(`CACHE_RESTORE_MODE=blocking`) it runs **before NGINX starts**, so a fresh
+instance only answers requests once its cache is fully warm; port 80 stays
+closed meanwhile, so give the orchestrator's health check enough time (on
+Rancher, raise the service's *initializing timeout* or rely on the load
+balancer's check) and keep a second instance serving. A restore of ~1 TB /
+millions of files takes hours. With `CACHE_RESTORE_MODE=background` NGINX
+starts immediately and the copy runs alongside it: NGINX serves a cache file
+that appears on disk after it has started (verified against 1.26), and
+requests whose entry has not landed yet are ordinary misses. Either way,
+progress is under `archive.restore` in `/status` (in blocking mode `/status`
+only becomes reachable when NGINX starts; watch the container log until
+then). If the local volume persists across restarts, the `.restored` marker
+makes later starts skip the restore (`CACHE_RESTORE=always` forces it; `off`
+disables it). rsync writes its partial files to `.rsync-tmp` beside the
+cache tree, never inside it, because NGINX's cache loader deletes any file
+in the tree that does not look like a complete entry.
 
 **Backup** (`cache-backup.sh`) lists entries written since the previous run
 from the local disk (`find -newer`; the NFS side is never walked) and copies
@@ -221,6 +228,7 @@ Variables (all optional):
 
 - `CACHE_ARCHIVE_DIR` (`/cache`), `CACHE_LOCAL_DIR` (`/var/cache/nginx`): the two roots; both hold an `owlery/` tree.
 - `CACHE_RESTORE`: `auto` (default; skip if `.restored` exists), `always`, `off`.
+- `CACHE_RESTORE_MODE`: `blocking` (default; restore, then start NGINX) or `background` (start NGINX, restore alongside).
 - `CACHE_RESTORE_BWLIMIT`, `CACHE_BACKUP_BWLIMIT`: rsync `--bwlimit` in KiB/s (default unlimited).
 - `CACHE_RESTORE_MAX_BYTES`: stop the restore after this many bytes of the newest entries (default: whole archive).
 - `CACHE_RESTORE_BATCH`, `CACHE_BACKUP_BATCH`: entries per rsync invocation (default 5000).

cache-backup.sh

@@ -116,7 +116,7 @@ for batch in "$work"/batch.*; do
     [ -f "$batch" ] || continue
     cut -d' ' -f2- "$batch" > "$batch.paths"
     n="$(wc -l < "$batch" | tr -d ' ')"
-    if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_BACKUP_BWLIMIT"; then
+    if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_BACKUP_BWLIMIT" "$CACHE_ARCHIVE_DIR/.rsync-tmp"; then
         # Exit 24 (vanished source file) is normal: the cache manager evicts
         # entries under max_size while we run. Anything else is counted.
         errors=$(( errors + 1 ))

cache-lib.sh

@@ -153,12 +153,23 @@ cache_lock_release() {
 # Run rsync over a batch file of relative paths from <src> to <dst>.
 # Whole-file copies (no delta computation: entries are immutable blobs),
 # newest wins, never delete, atomic per file via rsync's temp+rename.
+#
+# <tmpdir> must be OUTSIDE the nginx cache tree but on the same filesystem.
+# By default rsync writes its partial `.<name>.XXXXXX` next to the target;
+# nginx's cache loader walks the tree at startup, sees such a file as a
+# too-small cache entry and deletes it from under rsync (seen in 2.0.0 as
+# `[crit] cache file "..." is too small` followed by rsync stat/rename
+# failures). Keeping partials one level up avoids the loader entirely.
 cache_rsync_batch() {
-    src="$1"; dst="$2"; list="$3"; bwlimit="${4:-0}"
+    src="$1"; dst="$2"; list="$3"; bwlimit="${4:-0}"; tmpdir="${5:-}"
     # --files-from implies --relative, so `a/bc/<hash>` lands at the same
     # levels path under <dst> and the intermediate directories are created.
     set -- -a --whole-file --update --modify-window="$CACHE_MODIFY_WINDOW" \
         --files-from="$list" --quiet
+    if [ -n "$tmpdir" ]; then
+        mkdir -p "$tmpdir"
+        set -- "$@" --temp-dir="$tmpdir"
+    fi
     if [ "$bwlimit" != "0" ] && [ -n "$bwlimit" ]; then
         set -- "$@" --bwlimit="$bwlimit"
     fi

cache-restore.sh

@@ -129,23 +129,43 @@ awk -v n="$CACHE_RESTORE_BATCH" -v dir="$work" '{
     if (NR % n == 0) close(f)
 }' "$work/entries.lst"
 
+# rsync partials go here, outside the tree nginx's cache loader walks.
+TMPDIR_RSYNC="$CACHE_LOCAL_DIR/.rsync-tmp"
+retry=""
 for batch in "$work"/batch.*; do
     [ -f "$batch" ] || continue
     cut -d' ' -f2- "$batch" > "$batch.paths"
     n="$(wc -l < "$batch" | tr -d ' ')"
-    if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_RESTORE_BWLIMIT"; then
+    if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_RESTORE_BWLIMIT" "$TMPDIR_RSYNC"; then
         # Entries evicted from the archive between listing and copy show up as
         # vanished files (rsync exit 24); anything else is worth surfacing but
-        # must not abandon the remaining batches.
-        cache_log "restore: rsync reported errors on batch $(basename "$batch"); continuing"
+        # must not abandon the remaining batches. Failed batches get one more
+        # pass at the end (--update makes the repeat cheap).
+        cache_log "restore: rsync reported errors on batch $(basename "$batch"); will retry once"
+        retry="$retry $batch.paths"
+    else
+        rm -f "$batch.paths"
     fi
     files_done=$(( files_done + n ))
     # Bytes are accounted from the listing, not from rsync, so this is the
     # size of the entries considered so far (already-current files included).
     bytes_done=$(( bytes_done + $(awk '{ s += $1 } END { print s + 0 }' "$batch") ))
-    rm -f "$batch" "$batch.paths"
+    rm -f "$batch"
     cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "$files_done/$total_files files"
 done
 
+failed=0
+for paths in $retry; do
+    cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "retrying $(basename "$paths" .paths)"
+    if ! cache_rsync_batch "$SRC" "$DST" "$paths" "$CACHE_RESTORE_BWLIMIT" "$TMPDIR_RSYNC"; then
+        failed=$(( failed + 1 ))
+        cache_log "restore: batch $(basename "$paths" .paths) still reported errors on retry"
+    fi
+done
+
 printf '%s %s files\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$files_done" > "$MARKER"
-finish "done" "$files_done/$total_files entries synchronised from $SRC"
+if [ "$failed" -gt 0 ]; then
+    finish "done" "$files_done/$total_files entries synchronised from $SRC ($failed batches reported errors after retry; run cache-restore.sh --force to repeat)"
+else
+    finish "done" "$files_done/$total_files entries synchronised from $SRC"
+fi

docker-entrypoint.sh

@@ -120,8 +120,25 @@ mkdir -p /var/run/nginx "$CACHE_LOCAL_DIR/owlery"
 chown nginx:nginx /var/run/nginx "$CACHE_LOCAL_DIR" "$CACHE_LOCAL_DIR/owlery" 2>/dev/null || true
 
 /usr/local/bin/health-monitor.sh &
-# Warm the local cache from the archive in the background; nginx serves
-# whatever has landed and treats the rest as ordinary misses meanwhile.
-/usr/local/bin/cache-restore.sh &
+
+# CACHE_RESTORE_MODE=blocking (default): copy the archive into the local
+# cache BEFORE nginx starts, so the instance only answers once it is fully
+# warm. Nothing listens on port 80 meanwhile, so the orchestrator's health
+# check must allow for the restore time (Rancher: raise the service's
+# "initializing timeout", or rely on the load balancer's check instead).
+# CACHE_RESTORE_MODE=background: start nginx immediately and warm the cache
+# concurrently; nginx serves whatever has landed and treats the rest as
+# ordinary misses. Use this when there is no redundant instance to cover.
+export CACHE_RESTORE_MODE="${CACHE_RESTORE_MODE:-blocking}"
+case "$(printf '%s' "$CACHE_RESTORE_MODE" | tr '[:upper:]' '[:lower:]')" in
+    background|async)
+        echo "Cache restore runs in the background (CACHE_RESTORE_MODE=$CACHE_RESTORE_MODE)"
+        /usr/local/bin/cache-restore.sh &
+        ;;
+    *)
+        echo "Cache restore runs before nginx starts (CACHE_RESTORE_MODE=$CACHE_RESTORE_MODE); port 80 stays closed until it finishes"
+        /usr/local/bin/cache-restore.sh || echo "cache-restore.sh exited with status $?; starting nginx anyway"
+        ;;
+esac
 start_backup_scheduler
 exec nginx -g 'daemon off;'

test/cache-sync-test.sh

@@ -97,6 +97,8 @@ assert_file "restores entry 4" "$LOCAL/0/00/$(h 4)"
 assert_eq "--update keeps the newer local entry 1" "$(tail -n 1 "$LOCAL/0/00/$(h 1)")" "fresh-local"
 assert_no_file "temp file is not restored" "$LOCAL/a/bc/$(h 9).0000000042"
 assert_file "marker written" "$CACHE_LOCAL_DIR/.restored"
+[ -d "$CACHE_LOCAL_DIR/.rsync-tmp" ] && ok "rsync temp dir is outside the cache tree" || fail "rsync temp dir missing"
+assert_eq "no rsync partials left in the tree" "$(find "$LOCAL" -name '.*' -type f | wc -l | tr -d ' ')" "0"
 assert_eq "state is done" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "done"
 assert_eq "state counts 4 entries" "$(sed -n 's/.*"files": \([0-9]*\).*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "4"
 grep -q "4 entries" "$WORK/restore.log" && ok "log reports entries" || fail "log: $(cat "$WORK/restore.log")"
@@ -128,6 +130,7 @@ assert_file "new local entry reaches archive" "$ARCHIVE/0/00/$(h 1)"
 assert_eq "--update keeps newer archive copy of 2" "$(tail -n 1 "$ARCHIVE/0/00/$(h 2)")" "archive-newer"
 assert_no_file "temp file is not backed up" "$ARCHIVE/0/00/$(h 7).0000000001"
 assert_file "backup marker written" "$CACHE_LOCAL_DIR/.last-backup"
+[ -d "$CACHE_ARCHIVE_DIR/.rsync-tmp" ] && ok "backup partials kept outside the archive tree" || fail "archive rsync temp dir missing"
 assert_no_file "lock released" "$CACHE_LOCK_DIR"
 assert_eq "state is done" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-backup.json")" "done"