diff --git a/README.md b/README.md index 8e22e02..5f9c31f 100644 --- a/README.md +++ b/README.md @@ -179,17 +179,24 @@ with the newest version of each entry winning. The comparison is by mtime, so the nodes and the NAS must agree on time (NTP); `--modify-window=2` absorbs filesystem timestamp granularity, not clock skew. -**Restore** (`cache-restore.sh`, started by the entrypoint in the background) -walks the archive once, sorts entries newest first, and copies them in -batches while NGINX is already serving. NGINX serves a cache file that -appears on disk after it has started (verified against 1.26: a lookup that -misses the in-memory index still opens and validates the file), so startup -never waits for the copy. Requests whose entry has not landed yet are -ordinary misses. Progress is reported under `archive.restore` in `/status`. -A restore of ~1 TB / millions of files takes hours; an instance is fully -warm when `archive.restore.state` is `done`. If the local volume persists -across restarts, the `.restored` marker makes later starts skip the restore -(`CACHE_RESTORE=always` forces it; `off` disables it). +**Restore** (`cache-restore.sh`) walks the archive once, sorts entries +newest first, and copies them in batches. By default +(`CACHE_RESTORE_MODE=blocking`) it runs **before NGINX starts**, so a fresh +instance only answers requests once its cache is fully warm; port 80 stays +closed meanwhile, so give the orchestrator's health check enough time (on +Rancher, raise the service's *initializing timeout* or rely on the load +balancer's check) and keep a second instance serving. A restore of ~1 TB / +millions of files takes hours. With `CACHE_RESTORE_MODE=background` NGINX +starts immediately and the copy runs alongside it: NGINX serves a cache file +that appears on disk after it has started (verified against 1.26), and +requests whose entry has not landed yet are ordinary misses. Either way, +progress is under `archive.restore` in `/status` (in blocking mode `/status` +only becomes reachable when NGINX starts; watch the container log until +then). If the local volume persists across restarts, the `.restored` marker +makes later starts skip the restore (`CACHE_RESTORE=always` forces it; `off` +disables it). rsync writes its partial files to `.rsync-tmp` beside the +cache tree, never inside it, because NGINX's cache loader deletes any file +in the tree that does not look like a complete entry. **Backup** (`cache-backup.sh`) lists entries written since the previous run from the local disk (`find -newer`; the NFS side is never walked) and copies @@ -221,6 +228,7 @@ Variables (all optional): - `CACHE_ARCHIVE_DIR` (`/cache`), `CACHE_LOCAL_DIR` (`/var/cache/nginx`): the two roots; both hold an `owlery/` tree. - `CACHE_RESTORE`: `auto` (default; skip if `.restored` exists), `always`, `off`. +- `CACHE_RESTORE_MODE`: `blocking` (default; restore, then start NGINX) or `background` (start NGINX, restore alongside). - `CACHE_RESTORE_BWLIMIT`, `CACHE_BACKUP_BWLIMIT`: rsync `--bwlimit` in KiB/s (default unlimited). - `CACHE_RESTORE_MAX_BYTES`: stop the restore after this many bytes of the newest entries (default: whole archive). - `CACHE_RESTORE_BATCH`, `CACHE_BACKUP_BATCH`: entries per rsync invocation (default 5000). diff --git a/cache-backup.sh b/cache-backup.sh index df7cf88..5000fd7 100644 --- a/cache-backup.sh +++ b/cache-backup.sh @@ -116,7 +116,7 @@ for batch in "$work"/batch.*; do [ -f "$batch" ] || continue cut -d' ' -f2- "$batch" > "$batch.paths" n="$(wc -l < "$batch" | tr -d ' ')" - if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_BACKUP_BWLIMIT"; then + if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_BACKUP_BWLIMIT" "$CACHE_ARCHIVE_DIR/.rsync-tmp"; then # Exit 24 (vanished source file) is normal: the cache manager evicts # entries under max_size while we run. Anything else is counted. errors=$(( errors + 1 )) diff --git a/cache-lib.sh b/cache-lib.sh index b425bc2..3fca1be 100644 --- a/cache-lib.sh +++ b/cache-lib.sh @@ -153,12 +153,23 @@ cache_lock_release() { # Run rsync over a batch file of relative paths from to . # Whole-file copies (no delta computation: entries are immutable blobs), # newest wins, never delete, atomic per file via rsync's temp+rename. +# +# must be OUTSIDE the nginx cache tree but on the same filesystem. +# By default rsync writes its partial `..XXXXXX` next to the target; +# nginx's cache loader walks the tree at startup, sees such a file as a +# too-small cache entry and deletes it from under rsync (seen in 2.0.0 as +# `[crit] cache file "..." is too small` followed by rsync stat/rename +# failures). Keeping partials one level up avoids the loader entirely. cache_rsync_batch() { - src="$1"; dst="$2"; list="$3"; bwlimit="${4:-0}" + src="$1"; dst="$2"; list="$3"; bwlimit="${4:-0}"; tmpdir="${5:-}" # --files-from implies --relative, so `a/bc/` lands at the same # levels path under and the intermediate directories are created. set -- -a --whole-file --update --modify-window="$CACHE_MODIFY_WINDOW" \ --files-from="$list" --quiet + if [ -n "$tmpdir" ]; then + mkdir -p "$tmpdir" + set -- "$@" --temp-dir="$tmpdir" + fi if [ "$bwlimit" != "0" ] && [ -n "$bwlimit" ]; then set -- "$@" --bwlimit="$bwlimit" fi diff --git a/cache-restore.sh b/cache-restore.sh index 8abdc1a..a30f96b 100644 --- a/cache-restore.sh +++ b/cache-restore.sh @@ -129,23 +129,43 @@ awk -v n="$CACHE_RESTORE_BATCH" -v dir="$work" '{ if (NR % n == 0) close(f) }' "$work/entries.lst" +# rsync partials go here, outside the tree nginx's cache loader walks. +TMPDIR_RSYNC="$CACHE_LOCAL_DIR/.rsync-tmp" +retry="" for batch in "$work"/batch.*; do [ -f "$batch" ] || continue cut -d' ' -f2- "$batch" > "$batch.paths" n="$(wc -l < "$batch" | tr -d ' ')" - if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_RESTORE_BWLIMIT"; then + if ! cache_rsync_batch "$SRC" "$DST" "$batch.paths" "$CACHE_RESTORE_BWLIMIT" "$TMPDIR_RSYNC"; then # Entries evicted from the archive between listing and copy show up as # vanished files (rsync exit 24); anything else is worth surfacing but - # must not abandon the remaining batches. - cache_log "restore: rsync reported errors on batch $(basename "$batch"); continuing" + # must not abandon the remaining batches. Failed batches get one more + # pass at the end (--update makes the repeat cheap). + cache_log "restore: rsync reported errors on batch $(basename "$batch"); will retry once" + retry="$retry $batch.paths" + else + rm -f "$batch.paths" fi files_done=$(( files_done + n )) # Bytes are accounted from the listing, not from rsync, so this is the # size of the entries considered so far (already-current files included). bytes_done=$(( bytes_done + $(awk '{ s += $1 } END { print s + 0 }' "$batch") )) - rm -f "$batch" "$batch.paths" + rm -f "$batch" cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "$files_done/$total_files files" done +failed=0 +for paths in $retry; do + cache_write_state "$STATE_FILE" running "$files_done" "$bytes_done" "$started" "" "retrying $(basename "$paths" .paths)" + if ! cache_rsync_batch "$SRC" "$DST" "$paths" "$CACHE_RESTORE_BWLIMIT" "$TMPDIR_RSYNC"; then + failed=$(( failed + 1 )) + cache_log "restore: batch $(basename "$paths" .paths) still reported errors on retry" + fi +done + printf '%s %s files\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$files_done" > "$MARKER" -finish "done" "$files_done/$total_files entries synchronised from $SRC" +if [ "$failed" -gt 0 ]; then + finish "done" "$files_done/$total_files entries synchronised from $SRC ($failed batches reported errors after retry; run cache-restore.sh --force to repeat)" +else + finish "done" "$files_done/$total_files entries synchronised from $SRC" +fi diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 59684fd..ecb6826 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -120,8 +120,25 @@ mkdir -p /var/run/nginx "$CACHE_LOCAL_DIR/owlery" chown nginx:nginx /var/run/nginx "$CACHE_LOCAL_DIR" "$CACHE_LOCAL_DIR/owlery" 2>/dev/null || true /usr/local/bin/health-monitor.sh & -# Warm the local cache from the archive in the background; nginx serves -# whatever has landed and treats the rest as ordinary misses meanwhile. -/usr/local/bin/cache-restore.sh & + +# CACHE_RESTORE_MODE=blocking (default): copy the archive into the local +# cache BEFORE nginx starts, so the instance only answers once it is fully +# warm. Nothing listens on port 80 meanwhile, so the orchestrator's health +# check must allow for the restore time (Rancher: raise the service's +# "initializing timeout", or rely on the load balancer's check instead). +# CACHE_RESTORE_MODE=background: start nginx immediately and warm the cache +# concurrently; nginx serves whatever has landed and treats the rest as +# ordinary misses. Use this when there is no redundant instance to cover. +export CACHE_RESTORE_MODE="${CACHE_RESTORE_MODE:-blocking}" +case "$(printf '%s' "$CACHE_RESTORE_MODE" | tr '[:upper:]' '[:lower:]')" in + background|async) + echo "Cache restore runs in the background (CACHE_RESTORE_MODE=$CACHE_RESTORE_MODE)" + /usr/local/bin/cache-restore.sh & + ;; + *) + echo "Cache restore runs before nginx starts (CACHE_RESTORE_MODE=$CACHE_RESTORE_MODE); port 80 stays closed until it finishes" + /usr/local/bin/cache-restore.sh || echo "cache-restore.sh exited with status $?; starting nginx anyway" + ;; +esac start_backup_scheduler exec nginx -g 'daemon off;' diff --git a/test/cache-sync-test.sh b/test/cache-sync-test.sh index 08fce29..79c611b 100644 --- a/test/cache-sync-test.sh +++ b/test/cache-sync-test.sh @@ -97,6 +97,8 @@ assert_file "restores entry 4" "$LOCAL/0/00/$(h 4)" assert_eq "--update keeps the newer local entry 1" "$(tail -n 1 "$LOCAL/0/00/$(h 1)")" "fresh-local" assert_no_file "temp file is not restored" "$LOCAL/a/bc/$(h 9).0000000042" assert_file "marker written" "$CACHE_LOCAL_DIR/.restored" +[ -d "$CACHE_LOCAL_DIR/.rsync-tmp" ] && ok "rsync temp dir is outside the cache tree" || fail "rsync temp dir missing" +assert_eq "no rsync partials left in the tree" "$(find "$LOCAL" -name '.*' -type f | wc -l | tr -d ' ')" "0" assert_eq "state is done" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "done" assert_eq "state counts 4 entries" "$(sed -n 's/.*"files": \([0-9]*\).*/\1/p' "$CACHE_STATE_DIR/cache-restore.json")" "4" grep -q "4 entries" "$WORK/restore.log" && ok "log reports entries" || fail "log: $(cat "$WORK/restore.log")" @@ -128,6 +130,7 @@ assert_file "new local entry reaches archive" "$ARCHIVE/0/00/$(h 1)" assert_eq "--update keeps newer archive copy of 2" "$(tail -n 1 "$ARCHIVE/0/00/$(h 2)")" "archive-newer" assert_no_file "temp file is not backed up" "$ARCHIVE/0/00/$(h 7).0000000001" assert_file "backup marker written" "$CACHE_LOCAL_DIR/.last-backup" +[ -d "$CACHE_ARCHIVE_DIR/.rsync-tmp" ] && ok "backup partials kept outside the archive tree" || fail "archive rsync temp dir missing" assert_no_file "lock released" "$CACHE_LOCK_DIR" assert_eq "state is done" "$(sed -n 's/.*"state": "\([a-z]*\)".*/\1/p' "$CACHE_STATE_DIR/cache-backup.json")" "done"