diff --git a/README.md b/README.md index 7d1b88b..117c8a7 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ and `smoke-mirror` do nothing until a mirror fills them. | `smoke` | A sample of the run's keys read back through `HOST`, sizes against the listing; the tlpdb sha512 when `TL` is set; then `smoke-mirror` | | `smoke-mirror` | Hook. Nothing here; a mirror with more to read back defines it | | `report-engine` | Hook. The engine's rows of the run summary | -| `retry` | Run a command, retrying rsync's transport exit codes with backoff; 24 is a success | +| `retry` | Run a command, retrying rsync's transport exit codes with backoff; 23 (an unreadable path, skipped) and 24 (a file vanished mid-transfer) are successes | `normalise`, `pull`, `push`, `batch`, `merge` and `remove` are the verbs those call; `pull` and `push` move an xz-compressed key between the bucket and `.run`, and a diff --git a/engines/rsync.yml b/engines/rsync.yml index ed0e93f..b5d126a 100644 --- a/engines/rsync.yml +++ b/engines/rsync.yml @@ -106,6 +106,9 @@ tasks: # the report count old lines. - rm -f {{.RUN}}/publish.txt {{.RUN}}/orphans.txt - mkdir -p {{.RUN}} {{.STAGING}} + # -L lists every symlink as its file. One whose target is gone upstream is reported + # and skipped, and rsync exits 23 with the rest of the listing intact; retry passes + # that. A listing cut short is caught by LIST_FLOOR, not by the exit code. - task: retry vars: {CMD: '{{.RSYNC}}{{if .FILTER}} {{.FILTER}}{{end}} -rL --list-only {{.SOURCE}} > {{.RUN}}/listing.txt'} - {task: normalise} @@ -487,12 +490,12 @@ tasks: - test ! -d {{.RUN}}/tl || echo "| Signature | $(test -f {{.RUN}}/tl/tlpkg/texlive.tlpdb && echo "tlpdb signed by {{.TL_KEY}}; $(test -f {{.RUN}}/tl/missing.txt && wc -l < {{.RUN}}/tl/missing.txt || echo 0) named containers missing from the bucket" || echo "the tlpdb did not verify") |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" retry: - desc: Run CMD, retrying rsync's transport exit codes (5 10 12 30 35) five times with backoff and jitter; exit 24 (file vanished mid-transfer) is a success + desc: Run CMD, retrying rsync's transport exit codes (5 10 12 30 35) five times with backoff and jitter; 23 (a path it could not read, a dangling symlink under -L) and 24 (file vanished mid-transfer) are successes cmds: - | for i in 1 2 3 4 5; do rc=0; ( {{.CMD}} ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot - case $rc in 0|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac + case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$(({{.RETRY_S}} * 2 ** i + RANDOM % (2 * {{.RETRY_S}} + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml index 097545b..f1e7003 100644 --- a/examples/rsync/Taskfile.yml +++ b/examples/rsync/Taskfile.yml @@ -62,7 +62,8 @@ tasks: - {task: diff, vars: {RUN: '{{.F}}/run-empty'}} - {task: merge, vars: {RUN: '{{.F}}/run-empty', STAGING: '{{.F}}/run-empty/staging', B: '{{.F}}/run-empty/changed.txt'}} - cmp {{.F}}/run-empty/applied.new {{.F}}/run-empty/applied.txt - # retry: 24 is a success, a transport code is retried five times, anything else is final. + # retry: 23 and 24 are successes, a transport code is retried five times, anything else is final. + - task retry CMD='exit 23' RETRY_BASE=0 - task retry CMD='exit 24' RETRY_BASE=0 - '! task retry CMD=''exit 5'' RETRY_BASE=0 2>/dev/null' - '! task retry CMD=''exit 3'' RETRY_BASE=0 2>/dev/null' diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt index a9fe0c7..14c3c3b 100644 --- a/examples/rsync/render.txt +++ b/examples/rsync/render.txt @@ -3,7 +3,7 @@ task: [list] rm -f /work/examples/rsync/.run/publish.txt /work/examples/rsync/.r task: [list] mkdir -p /work/examples/rsync/.run /work/examples/rsync/staging task: [retry] for i in 1 2 3 4 5; do rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://rsync.example.org/pub/ > /work/examples/rsync/.run/listing.txt ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot - case $rc in 0|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac + case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc @@ -53,7 +53,7 @@ task: [prepare] mkdir -p /work/examples/rsync/.run/tl task: [prepare] printf '%s\n' tlpkg/texlive.tlpdb.xz tlpkg/texlive.tlpdb.sha512 tlpkg/texlive.tlpdb.sha512.asc tlpkg/gpg/pubring.gpg > /work/examples/rsync/.run/tl/files.txt task: [retry] for i in 1 2 3 4 5; do rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -Lt --files-from=/work/examples/rsync/.run/tl/files.txt rsync://rsync.example.org/pub// /work/examples/rsync/.run/tl/ ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot - case $rc in 0|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac + case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc @@ -64,7 +64,7 @@ task: [fetch] find /work/examples/rsync/staging -mindepth 1 -delete task: [fetch] cut -f1 > /work/examples/rsync/.run/files.txt task: [retry] for i in 1 2 3 4 5; do rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -Lt --files-from=/work/examples/rsync/.run/files.txt --ignore-missing-args rsync://rsync.example.org/pub/ /work/examples/rsync/staging/ ) || rc=$? # subshell so CMD's exit cannot end the loop; || so Task's errexit cannot - case $rc in 0|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac + case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s done; exit $rc