diff --git a/.github/workflows/cluster-smoke.yml b/.github/workflows/cluster-smoke.yml index 14d161e8e..dce7dcbd9 100644 --- a/.github/workflows/cluster-smoke.yml +++ b/.github/workflows/cluster-smoke.yml @@ -7,11 +7,12 @@ name: HFS Cluster Smoke # (discussion #223). It lands on main ahead of the feature phases because a # workflow is only dispatchable once it exists on the default branch; dispatch # it with ref= to run the branch's version of this file and of -# the smoke script. The skeleton asserts only behavior that is already +# the smoke script. The base checks assert behavior that is already # cluster-safe (shared-Postgres CRUD visibility A→B, round-robin through the -# front) — it calibrates the two-instance harness. Phase 1 adds the SoF -# $export A→B round-trip; Phase 3 adds the WebSocket fan-out A→B case; the -# nightly kill-9 recovery cases (A1/E1) get a schedule trigger when they land. +# front, SIGTERM drain on both instances) — they calibrate the two-instance +# harness. Later cluster PRs add the SoF $sql-export A→B round-trip, the +# WebSocket fan-out A→B case, and the HTS cache-coherency case; the nightly +# tier (kill -9 recovery for the durable job paths) gates on NIGHTLY below. # # Cloned from bulk-export-smoke.yml: same self-hosted runners, remote Docker # host (containers are reached at $DOCKER_HOST_IP, the hfs binaries run on the @@ -19,9 +20,21 @@ name: HFS Cluster Smoke on: workflow_dispatch: + inputs: + nightly: + description: "Also run the nightly tier (kill -9 recovery cases, once they land)" + type: boolean + default: false + schedule: + # Nightly tier: the slow, timing-sensitive recovery cases run here, not + # on every dispatch. Scheduled runs execute main's copy of this file. + - cron: "0 6 * * *" env: CARGO_TERM_COLOR: always + # True on the nightly schedule or a dispatch with nightly=true; the + # recovery steps gate on it. Empty otherwise. + NIGHTLY: ${{ github.event_name == 'schedule' || inputs.nightly == true }} CARGO_BUILD_JOBS: 1 CARGO_PROFILE_DEV_DEBUG: 0 DOCKER_HOST: ${{ secrets.DOCKER_HOST }} @@ -49,9 +62,12 @@ jobs: echo 'rustflags = ["-C", "link-arg=-fuse-ld=lld", "-C", "link-arg=-Wl,-zstack-size=8388608"]' >> ~/.cargo/config.toml - name: Build HFS binary + # subscriptions and elasticsearch are built in now so the later + # cluster cases (WebSocket fan-out, composite-sync durability) do not + # have to touch this job; the R4/postgres smoke below ignores them. run: | cargo build -p helios-hfs --no-default-features \ - --features R4,postgres + --features R4,postgres,subscriptions,elasticsearch - name: Upload HFS binary uses: actions/upload-artifact@v7 @@ -60,6 +76,41 @@ jobs: path: target/debug/hfs retention-days: 1 + build-hts: + name: Build HTS for cluster smoke + runs-on: [self-hosted, Linux] + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - name: Configure Rust to use LLD + run: | + mkdir -p ~/.cargo + rm -f ~/.cargo/config.toml + echo '[target.x86_64-unknown-linux-gnu]' >> ~/.cargo/config.toml + echo 'linker = "clang"' >> ~/.cargo/config.toml + echo 'rustflags = ["-C", "link-arg=-fuse-ld=lld", "-C", "link-arg=-Wl,-zstack-size=8388608"]' >> ~/.cargo/config.toml + + - name: Build HTS binary + # Consumed by the terminology cache-coherency case (two hts processes + # on the shared Postgres) once it lands; built here so that case only + # adds a job that downloads this artifact. + run: | + cargo build -p helios-hts --no-default-features \ + --features R4,postgres + + - name: Upload HTS binary + uses: actions/upload-artifact@v7 + with: + name: hts-cluster-smoke-binary + path: target/debug/hts + retention-days: 1 + cluster-smoke: name: Cluster smoke (R4 / postgres) needs: build @@ -142,11 +193,20 @@ jobs: '' close; } upstream hfs_cluster { - server $RUNNER_IP:$HFS_PORT_A; - server $RUNNER_IP:$HFS_PORT_B; + # max_fails=0: never blacklist an upstream. The default + # (max_fails=1 fail_timeout=10s) turns one refused connect into + # a 10s "no live upstreams" window — a probe that races instance + # startup would poison every request that follows it. + server $RUNNER_IP:$HFS_PORT_A max_fails=0; + server $RUNNER_IP:$HFS_PORT_B max_fails=0; } server { listen 80; + # Answered by nginx itself — readiness probes must not go + # through the upstreams, which may not be started yet. + location = /nginx-health { + return 200 "ok\n"; + } location / { proxy_pass http://hfs_cluster; proxy_http_version 1.1; @@ -166,12 +226,12 @@ jobs: docker start "$NGINX_CONTAINER" echo "NGINX_CONTAINER=$NGINX_CONTAINER" >> "$GITHUB_ENV" - # hfs is not up yet, so accept any HTTP answer (502 included) as - # proof that nginx itself is serving. + # Probe the nginx-local endpoint: hfs is not up yet, and a proxied + # request would count as an upstream failure. for i in {1..30}; do NGINX_PORT=$(docker port "$NGINX_CONTAINER" 80 2>/dev/null | head -1 | sed 's/.*://') if [ -n "$NGINX_PORT" ] \ - && curl -s -o /dev/null "http://$DOCKER_HOST_IP:$NGINX_PORT/health"; then + && curl -sf -o /dev/null "http://$DOCKER_HOST_IP:$NGINX_PORT/nginx-health"; then echo "NGINX_PORT=$NGINX_PORT" >> "$GITHUB_ENV" echo "FRONT_URL=http://$DOCKER_HOST_IP:$NGINX_PORT" >> "$GITHUB_ENV" echo "nginx front is ready on port $NGINX_PORT" @@ -250,22 +310,41 @@ jobs: SMOKE_RUN_SUFFIX="${{ github.run_id }}-${{ github.run_attempt }}" \ ./crates/hfs/tests/cluster/run_external_cluster_smoke.sh - - name: Stop HFS instances gracefully + - name: Stop HFS instances gracefully (SIGTERM) if: always() + # SIGTERM is what a rolling deploy sends. Each instance must log the + # drain and exit on its own within the window; needing SIGKILL is a + # failure of this step (the process is still killed so nothing leaks). run: | - for pid in "${HFS_PID_A:-}" "${HFS_PID_B:-}"; do - if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then - kill -INT "$pid" 2>/dev/null || true - for _ in {1..50}; do - if kill -0 "$pid" 2>/dev/null; then - sleep 0.2 - else - break - fi - done + failed=0 + stop_one() { + local name="$1" pid="$2" log="$3" + if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then + echo "HFS instance $name is not running" + return 0 + fi + kill -TERM "$pid" 2>/dev/null || true + for _ in {1..50}; do + if kill -0 "$pid" 2>/dev/null; then + sleep 0.2 + else + break + fi + done + if kill -0 "$pid" 2>/dev/null; then + echo "::error::HFS instance $name did not exit within 10s of SIGTERM" kill -9 "$pid" 2>/dev/null || true + return 1 fi - done + if ! grep -q 'Shutdown signal received' "$log"; then + echo "::error::HFS instance $name exited on SIGTERM without logging the drain" + return 1 + fi + echo "HFS instance $name drained on SIGTERM" + } + stop_one A "${HFS_PID_A:-}" "${HFS_LOG_A:-/dev/null}" || failed=1 + stop_one B "${HFS_PID_B:-}" "${HFS_LOG_B:-/dev/null}" || failed=1 + exit "$failed" - name: Collect container logs if: always()