diff --git a/.github/scripts/deploy-mieweb-container.sh b/.github/scripts/deploy-mieweb-container.sh index 95534af8..a9ec38c0 100755 --- a/.github/scripts/deploy-mieweb-container.sh +++ b/.github/scripts/deploy-mieweb-container.sh @@ -33,59 +33,8 @@ api_root="${api_root%/api/v1}" api_root="${api_root%/v1}" api_root="${api_root%/api}" api_base="${api_root}/api/v1" - -request() { - local method="$1" - local path="$2" - local body_file="${3:-}" - local response_file status curl_exit - response_file="$(mktemp)" - - set +e - if [ -n "$body_file" ]; then - status=$(curl -sS -o "$response_file" -w "%{http_code}" \ - -X "$method" "${api_base}${path}" \ - -H "Authorization: Bearer ${MIEWEB_API_KEY}" \ - -H "Accept: application/json" \ - -H "Content-Type: application/json" \ - --data-binary "@$body_file") - curl_exit=$? - else - status=$(curl -sS -o "$response_file" -w "%{http_code}" \ - -X "$method" "${api_base}${path}" \ - -H "Authorization: Bearer ${MIEWEB_API_KEY}" \ - -H "Accept: application/json") - curl_exit=$? - fi - set -e - - if [ "$curl_exit" -ne 0 ]; then - echo "::error::${method} ${path} failed before HTTP response (curl exit ${curl_exit})" >&2 - cat "$response_file" >&2 || true - return 1 - fi - - if [ "$status" -lt 200 ] || [ "$status" -ge 300 ]; then - echo "::error::${method} ${path} failed with HTTP ${status}" >&2 - cat "$response_file" >&2 - return 1 - fi - - if [ "$status" = "204" ] || { [ "$method" = "DELETE" ] && [ ! -s "$response_file" ]; }; then - return 0 - fi - - if ! jq -e . "$response_file" >/dev/null 2>&1; then - echo "::error::${method} ${path} returned a non-JSON response from ${api_base}${path}." >&2 - echo "::error::Check LAUNCHPAD_API_URL. The web UI serves HTML at the origin and Swagger at /api; the JSON REST API is at /api/v1." >&2 - echo "Response preview:" >&2 - head -c 500 "$response_file" >&2 || true - echo >&2 - return 1 - fi - - cat "$response_file" -} +# shellcheck source=mieweb-api-request.sh +source "$(dirname "${BASH_SOURCE[0]}")/mieweb-api-request.sh" echo "$CONTAINER_ENV_VARS_JSON" | jq -e ' type == "array" and all(.[]; type == "object" and has("key") and has("value")) diff --git a/.github/scripts/mieweb-api-request.sh b/.github/scripts/mieweb-api-request.sh new file mode 100644 index 00000000..0bd4ea91 --- /dev/null +++ b/.github/scripts/mieweb-api-request.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# Shared request boundary for deploy-mieweb-container.sh. Callers provide +# api_base and MIEWEB_API_KEY. GET requests are safe to retry; state-changing +# requests are attempted once because a lost response is operationally +# ambiguous (the manager may already have applied the change). + +is_transient_curl_exit() { + case "$1" in + 5|6|7|18|28|52|55|56) return 0 ;; + *) return 1 ;; + esac +} + +require_positive_request_integer() { + local name="$1" value="$2" + if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::${name} must be a positive integer (got '${value}')." >&2 + return 1 + fi +} + +request() { + local method="$1" + local path="$2" + local body_file="${3:-}" + local configured_attempts="${MIEWEB_REQUEST_ATTEMPTS:-6}" + local retry_delay="${MIEWEB_REQUEST_RETRY_DELAY_SECONDS:-20}" + local connect_timeout="${MIEWEB_REQUEST_CONNECT_TIMEOUT_SECONDS:-10}" + local max_time="${MIEWEB_REQUEST_MAX_TIME_SECONDS:-30}" + local attempts=1 response_file status curl_exit attempt + + require_positive_request_integer MIEWEB_REQUEST_ATTEMPTS "$configured_attempts" || return 1 + require_positive_request_integer MIEWEB_REQUEST_CONNECT_TIMEOUT_SECONDS "$connect_timeout" || return 1 + require_positive_request_integer MIEWEB_REQUEST_MAX_TIME_SECONDS "$max_time" || return 1 + if ! [[ "$retry_delay" =~ ^[0-9]+$ ]]; then + echo "::error::MIEWEB_REQUEST_RETRY_DELAY_SECONDS must be a non-negative integer (got '${retry_delay}')." >&2 + return 1 + fi + if [ "$method" = "GET" ]; then + attempts="$configured_attempts" + fi + + response_file="$(mktemp)" + for attempt in $(seq 1 "$attempts"); do + : > "$response_file" + set +e + if [ -n "$body_file" ]; then + status=$(curl -sS --connect-timeout "$connect_timeout" --max-time "$max_time" \ + -o "$response_file" -w "%{http_code}" \ + -X "$method" "${api_base}${path}" \ + -H "Authorization: Bearer ${MIEWEB_API_KEY}" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + --data-binary "@$body_file") + curl_exit=$? + else + status=$(curl -sS --connect-timeout "$connect_timeout" --max-time "$max_time" \ + -o "$response_file" -w "%{http_code}" \ + -X "$method" "${api_base}${path}" \ + -H "Authorization: Bearer ${MIEWEB_API_KEY}" \ + -H "Accept: application/json") + curl_exit=$? + fi + set -e + + if [ "$curl_exit" -ne 0 ]; then + if [ "$method" = "GET" ] && is_transient_curl_exit "$curl_exit" && [ "$attempt" -lt "$attempts" ]; then + echo "::warning::${method} ${path} transient failure (curl exit ${curl_exit}), attempt ${attempt}/${attempts}; retrying in ${retry_delay}s." >&2 + [ "$retry_delay" -gt 0 ] && sleep "$retry_delay" + continue + fi + echo "::error::${method} ${path} failed before HTTP response (curl exit ${curl_exit}) after ${attempt} attempt(s)." >&2 + cat "$response_file" >&2 || true + rm -f "$response_file" + return 1 + fi + + if [ "$status" -lt 200 ] || [ "$status" -ge 300 ]; then + echo "::error::${method} ${path} failed with HTTP ${status} after ${attempt} attempt(s)." >&2 + cat "$response_file" >&2 + rm -f "$response_file" + return 1 + fi + + if [ "$status" = "204" ] || { [ "$method" = "DELETE" ] && [ ! -s "$response_file" ]; }; then + rm -f "$response_file" + return 0 + fi + + if ! jq -e . "$response_file" >/dev/null 2>&1; then + echo "::error::${method} ${path} returned a non-JSON response from ${api_base}${path}." >&2 + echo "::error::Check LAUNCHPAD_API_URL. The web UI serves HTML at the origin and Swagger at /api; the JSON REST API is at /api/v1." >&2 + echo "Response preview:" >&2 + head -c 500 "$response_file" >&2 || true + echo >&2 + rm -f "$response_file" + return 1 + fi + + cat "$response_file" + rm -f "$response_file" + return 0 + done + + rm -f "$response_file" + return 1 +} diff --git a/.github/scripts/mieweb-api-request.test.sh b/.github/scripts/mieweb-api-request.test.sh new file mode 100644 index 00000000..78ca12b7 --- /dev/null +++ b/.github/scripts/mieweb-api-request.test.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=mieweb-api-request.sh +source "${repo_root}/.github/scripts/mieweb-api-request.sh" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT +counter_file="${tmp_dir}/counter" +args_file="${tmp_dir}/args" +mode="" + +# Test double for curl. request() still owns all retry, timeout, response-file, +# HTTP-status, and method-safety behavior under test. +curl() { + local response_file="" arg count + printf '%s\n' "$*" >> "$args_file" + while [ "$#" -gt 0 ]; do + arg="$1" + shift + if [ "$arg" = "-o" ]; then + response_file="$1" + shift + fi + done + [ -n "$response_file" ] || fail "curl was not given a response file" + + count="$(cat "$counter_file" 2>/dev/null || echo 0)" + count=$((count + 1)) + echo "$count" > "$counter_file" + + case "$mode" in + transient-then-success) + if [ "$count" -lt 3 ]; then + echo "simulated connect timeout" >&2 + return 28 + fi + printf '{"data":[]}' > "$response_file" + printf '200' + ;; + unauthorized) + printf '{"error":"unauthorized"}' > "$response_file" + printf '401' + ;; + always-transient) + echo "simulated connect timeout" >&2 + return 28 + ;; + *) + fail "unknown curl mode: $mode" + ;; + esac +} + +# JSON validation is not the behavior under test; production and CI provide jq. +jq() { return 0; } + +api_base="https://manager.example/api/v1" +MIEWEB_API_KEY="test-key" +MIEWEB_REQUEST_ATTEMPTS=3 +MIEWEB_REQUEST_RETRY_DELAY_SECONDS=0 +MIEWEB_REQUEST_CONNECT_TIMEOUT_SECONDS=10 +MIEWEB_REQUEST_MAX_TIME_SECONDS=30 + +mode="transient-then-success" +echo 0 > "$counter_file" +: > "$args_file" +result="$(request GET /sites 2>"${tmp_dir}/retry.err")" +[ "$result" = '{"data":[]}' ] || fail "GET did not return the successful JSON response" +[ "$(cat "$counter_file")" = "3" ] || fail "transient GET was not attempted three times" +grep -q -- '--connect-timeout 10' "$args_file" || fail "curl connect timeout is missing" +grep -q -- '--max-time 30' "$args_file" || fail "curl overall timeout is missing" +grep -q 'transient failure.*attempt 1/3' "${tmp_dir}/retry.err" || fail "retry warning is missing" + +mode="unauthorized" +echo 0 > "$counter_file" +if request GET /sites >"${tmp_dir}/unauthorized.out" 2>"${tmp_dir}/unauthorized.err"; then + fail "HTTP 401 unexpectedly succeeded" +fi +[ "$(cat "$counter_file")" = "1" ] || fail "permanent HTTP 401 was retried" + +mode="always-transient" +echo 0 > "$counter_file" +if request POST /sites >"${tmp_dir}/post.out" 2>"${tmp_dir}/post.err"; then + fail "transient POST unexpectedly succeeded" +fi +[ "$(cat "$counter_file")" = "1" ] || fail "non-idempotent POST was retried" + +echo "PASS: MIE manager request retries are bounded and method-safe" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab313c79..ff5d0c02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,14 @@ concurrency: cancel-in-progress: true jobs: + deploy-helper: + name: Deploy helper — Shell test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Test bounded MIE manager requests + run: bash .github/scripts/mieweb-api-request.test.sh + frontend: name: Frontend — Lint, Test, Build runs-on: ubuntu-latest diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index f7f29db4..b220985a 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -54,6 +54,14 @@ responses are wrapped in a `{"data": ...}` envelope, the create body uses `templ > ~436 MB production runtime to keep the pull fast. If a genuinely slow MIE pull still times out, bump > `DEPLOY_JOB_POLL_ATTEMPTS` (up to 360) on a `workflow_dispatch` run. +> **Manager-request resilience.** Every Container Manager call has a 10-second connection timeout +> and a 30-second overall timeout. Safe `GET` calls retry transient curl transport failures up to +> six times with 20-second spacing; `POST` and `DELETE` are attempted +> once because a lost response is ambiguous (the manager may already have applied the state change). +> CI runs `.github/scripts/mieweb-api-request.test.sh` to pin the timeout, retry, and method-safety +> behavior. A manager outage longer than this bounded window still fails the deploy honestly; wait +> for `manager.os.mieweb.org:443` to recover, then rerun the failed workflow jobs. + ### Required GitHub Secrets for MIE deploy | Secret | Purpose | @@ -686,6 +694,9 @@ If any approaches limit, fix that day. Don't wait. **Backend deploy job fails at the MIE manager API** - Confirm the API base resolves to `/api/v1` (the origin serves the SPA; `/api` serves Swagger) - Responses are `{"data": ...}` enveloped; the create body uses `template` + `services[]`; job polling reads `.data.status` (`"success"`) +- For `curl exit 7/28`, verify TCP 443 reachability to the manager. The deploy + retries safe reads within a bounded window; if the control plane remains unavailable, do not loop + state-changing requests manually. Once it recovers, use `gh run rerun --failed`. **Deploy fails with `Container is 'offline', expected running`** - This is a **startup race**, not a crash: the create job reports `success` once the container is diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index 4b9cf575..a34b7c44 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -1,5 +1,17 @@ # Journal +## 2026-07-16 — MIE manager outage: bounded, method-safe deploy retries + +The WebChart stack's final deploy exposed a pre-existing failure mode in +`deploy-mieweb-container.sh`: when `manager.os.mieweb.org:443` was unreachable, the first +`GET /sites` inherited curl's long platform timeout and then failed permanently, even though safe +reads can be retried. The request boundary now applies 10-second connect/30-second overall limits +and retries transient **GET** transport failures six times with bounded backoff. +State-changing POST/DELETE calls remain single-attempt because retrying a lost response could +duplicate or mis-handle an already-applied operation. A no-network shell regression test runs in +CI. This changes deployment tooling only: no runtime, dependency, schema, PHI, or Outcome Status +behavior. + ## 2026-07-16 — WebChart live-integration six-PR prerequisite stack merged Merged the six-PR follow-through stack bottom-up: Doug's confirmed contract and trial findings