From 2a9ed0462e08e4d80f8dfe804a331a0333d1b927 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Wed, 3 Jun 2026 23:59:53 -0500 Subject: [PATCH 01/18] Add end-to-end integration test harness against a real server (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stand up the runtime/integration half of our testing: a suite that drives a REAL, already-synced Pithead server through the config matrix and asserts the stack behaves — not just the client-side/unit checks we have today. The box is assumed already deployed and synced with miners connected, so the harness moves between scenarios with non-interactive `pithead apply -y` (recreates only changed containers, reuses the synced chain data dirs — never re-syncs, never re-provisions Tor, preserves secrets). It waits on real readiness signals (container health, `pithead status`, dashboard sync %, miner-released) with timeouts — never fixed sleeps — then asserts per scenario: expected containers up / unexpected absent (no monerod in remote mode), Monero synced + pruned/full display, sidechain selection, end-to-end mining (workers online, hashes accumulating), posture propagated to .env, status exit codes, apply idempotency, and secret preservation. `--lifecycle` adds restart, a pool-change apply, and node-down failover (#31). tests/integration/: run.sh entry point — SSH or --local, iterate matrix, assert, restore scenarios.sh declarative config matrix (data, not code) lib.sh target I/O, assertions, readiness waiters, config render, redaction selftest.sh pure-logic self-test (no server) — runs in CI on every PR Safety: never mutates the canonical chains; the destructive prune axis only runs against a separate --pruned/--full-data-dir (else SKIPPED, never silent); secrets are hashed on-box for comparison and redacted from all artifacts; continue-on-error collects the whole matrix. Wired as `make test-integration` (the blocking pre-release gate per #44) with the pure-logic selftest in CI. New docs/integration-testing.md plus index/releasing/ changelog updates. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 7 +- .gitignore | 3 + CHANGELOG.md | 7 + Makefile | 16 +- docs/README.md | 1 + docs/integration-testing.md | 205 +++++++++++++++ docs/releasing.md | 4 + tests/integration/README.md | 28 ++ tests/integration/lib.sh | 214 +++++++++++++++ tests/integration/run.sh | 466 +++++++++++++++++++++++++++++++++ tests/integration/scenarios.sh | 80 ++++++ tests/integration/selftest.sh | 95 +++++++ 12 files changed, 1122 insertions(+), 4 deletions(-) create mode 100644 docs/integration-testing.md create mode 100644 tests/integration/README.md create mode 100644 tests/integration/lib.sh create mode 100755 tests/integration/run.sh create mode 100644 tests/integration/scenarios.sh create mode 100755 tests/integration/selftest.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 342475e..96e2ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,14 @@ jobs: # the job when one is briefly out of sync — see issue #64. - name: Lint pithead and test scripts # Gate on warnings+errors (real issues); info-level style nits vary by shellcheck version. - run: shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh + run: shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh tests/integration/*.sh - name: Run pithead test suite run: bash tests/stack/run.sh + - name: Run integration harness self-test + # Pure-logic checks for the tests/integration/ harness (config rendering, matrix + # coverage, redaction). The LIVE matrix (tests/integration/run.sh) needs a real test + # server and runs as a gated/manual release gate (#54), not on every PR. + run: bash tests/integration/selftest.sh compose: name: Compose config validation diff --git a/.gitignore b/.gitignore index e299bf9..8637a0f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,8 @@ htmlcov/ *.egg-info/ .eggs/ +# Integration test artifacts (manifest, per-scenario logs, captured state) +/tests/integration/results/ + # OS .DS_Store \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b8213..f68e0e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ per the process in [`docs/releasing.md`](docs/releasing.md). ### Added +- End-to-end integration test suite (`tests/integration/`) that drives a real, already-synced + Pithead server through the config matrix and asserts the stack behaves — containers healthy, + nodes synced, miners mining, the dashboard reading correct live state, `status` exit codes, + and secrets preserved across re-applies. Runs over SSH or `--local`, reuses the synced chain + data dirs (never re-syncs), and is the blocking pre-release gate (#54). Surfaced as `make + test-integration`; a pure-logic `selftest` runs in CI on every PR. See + `docs/integration-testing.md`. - Release & versioning scaffold: top-level `VERSION` file (single source of truth), this changelog, and `docs/releasing.md` documenting the release process. The GHCR publishing pipeline and `make release` / `pithead release` command are still diff --git a/Makefile b/Makefile index bfa8c12..ae625ff 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Local test entry points (mirror the GitHub Actions CI jobs). -.PHONY: test test-dashboard test-stack test-compose lint +.PHONY: test test-dashboard test-stack test-compose test-integration test-integration-selftest lint -test: lint test-dashboard test-stack test-compose ## Run everything +test: lint test-dashboard test-stack test-compose test-integration-selftest ## Run everything that doesn't need a server test-dashboard: ## Dashboard unit/component tests with coverage gate cd build/dashboard && PYTHONPATH=. python3 -m pytest \ @@ -13,5 +13,15 @@ test-stack: ## pithead shell test suite test-compose: ## Validate docker-compose.yml interpolation bash tests/stack/test_compose.sh +test-integration-selftest: ## Integration harness pure-logic self-test (no server needed) + bash tests/integration/selftest.sh + +# End-to-end matrix against a REAL test server (issue #54). Needs a provisioned box; pass +# connection + options through ARGS, e.g.: +# make test-integration ARGS="--host miner@10.0.0.5 --dir pithead --lifecycle" +# See docs/integration-testing.md. +test-integration: ## Run the live config-matrix integration suite (requires a test box; pass ARGS=...) + bash tests/integration/run.sh $(ARGS) + lint: ## shellcheck the stack scripts - shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh + shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh tests/integration/*.sh diff --git a/docs/README.md b/docs/README.md index b30dd61..6885d40 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ deeper on individual topics once you're up and running. | [Connecting Miners](workers.md) | Pointing any existing rig at the stack, plus [RigForge](https://github.com/p2pool-starter-stack/rigforge) for setting up new miners. | | [Architecture](architecture.md) | The nine services, how they fit together, the privacy model, and the algorithmic XvB switching engine. | | [Operations & Maintenance](operations.md) | The full `pithead` command reference, upgrades, backups, and troubleshooting. | +| [Integration Testing](integration-testing.md) | The end-to-end config-matrix suite that validates the stack against real Monero + Tari nodes — the blocking pre-release gate. | | [Releasing](releasing.md) | How Pithead is versioned and released — one product, one version, the `VERSION` source of truth, and the GHCR stage→promote pipeline. | | [FAQ](faq.md) | Common questions, plus why Pithead vs. doing it yourself or Gupax. | diff --git a/docs/integration-testing.md b/docs/integration-testing.md new file mode 100644 index 0000000..82c1316 --- /dev/null +++ b/docs/integration-testing.md @@ -0,0 +1,205 @@ +# Integration Testing + +How Pithead is validated end-to-end against a **real Ubuntu server** running full Monero and +full Tari nodes — the runtime/integration half of our testing, and the **blocking pre-release +gate** described in [Releasing](releasing.md) (issue +[#54](https://github.com/p2pool-starter-stack/pithead/issues/54)). + +Our other suites are client-side and never touch a daemon: the `pithead` shell tests stub out +`docker`/`sudo`, the compose test only checks `docker compose config` interpolation, and the +dashboard pytest mocks its clients. They prove the *code* is correct; they can't prove that a +real `apply → sync-gate → mine → status` flow works on a real host. That's what this suite is +for. + +The lives under [`tests/integration/`](../tests/integration/): + +| File | Role | +|---|---| +| `run.sh` | Entry point. Connects to the box (SSH or `--local`), iterates the config matrix, asserts, captures artifacts, restores. | +| `scenarios.sh` | The **declarative config matrix** — adding a case is a one-line data edit. | +| `lib.sh` | Shared helpers: target I/O (SSH/local), assertions, readiness waiters, config rendering, secret redaction. | +| `selftest.sh` | Pure-logic self-test (no server). Runs in CI on every PR. | + +--- + +## How it works + +The suite assumes the box is **already deployed and synced with miners connected** — the whole +point of a dedicated test server is that the full Monero and Tari nodes are synced once and +*reused*, so each scenario runs in minutes instead of waiting days for a chain sync. + +Given that, the harness moves between matrix scenarios with non-interactive **`pithead apply +-y`**, which: + +- recreates only the containers whose resolved config changed, +- **reuses the synced chain data dirs** (it never re-syncs, never re-provisions Tor), and +- **preserves secrets** (`PROXY_AUTH_TOKEN`, onion addresses). + +For each scenario it writes a `config.json`, applies it, **waits on real readiness signals** +(container health, `pithead status`, dashboard sync %, miner-released) with timeouts — never a +fixed `sleep` — then runs the assertion battery below. All reads happen *on the box* +(`pithead status`/`doctor` and `curl http://127.0.0.1:8000/api/state`), so SSH and `--local` +behave identically and we never depend on resolving the box's dashboard hostname. + +Before the first scenario it snapshots the box's original `config.json` and a fingerprint of +its secrets; after the run it **restores the original config** and re-applies (unless +`--keep`). + +### Safety model + +The test box holds real synced nodes and real keys — treat it as production-sensitive. + +- **Never mutates the canonical chains.** The harness only ever writes `config.json` and lets + `apply` recreate containers. It does not `rm -rf` data dirs. The destructive `monero.prune` + axis (a pruned vs. full DB are different on disk) is only exercised against a *separate* + synced data dir you pass with `--pruned-data-dir` / `--full-data-dir`; without it the case + is reported **SKIPPED**, never run against the canonical DB. +- **No silent coverage drops.** Any scenario whose prerequisite is missing (an alt data dir, a + remote endpoint) is logged as `SKIPPED` with the reason — it never quietly disappears. +- **Secrets hygiene.** RPC creds, the proxy token, and onion addresses are never printed. + Secret-preservation is checked by hashing them **on the box** (`sha256sum`) and comparing the + hash — the plaintext never crosses the wire. All captured artifacts are passed through a + redactor. +- **Continue-on-error.** A failing assertion doesn't abort the run; the whole matrix is + collected and summarized, with per-scenario artifacts for the failures. + +--- + +## Provisioning the test box + +A one-time setup. Target the Ubuntu LTS releases we support (22.04 / 24.04). + +1. **Install and deploy Pithead** normally (see [Getting Started](getting-started.md)) and let + it fully sync. You want the box in the steady state: all containers healthy, Monero + Tari + synced, and at least one miner (ideally two) connected and submitting shares. +2. **Reusable synced data.** The synced `monero.data_dir` and `tari.data_dir` are the key + enabler — they're reused across every scenario. The same synced full monerod is also what + the `remote` scenario points at as an external node (see `--remote-monero-host`). +3. **Tools on the box:** `jq`, `curl`, `docker` (with compose v2), and `sha256sum`. The first + three are already Pithead prerequisites; `sha256sum` ships with coreutils. +4. **Access.** Key-based SSH from wherever you run the suite (or run it on the box with + `--local`). If Docker needs root there, use `--pithead "sudo ./pithead"`. +5. *(Optional)* A second synced data dir for the **opposite** prune mode if you want to cover + both pruned and full in one run — see the prune axis above. + +> **Runner security.** Keep the box least-privilege and network-isolated; it holds real keys. +> This is a self-hosted/manual gate, not something we run on public CI. + +--- + +## Running it + +```bash +# Whole matrix over SSH +make test-integration ARGS="--host miner@10.0.0.5 --dir pithead" + +# …or directly +tests/integration/run.sh --host miner@10.0.0.5 --dir pithead + +# On the box itself, plus the lifecycle + node-down failover phase +tests/integration/run.sh --local --dir /home/miner/pithead --lifecycle + +# A single scenario (see --list for names) +tests/integration/run.sh --host miner@10.0.0.5 --scenario remote-main-secure-tari \ + --remote-monero-host 10.0.0.5:18081 + +# Cover both prune modes (needs a second synced DB) +tests/integration/run.sh --host miner@10.0.0.5 --full-data-dir /srv/monero-full +``` + +Useful flags (full list in `run.sh --help`): + +| Flag | Purpose | +|---|---| +| `--host ` / `--local` | Drive the box over SSH, or a stack on this machine. | +| `--dir ` | The Pithead stack directory **on the box** — relative to the SSH login dir or absolute (default `pithead`). Avoid a literal `~`; your local shell expands it before the box sees it. | +| `--pithead ` | How to invoke pithead there (e.g. `"sudo ./pithead"`). | +| `--scenario ` | Run just one scenario. | +| `--workers ` | Miners expected online while mining (default `2`). | +| `--remote-monero-host ` | External node endpoint for the `remote` scenario. | +| `--pruned-data-dir` / `--full-data-dir` | Synced alt DB to enable the opposite prune mode. | +| `--lifecycle` | Also run the lifecycle + node-down failover phase. | +| `--keep` | Don't restore the original config (leave the box on the last scenario). | +| `--out ` | Where to write the manifest and failure artifacts. | +| `--list` | Print the matrix and axis coverage and exit. | + +The runner exits non-zero if any assertion failed. + +--- + +## The config matrix + +Every axis below changes a real runtime path. The matrix covers the realistic combinations and +guarantees **every value of every axis is exercised at least once** (the `selftest` enforces +this, and `--list` prints it). + +| Axis | Values | What it exercises | +|---|---|---| +| `monero.mode` | `local` / `remote` | profile gating, RPC wiring, `status` ignoring monerod in remote mode | +| `monero.prune` | `true` (pruned) / `false` (full) | pruned vs. full display ([#32](https://github.com/p2pool-starter-stack/pithead/issues/32)), DB size | +| `monero.rpc_lan_access` | `false` (127.0.0.1) / `true` (LAN) | RPC bind address, security posture | +| `p2pool.pool` | `main` / `mini` / `nano` | `P2POOL_FLAGS`, sidechain selection | +| `xvb.enabled` | `true` / `false` | XvB tunnel/donor wiring | +| `dashboard.secure` | `true` (Caddy TLS) / `false` | Caddy config / scheme | +| `dashboard.tari_required` | `true` (blocking) / `false` | sync-gate behavior ([#35](https://github.com/p2pool-starter-stack/pithead/issues/35)/[#51](https://github.com/p2pool-starter-stack/pithead/issues/51)) | + +### What each scenario asserts + +- **Expected containers up, unexpected absent** — every service for that config is running and + healthy; in `remote` mode there is **no** `monerod`. +- **`pithead status` exit code** — `0` for a healthy config. +- **Dashboard reads live state** — `/api/state` is reachable; Monero is synced (`done`); + pruned/full display matches `monero.prune` ([#32](https://github.com/p2pool-starter-stack/pithead/issues/32)); the sidechain `pool.type` matches `p2pool.pool`. +- **End-to-end mining** — workers are online (`proxy_workers >= --workers`), stratum has + connections, and total hashes are accumulating ([#28](https://github.com/p2pool-starter-stack/pithead/issues/28)). +- **Posture propagated** — `MONERO_RPC_BIND`, `DASHBOARD_SECURE`, `XVB_ENABLED`, and + `TARI_REQUIRED` in `.env` match the config; the Caddyfile uses the right scheme. +- **Idempotency** — a second `apply -y` with no change is a clean no-op. +- **Secrets preserved** — the proxy token and onion addresses are unchanged across every apply. + +### Lifecycle + failover (`--lifecycle`) + +For one representative config: + +- `restart` brings the stack back healthy (`status` → `0`). +- An `apply` that changes the sidechain recreates only the affected containers and + **preserves secrets**; the dashboard reflects the new pool; then it's reverted. +- **Node-down failover ([#31](https://github.com/p2pool-starter-stack/pithead/issues/31)):** + stop `monerod` → `status` returns non-zero (node down) and the dashboard rejects workers + (stops `xmrig-proxy`) → start `monerod` → workers readmitted → `status` → `0`. + +> `upgrade` (which rebuilds/pulls images) is intentionally **not** run unattended — it's slow +> and changes the bundle under test. Validate it as part of the [release](releasing.md) +> staging smoke test instead. + +--- + +## Artifacts & triage + +Each run writes a **manifest** (`results/manifest.txt`) recording exactly what was under test +— the stack `VERSION`, git revision, and `docker compose images` — so a run is reproducible. + +On a scenario failure, the harness captures (redacted) to `results//`: +`compose-ps.txt`, `status.txt`, `doctor.txt`, `config.json`, `env.redacted.txt`, +`api-state.json`, and `logs.txt` (last 200 lines per service). The end-of-run summary lists +each failed assertion and points at these. + +--- + +## The self-test (CI) + +`tests/integration/selftest.sh` exercises the harness's pure logic — config rendering and +value typing, expectation derivation (profile gating), secret redaction, the SSH/local exec +wrapper, JSON parsing, and **matrix axis coverage** — with no server. It runs in CI on every +PR (the `shell` job) and via `make test-integration-selftest`, so the harness itself is held to +the same lint/test standard as the rest of the stack. + +--- + +## Release gate (#44) + +The live matrix is the **required, blocking pre-release gate**: a release is not promoted or +published unless it's green against the real Monero + Tari nodes. It's surfaced as `make +test-integration` and wired into the `make release` pipeline's test gate — see +[Releasing › Pre-release gate](releasing.md#pre-release-gate-54). The version tagged/published +is the exact bundle this run validated. diff --git a/docs/releasing.md b/docs/releasing.md index 612328e..3636925 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -133,6 +133,10 @@ What exists today: - ✅ Top-level `VERSION` file (single source of truth). - ✅ `CHANGELOG.md` (Keep a Changelog + SemVer, with an `Unreleased` section). - ✅ This document. +- ✅ The [#54](https://github.com/p2pool-starter-stack/pithead/issues/54) integration test + suite — the live config-matrix gate against real nodes (`tests/integration/`, `make + test-integration`). See [Integration Testing](integration-testing.md). Still to wire: making + it a *blocking step* inside the (not-yet-built) `make release` pipeline. **TODO — not yet implemented:** diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000..3c32c58 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,28 @@ +# Integration tests (`tests/integration/`) + +End-to-end suite that drives a **real, already-provisioned Pithead server** through the +config matrix and asserts the stack behaves (issue +[#54](https://github.com/p2pool-starter-stack/pithead/issues/54)). + +``` +run.sh entry point — connects (SSH or --local) and runs the matrix +scenarios.sh the declarative config matrix (data, not code) +lib.sh shared helpers: target I/O, assertions, readiness waiters, redaction +selftest.sh pure-logic self-test (no server) — runs in CI on every PR +``` + +Quick start: + +```bash +# Against a remote box over SSH +make test-integration ARGS="--host miner@10.0.0.5 --dir pithead" + +# On the box itself +./run.sh --local --dir /home/miner/pithead --lifecycle + +# Just the pure-logic checks (no server) +make test-integration-selftest +``` + +**Full guide — provisioning the box, the safety model, the matrix, artifacts, and +CI/release wiring — is in [`docs/integration-testing.md`](../../docs/integration-testing.md).** diff --git a/tests/integration/lib.sh b/tests/integration/lib.sh new file mode 100644 index 0000000..9fd4796 --- /dev/null +++ b/tests/integration/lib.sh @@ -0,0 +1,214 @@ +# shellcheck shell=bash +# +# Shared library for the Pithead integration test harness (tests/integration/). +# +# This file is *sourced*, never executed. It defines pure helpers (config rendering, +# expectation derivation, redaction) plus thin I/O wrappers (run a command on the target, +# poll for readiness) that the runner and the self-test build on. Keeping the pure logic +# here lets tests/integration/selftest.sh exercise it without a real server. +# +# Target model: every command runs *on the box* — either over SSH or, with --local, directly. +# Reads (dashboard JSON, pithead status) therefore behave identically in both modes, and we +# never depend on the runner being able to resolve the box's dashboard hostname. + +# --- Output ----------------------------------------------------------------- +# Colour only on a TTY with NO_COLOR unset (https://no-color.org), matching pithead. +if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then + IT_RESET='\033[0m'; IT_GREEN='\033[1;32m'; IT_YELLOW='\033[1;33m'; IT_RED='\033[1;31m'; IT_DIM='\033[2m' +else + IT_RESET=''; IT_GREEN=''; IT_YELLOW=''; IT_RED=''; IT_DIM='' +fi + +it_log() { echo -e "${IT_GREEN}[ITEST]${IT_RESET} $1"; } +it_warn() { echo -e "${IT_YELLOW}[ITEST]${IT_RESET} $1" >&2; } +it_err() { echo -e "${IT_RED}[ITEST]${IT_RESET} $1" >&2; } +it_step() { echo -e "${IT_DIM} → $1${IT_RESET}"; } + +# --- Secrets hygiene -------------------------------------------------------- +# The box holds real RPC creds, a proxy token, and onion addresses. Redact anything that +# looks secret before it reaches a log file or the terminal. Defence-in-depth: we also avoid +# printing these values in the first place. Patterns cover .env KEY=VALUE lines and .onion +# hostnames. Keep this conservative — over-redaction is safe, leaks are not. +redact() { + sed -E \ + -e 's/(PROXY_AUTH_TOKEN|MONERO_NODE_PASSWORD|MONERO_NODE_USERNAME|.*_PASSWORD|.*_TOKEN|.*_SECRET)=.*/\1=/' \ + -e 's/[a-z2-7]{56}\.onion/.onion/g' +} + +# --- Assertions ------------------------------------------------------------- +# Counters are global so the runner can total them across scenarios. +IT_PASS=0 +IT_FAIL=0 +IT_FAILED_NAMES="" + +it_pass() { IT_PASS=$((IT_PASS + 1)); printf ' %b✓%b %s\n' "$IT_GREEN" "$IT_RESET" "$1"; } +it_fail() { + IT_FAIL=$((IT_FAIL + 1)) + IT_FAILED_NAMES="${IT_FAILED_NAMES}\n - ${IT_CURRENT_SCENARIO:-?}: $1" + printf ' %b✗%b %s\n %s\n' "$IT_RED" "$IT_RESET" "$1" "${2:-}" +} + +assert_eq() { if [ "$2" = "$3" ]; then it_pass "$1"; else it_fail "$1" "expected [$3], got [$2]"; fi; } +assert_ne() { if [ "$2" != "$3" ]; then it_pass "$1"; else it_fail "$1" "expected not [$3]"; fi; } +assert_rc() { if [ "$2" = "$3" ]; then it_pass "$1"; else it_fail "$1" "expected rc $3, got $2"; fi; } +assert_contains() { case "$2" in *"$3"*) it_pass "$1" ;; *) it_fail "$1" "[$2] missing [$3]" ;; esac; } +# Numeric "greater than / >=" with a graceful non-number guard. +assert_num_ge() { + if [ -n "$2" ] && [ "$2" -ge "$3" ] 2>/dev/null; then it_pass "$1"; else it_fail "$1" "expected >= $3, got [$2]"; fi +} +assert_num_gt() { + if [ -n "$2" ] && [ "$2" -gt "$3" ] 2>/dev/null; then it_pass "$1"; else it_fail "$1" "expected > $3, got [$2]"; fi +} + +# --- Config rendering (pure) ------------------------------------------------ +# Map a space-separated list of `dotted.path=value` overrides into a jq program that applies +# them to a config.json. Values are typed: true/false -> boolean, integers -> number, +# everything else -> string. Pure and deterministic so selftest.sh can verify it. +overrides_to_jq() { + local program="." pair path value jsonval + for pair in "$@"; do + [ -z "$pair" ] && continue + path="${pair%%=*}" + value="${pair#*=}" + case "$value" in + true|false) jsonval="$value" ;; + ''|*[!0-9-]*) jsonval="\"$value\"" ;; # has a non-digit -> string + *) jsonval="$value" ;; # all digits (+ optional leading -) -> number + esac + program="${program} | .${path}=${jsonval}" + done + printf '%s' "$program" +} + +# Render a scenario's config.json to stdout: start from the box's baseline config (real +# wallets / data dirs / host preserved) and apply the scenario overrides. Requires jq. +render_scenario_config() { + local baseline_json="$1"; shift + local program; program="$(overrides_to_jq "$@")" + printf '%s' "$baseline_json" | jq "$program" +} + +# --- Expectation derivation (pure) ------------------------------------------ +# Given a rendered config.json, list the services we expect to be running. The bundled +# monerod only runs in local mode (the local_node compose profile); in remote mode it must +# be ABSENT. Everything else is always expected. Mirrors stack_status()'s profile gating. +EXPECTED_ALWAYS="caddy dashboard docker-control docker-proxy p2pool tari tor xmrig-proxy" + +expected_services() { + local config_json="$1" mode + mode="$(printf '%s' "$config_json" | jq -r '.monero.mode // "local"')" + if [ "$mode" = "local" ]; then + printf '%s\n' "monerod $EXPECTED_ALWAYS" | tr ' ' '\n' | sort + else + printf '%s\n' "$EXPECTED_ALWAYS" | tr ' ' '\n' | sort + fi +} + +# Services that must NOT exist for this config (remote mode -> no local monerod). +absent_services() { + local config_json="$1" mode + mode="$(printf '%s' "$config_json" | jq -r '.monero.mode // "local"')" + [ "$mode" = "remote" ] && printf 'monerod\n' +} + +# Human-readable pool label as the dashboard reports it, from the config pool key. +pool_label() { + case "$1" in + main) printf 'Main' ;; + mini) printf 'Mini' ;; + nano) printf 'Nano' ;; + *) printf '%s' "$1" ;; + esac +} + +# --- Target I/O (SSH or local) ---------------------------------------------- +# Globals set by the runner: IT_MODE (ssh|local), IT_SSH_DEST, IT_SSH_OPTS (array), +# IT_REMOTE_DIR, IT_PITHEAD (the pithead invocation, e.g. "./pithead" or "sudo ./pithead"). + +# Run a shell snippet on the target, in the stack directory. The snippet is our own trusted +# code; we never interpolate untrusted data into it. Returns the remote command's exit code. +rx() { + local snippet="$1" + if [ "$IT_MODE" = "local" ]; then + ( cd "$IT_REMOTE_DIR" && bash -c "$snippet" ) + else + local remote + remote="cd $(quote_arg "$IT_REMOTE_DIR") && { $snippet; }" + ssh "${IT_SSH_OPTS[@]}" "$IT_SSH_DEST" "$remote" + fi +} + +# Quote a single argument for safe expansion inside the remote shell string. +quote_arg() { printf '%q' "$1"; } + +# Run pithead with a subcommand on the target, e.g. `pithead status` or `pithead apply -y`. +pithead() { rx "$IT_PITHEAD $*"; } + +# Fetch the dashboard state JSON from the box (dashboard binds 127.0.0.1:8000 on the host +# network). Empty output on failure so callers can detect unreachable. +api_state() { rx "curl -fsS --max-time 10 http://127.0.0.1:8000/api/state" 2>/dev/null; } + +# Pull a jq path out of a JSON blob, printing nothing for an absent/null value. The `?` +# swallows "cannot index null" on a missing parent, and `values` drops nulls — but NOT +# boolean false (so `.monero.prune == false` reads as "false", not ""; `// empty` would +# wrongly swallow it because false is falsy in jq). +jq_get() { printf '%s' "$1" | jq -r "($2)? | values" 2>/dev/null; } + +# --- Readiness waiters ------------------------------------------------------ +# Poll a predicate until it succeeds or the timeout elapses. The interval is a *poll* cadence +# against a real readiness signal — not a fixed "sleep and hope" (issue #54). Returns 0 on +# success, 1 on timeout. +now_s() { date +%s; } + +wait_for() { # wait_for + local timeout="$1" interval="$2" desc="$3"; shift 3 + local deadline=$(( $(now_s) + timeout )) + it_step "waiting for ${desc} (timeout ${timeout}s)…" + while :; do + if "$@"; then return 0; fi + if [ "$(now_s)" -ge "$deadline" ]; then + it_warn "timed out after ${timeout}s waiting for ${desc}" + return 1 + fi + sleep "$interval" + done +} + +# Predicate: pithead status exits 0 (all expected services healthy / intentional-stops aside). +_pred_status_ok() { pithead status >/dev/null 2>&1; } + +# Predicate: the dashboard reports Monero done syncing. +_pred_monero_synced() { + local st; st="$(api_state)"; [ -n "$st" ] || return 1 + [ "$(jq_get "$st" '.sync.monero.state')" = "done" ] +} + +# Predicate: the sync gate has released the miner (xmrig-proxy + p2pool actually running). +_pred_miner_running() { + local st; st="$(api_state)"; [ -n "$st" ] || return 1 + # proxy_workers tracks online workers; >0 means xmrig-proxy is up and accepting miners. + local conns; conns="$(jq_get "$st" '.stratum.conns')" + [ -n "$conns" ] && [ "$conns" -ge 1 ] 2>/dev/null +} + +wait_status_ok() { wait_for "${1:-180}" 5 "pithead status OK" _pred_status_ok; } +wait_monero_synced() { wait_for "${1:-300}" 10 "Monero sync complete" _pred_monero_synced; } +wait_miner_running() { wait_for "${1:-180}" 5 "miner released" _pred_miner_running; } + +# --- Artifact capture ------------------------------------------------------- +# On a scenario failure, collect everything needed to debug it — redacted. Writes into +# //. Best-effort: never let capture failures mask the test result. +capture_artifacts() { + local scenario="$1" outdir="$2" + local dir="${outdir}/${scenario}" + mkdir -p "$dir" + it_step "capturing artifacts to ${dir}" + rx "docker compose ps" 2>&1 | redact > "${dir}/compose-ps.txt" || true + rx "$IT_PITHEAD status" 2>&1 | redact > "${dir}/status.txt" || true + rx "$IT_PITHEAD doctor" 2>&1 | redact > "${dir}/doctor.txt" || true + rx "cat config.json" 2>&1 | redact > "${dir}/config.json" || true + rx "cat .env" 2>&1 | redact > "${dir}/env.redacted.txt" || true + api_state | redact > "${dir}/api-state.json" || true + # Last 200 lines of each service's logs, redacted. + rx "docker compose logs --tail=200 --no-color" 2>&1 | redact > "${dir}/logs.txt" || true +} diff --git a/tests/integration/run.sh b/tests/integration/run.sh new file mode 100755 index 0000000..a98723d --- /dev/null +++ b/tests/integration/run.sh @@ -0,0 +1,466 @@ +#!/usr/bin/env bash +# +# Pithead end-to-end integration test runner (issue #54). +# +# Drives a REAL, already-provisioned Pithead server through the config matrix and asserts the +# stack behaves — containers healthy, nodes synced, miners mining, the dashboard reading the +# right live state, status exit codes correct, and secrets preserved across re-applies. +# +# The box is assumed already deployed and synced with miners connected; the harness moves +# between scenarios with non-interactive `pithead apply -y` (recreates only changed +# containers, reuses the synced chain data dirs — never re-syncs, never re-provisions Tor). +# It saves the box's original config.json up front and restores it at the end. +# +# ./run.sh --host user@1.2.3.4 [--dir ~/pithead] [options] +# ./run.sh --local [--dir /path/to/stack] [options] +# +# Read-only against the canonical chain data dirs; safe to run against the live box. See +# docs/integration-testing.md for provisioning, the safety model, and CI/release wiring. +# +set -uo pipefail # NOT -e: we deliberately continue-on-error to collect the whole matrix. + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/integration/lib.sh +source "$HERE/lib.sh" +# shellcheck source=tests/integration/scenarios.sh +source "$HERE/scenarios.sh" + +# --- Defaults / globals ----------------------------------------------------- +IT_MODE="ssh" +IT_SSH_DEST="" +IT_SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new) +IT_REMOTE_DIR="pithead" +IT_PITHEAD="./pithead" +IT_CURRENT_SCENARIO="" +ONLY_SCENARIO="" +RUN_LIFECYCLE=0 +KEEP_STATE=0 +EXPECTED_WORKERS=2 +REMOTE_MONERO_HOST="" +PRUNED_DATA_DIR="" +FULL_DATA_DIR="" +OUT_DIR="$HERE/results" +BASELINE_CONFIG="" +BASELINE_PRUNE="" +BASELINE_SECRET_FP="" + +usage() { + cat <<'EOF' +Pithead integration test runner + +USAGE: + run.sh --host [options] drive the box over SSH + run.sh --local [options] drive a stack on this machine + +CONNECTION: + --host SSH destination of the test server + --identity SSH private key (adds -i ) + --ssh-opt extra ssh -o option (repeatable), e.g. --ssh-opt Port=2222 + --local run against a stack on this machine instead of over SSH + --dir the Pithead stack directory ON THE BOX, relative to the SSH login + dir or absolute (default: pithead). Avoid a literal ~ — your local + shell would expand it before the box sees it. + --pithead how to invoke pithead on the box (default: ./pithead; + use "sudo ./pithead" if docker needs root there) + +MATRIX: + --scenario run only one scenario (see --list) + --workers miners expected online while mining (default: 2) + --remote-monero-host external node endpoint for the remote-mode scenario + (e.g. the box's own synced node on its LAN IP) + --pruned-data-dir synced PRUNED monero data dir (enables the pruned case when the + box's baseline is full) + --full-data-dir synced FULL monero data dir (enables the full case when the box's + baseline is pruned) + --lifecycle also run the lifecycle + node-down failover phase + --keep do NOT restore the original config.json at the end (leaves the box + on the last scenario — useful for debugging) + +OUTPUT: + --out where to write artifacts (default: tests/integration/results) + --list print the scenario matrix and axis coverage, then exit + -h, --help this help + +Scenarios whose prerequisites are missing (a full/pruned alt data dir, or a remote endpoint) +are reported SKIPPED — never silently dropped, never mutating the canonical synced chain. +EOF +} + +# --- Arg parsing ------------------------------------------------------------ +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --host) IT_SSH_DEST="$2"; IT_MODE="ssh"; shift 2 ;; + --identity) IT_SSH_OPTS+=(-i "$2"); shift 2 ;; + --ssh-opt) IT_SSH_OPTS+=(-o "$2"); shift 2 ;; + --local) IT_MODE="local"; shift ;; + --dir) IT_REMOTE_DIR="$2"; shift 2 ;; + --pithead) IT_PITHEAD="$2"; shift 2 ;; + --scenario) ONLY_SCENARIO="$2"; shift 2 ;; + --workers) EXPECTED_WORKERS="$2"; shift 2 ;; + --remote-monero-host) REMOTE_MONERO_HOST="$2"; shift 2 ;; + --pruned-data-dir) PRUNED_DATA_DIR="$2"; shift 2 ;; + --full-data-dir) FULL_DATA_DIR="$2"; shift 2 ;; + --lifecycle) RUN_LIFECYCLE=1; shift ;; + --keep) KEEP_STATE=1; shift ;; + --out) OUT_DIR="$2"; shift 2 ;; + --list) print_list; exit 0 ;; + -h|--help) usage; exit 0 ;; + *) it_err "Unknown option: $1 (try --help)"; exit 2 ;; + esac + done + + if [ "$IT_MODE" = "ssh" ] && [ -z "$IT_SSH_DEST" ]; then + it_err "Provide --host or --local. See --help." + exit 2 + fi +} + +print_list() { + echo "Scenarios:" + local name rest + while IFS=$'\t' read -r name rest; do + printf ' %-32s %s\n' "$name" "$rest" + done < <(scenario_matrix) + echo "" + echo "Axis coverage (every value below must appear at least once):" + axis_coverage | sed 's/^/ /' +} + +# --- Target I/O helpers (depend on globals set above) ----------------------- +# Write a config.json onto the box from stdin-less arg. +push_config() { + local json="$1" + if [ "$IT_MODE" = "local" ]; then + printf '%s\n' "$json" > "$IT_REMOTE_DIR/config.json" + else + printf '%s\n' "$json" | ssh "${IT_SSH_OPTS[@]}" "$IT_SSH_DEST" \ + "cd $(quote_arg "$IT_REMOTE_DIR") && cat > config.json" + fi +} + +# Read a single (non-secret) .env value off the box. +env_on_box() { rx "grep -E '^$1=' .env 2>/dev/null | head -n1 | cut -d= -f2-"; } + +# Services currently running, one per line, sorted. Honours active compose profiles, so +# monerod is absent in remote mode. +running_services() { rx "docker compose ps --services --status running 2>/dev/null | sort"; } + +# A stable fingerprint of the secrets we must preserve across applies (proxy token + onion +# addresses). Hashed ON THE BOX so the plaintext never crosses the wire or hits a log. +secret_fingerprint() { + rx "grep -E '^(PROXY_AUTH_TOKEN|[A-Z]+_ONION_ADDRESS)=' .env 2>/dev/null | sort | sha256sum | cut -d' ' -f1" +} + +# --- Preflight -------------------------------------------------------------- +preflight() { + it_log "Connecting to target ($IT_MODE${IT_SSH_DEST:+ $IT_SSH_DEST}) at $IT_REMOTE_DIR …" + if ! rx "true" >/dev/null 2>&1; then + it_err "Cannot reach the target. Check --host/--local, --dir, and SSH access." + exit 1 + fi + + # The stack dir must contain a deployed pithead. + if ! rx "test -x $IT_PITHEAD" >/dev/null 2>&1; then + it_err "pithead not found/executable at $IT_REMOTE_DIR/$IT_PITHEAD (set --dir/--pithead)." + exit 1 + fi + if ! rx "grep -q '^DEPLOYMENT_COMPLETED=true' .env" >/dev/null 2>&1; then + it_err "Box is not fully deployed (.env missing DEPLOYMENT_COMPLETED). Run 'pithead setup' there first." + exit 1 + fi + + # Tools the harness leans on, on the box. + local tool + for tool in jq curl docker sha256sum; do + if ! rx "command -v $tool" >/dev/null 2>&1; then + it_err "Required tool '$tool' missing on the box." + exit 1 + fi + done + + mkdir -p "$OUT_DIR" + record_manifest + + # Snapshot the baseline so we can restore it and compare secrets later. + BASELINE_CONFIG="$(rx 'cat config.json')" + BASELINE_PRUNE="$(env_on_box MONERO_PRUNE)" # 1 = pruned, 0 = full + BASELINE_SECRET_FP="$(secret_fingerprint)" + if [ -z "$BASELINE_CONFIG" ]; then + it_err "Could not read baseline config.json from the box." + exit 1 + fi + it_log "Baseline captured (prune=$BASELINE_PRUNE). Original config will be restored at the end." +} + +# Record exactly what's under test, so a run is reproducible (#54 manifest). +record_manifest() { + local f="$OUT_DIR/manifest.txt" + { + echo "# Pithead integration run manifest" + echo "stack_version: $(rx 'cat VERSION 2>/dev/null' | tr -d '\n')" + echo "git_rev: $(rx 'git rev-parse --short HEAD 2>/dev/null' | tr -d '\n')" + echo "target_mode: $IT_MODE" + echo "remote_dir: $IT_REMOTE_DIR" + echo "expected_workers: $EXPECTED_WORKERS" + echo "" + echo "# docker compose images" + rx "docker compose images 2>/dev/null" + } | redact > "$f" 2>/dev/null || true + it_step "wrote run manifest to $f" +} + +# --- Scenario execution ----------------------------------------------------- +# Decide whether a scenario can run on this box, augmenting its overrides where needed (an +# alt data dir for the prune axis, a remote endpoint for remote mode). On success sets +# RESOLVED to the final override string and returns 0; on a missing prerequisite sets +# SKIP_REASON and returns 1. No silent drops, and never mutates the canonical chain. +RESOLVED="" +SKIP_REASON="" +resolve_overrides() { + local overrides="$1" prune mode out="$1" + RESOLVED=""; SKIP_REASON="" + + prune="$(printf '%s' "$overrides" | tr ' ' '\n' | sed -n 's/^monero\.prune=//p')" + mode="$(printf '%s' "$overrides" | tr ' ' '\n' | sed -n 's/^monero\.mode=//p')" + + # Prune axis: only flip away from the baseline DB if a matching synced dir is provided — + # flipping prune on the canonical dir would invalidate it (a DEST change). + if [ "$prune" = "true" ] && [ "$BASELINE_PRUNE" = "0" ]; then + [ -n "$PRUNED_DATA_DIR" ] || { SKIP_REASON="needs --pruned-data-dir (box baseline is full)"; return 1; } + out="$out monero.data_dir=$PRUNED_DATA_DIR" + fi + if [ "$prune" = "false" ] && [ "$BASELINE_PRUNE" = "1" ]; then + [ -n "$FULL_DATA_DIR" ] || { SKIP_REASON="needs --full-data-dir (box baseline is pruned)"; return 1; } + out="$out monero.data_dir=$FULL_DATA_DIR" + fi + + # Remote mode needs an external endpoint to point at. + if [ "$mode" = "remote" ]; then + [ -n "$REMOTE_MONERO_HOST" ] || { SKIP_REASON="needs --remote-monero-host"; return 1; } + out="$out monero.remote.host=$REMOTE_MONERO_HOST" + fi + + RESOLVED="$out" + return 0 +} + +run_scenario() { + local name="$1" overrides="$2" + IT_CURRENT_SCENARIO="$name" + echo "" + it_log "── scenario: ${name} ───────────────────────────────" + + if ! resolve_overrides "$overrides"; then + it_warn "SKIPPED ${name}: ${SKIP_REASON}" + IT_SKIPPED=$((IT_SKIPPED + 1)) + return 0 + fi + + # Render + push config, then apply non-interactively. + local config + # shellcheck disable=SC2086 # RESOLVED is a space-separated list of override tokens, on purpose + config="$(render_scenario_config "$BASELINE_CONFIG" $RESOLVED)" + if ! printf '%s' "$config" | jq empty 2>/dev/null; then + it_fail "rendered config is valid JSON" "jq rejected the rendered config" + return 0 + fi + push_config "$config" + + it_step "applying config (pithead apply -y)…" + if ! pithead apply -y > "$OUT_DIR/${name}.apply.log" 2>&1; then + it_fail "apply succeeded" "see $OUT_DIR/${name}.apply.log" + capture_artifacts "$name" "$OUT_DIR" + return 0 + fi + + # Wait for the stack to settle on real readiness signals before asserting. + wait_status_ok 240 || true + wait_monero_synced 120 || true + wait_miner_running 180 || true + + local fails_before="$IT_FAIL" + assert_scenario "$name" "$config" + # If this scenario turned anything red, grab artifacts for it. + [ "$IT_FAIL" -gt "$fails_before" ] && capture_artifacts "$name" "$OUT_DIR" + return 0 +} + +# The per-scenario assertion battery (infrastructure-level, from the issue). +assert_scenario() { + local name="$1" config="$2" + local st mode prune pool secure tari_req xvb rpc_lan + mode="$(jq_get "$config" '.monero.mode')"; mode="${mode:-local}" + prune="$(jq_get "$config" '.monero.prune')" + pool="$(jq_get "$config" '.p2pool.pool')"; pool="${pool:-main}" + secure="$(jq_get "$config" '.dashboard.secure')" + tari_req="$(jq_get "$config" '.dashboard.tari_required')" + xvb="$(jq_get "$config" '.xvb.enabled')" + rpc_lan="$(jq_get "$config" '.monero.rpc_lan_access')" + + # 1. Expected containers up; unexpected ones absent. + local running expected svc + running="$(running_services)" + expected="$(expected_services "$config")" + while IFS= read -r svc; do + [ -z "$svc" ] && continue + case "$running" in + *"$svc"*) it_pass "container up: $svc" ;; + *) it_fail "container up: $svc" "not in running services" ;; + esac + done <<< "$expected" + if [ "$mode" = "remote" ]; then + case "$running" in + *monerod*) it_fail "monerod absent in remote mode" "monerod is running" ;; + *) it_pass "monerod absent in remote mode" ;; + esac + fi + + # 2. pithead status is green for a healthy config. + pithead status >/dev/null 2>&1; assert_rc "status exit code is 0 (healthy)" "$?" "0" + + # 3. Dashboard reachable and reading live state. + st="$(api_state)" + if [ -z "$st" ]; then + it_fail "dashboard /api/state reachable" "empty response" + return 0 + fi + it_pass "dashboard /api/state reachable" + + # 4. Monero synced, and pruned/full matches config (#32). + assert_eq "monero sync complete" "$(jq_get "$st" '.sync.monero.state')" "done" + local want_mode; [ "$prune" = "false" ] && want_mode="Full" || want_mode="Pruned" + assert_eq "monero display mode" "$(jq_get "$st" '.monero.mode')" "$want_mode" + + # 5. Sidechain selection matches the pool axis. + assert_eq "pool type" "$(jq_get "$st" '.pool.type')" "$(pool_label "$pool")" + + # 6. End-to-end mining: workers online, hashes accumulating (#28). + local workers conns hashes + workers="$(jq_get "$st" '.proxy_workers')" + conns="$(jq_get "$st" '.stratum.conns')" + hashes="$(jq_get "$st" '.stratum.total_hashes')" + assert_num_ge "workers online (>= $EXPECTED_WORKERS)" "${workers:-0}" "$EXPECTED_WORKERS" + assert_num_ge "stratum connections" "${conns:-0}" 1 + assert_num_gt "stratum total hashes > 0" "${hashes:-0}" 0 + + # 7. Tari sync-gate posture matches tari_required. + assert_eq "TARI_REQUIRED env matches config" "$(env_on_box TARI_REQUIRED)" "${tari_req:-true}" + if [ "$tari_req" = "true" ]; then + assert_eq "tari synced (required)" "$(jq_get "$st" '.sync.tari.state')" "done" + fi + + # 8. Security/posture axes propagated to .env (#configuration). + local want_bind; [ "$rpc_lan" = "true" ] && want_bind="0.0.0.0" || want_bind="127.0.0.1" + assert_eq "MONERO_RPC_BIND matches rpc_lan_access" "$(env_on_box MONERO_RPC_BIND)" "$want_bind" + assert_eq "DASHBOARD_SECURE matches config" "$(env_on_box DASHBOARD_SECURE)" "${secure:-true}" + assert_eq "XVB_ENABLED matches config" "$(env_on_box XVB_ENABLED)" "${xvb:-true}" + + # 9. Caddy scheme matches dashboard.secure. + local scheme; [ "$secure" = "false" ] && scheme="http://" || scheme="https://" + assert_contains "Caddyfile uses correct scheme" "$(rx 'head -n1 Caddyfile 2>/dev/null')" "$scheme" + + # 10. Idempotency: a second apply with no change is a clean no-op. + local again; again="$(pithead apply -y 2>&1)" + assert_contains "re-apply is a no-op" "$again" "No configuration changes detected" + + # 11. Secrets preserved across every apply so far (proxy token + onions unchanged). + assert_eq "secrets preserved (token + onions)" "$(secret_fingerprint)" "$BASELINE_SECRET_FP" +} + +# --- Lifecycle + edge phase (--lifecycle) ----------------------------------- +run_lifecycle() { + IT_CURRENT_SCENARIO="lifecycle" + echo "" + it_log "── lifecycle + failover phase ──────────────────────" + + # restart brings the stack back healthy. + it_step "pithead restart…" + pithead restart >/dev/null 2>&1 + wait_status_ok 240 || true + pithead status >/dev/null 2>&1; assert_rc "status OK after restart" "$?" "0" + + # apply that changes the sidechain recreates only the affected containers, preserving + # secrets. We flip main<->mini and assert the token/onions are untouched, then revert. + local cur_pool fp_before + cur_pool="$(jq_get "$BASELINE_CONFIG" '.p2pool.pool')"; cur_pool="${cur_pool:-main}" + local other; [ "$cur_pool" = "mini" ] && other="main" || other="mini" + fp_before="$(secret_fingerprint)" + push_config "$(render_scenario_config "$BASELINE_CONFIG" "p2pool.pool=$other")" + it_step "apply pool $cur_pool -> $other…" + pithead apply -y >/dev/null 2>&1 + wait_status_ok 180 || true + assert_eq "secrets preserved across pool change" "$(secret_fingerprint)" "$fp_before" + assert_eq "pool actually changed" "$(jq_get "$(api_state)" '.pool.type')" "$(pool_label "$other")" + + # Node-down failover (#31): stop monerod -> status non-zero (node down), dashboard rejects + # workers (xmrig-proxy stopped) -> start monerod -> readmitted -> status 0 again. + if [ "$(env_on_box COMPOSE_PROFILES)" = "local_node" ]; then + it_step "stopping monerod to exercise node-down failover…" + rx "docker compose stop monerod" >/dev/null 2>&1 + wait_for 120 5 "status to report node down" _pred_status_down || true + pithead status >/dev/null 2>&1; assert_rc "status non-zero when node down" "$?" "1" + it_step "starting monerod and waiting for readmit…" + rx "docker compose start monerod" >/dev/null 2>&1 + wait_status_ok 240 || true + pithead status >/dev/null 2>&1; assert_rc "status OK after node recovery" "$?" "0" + else + it_warn "skipping node-down failover (remote mode: no local monerod to stop)" + fi +} + +# Predicate: status reports a problem (non-zero) — used to detect node-down deterministically. +_pred_status_down() { ! pithead status >/dev/null 2>&1; } + +# --- Restore + summary ------------------------------------------------------ +restore_baseline() { + [ "$KEEP_STATE" = "1" ] && { it_warn "--keep set: leaving the box on the last scenario."; return; } + [ -z "$BASELINE_CONFIG" ] && return + it_log "Restoring original config.json and re-applying…" + push_config "$BASELINE_CONFIG" + pithead apply -y >/dev/null 2>&1 || it_warn "restore apply reported a non-zero exit; check the box." + wait_status_ok 240 || true + assert_eq "secrets intact after restore" "$(secret_fingerprint)" "$BASELINE_SECRET_FP" +} + +summary() { + echo "" + it_log "════════════════ summary ════════════════" + it_log "passed: $IT_PASS" + it_log "skipped: $IT_SKIPPED" + if [ "$IT_FAIL" -gt 0 ]; then + it_err "failed: $IT_FAIL" + echo -e "$IT_FAILED_NAMES" >&2 + it_err "Artifacts for failed scenarios are under $OUT_DIR/" + return 1 + fi + it_log "failed: 0" + it_log "All assertions passed. Artifacts/manifest under $OUT_DIR/" + return 0 +} + +# --- Main ------------------------------------------------------------------- +IT_SKIPPED=0 + +main() { + parse_args "$@" + preflight + + local name rest + if [ -n "$ONLY_SCENARIO" ]; then + rest="$(scenario_overrides "$ONLY_SCENARIO")" || { it_err "Unknown scenario: $ONLY_SCENARIO"; exit 2; } + run_scenario "$ONLY_SCENARIO" "$rest" + else + while IFS=$'\t' read -r name rest; do + [ -z "$name" ] && continue + run_scenario "$name" "$rest" + done < <(scenario_matrix) + fi + + [ "$RUN_LIFECYCLE" = "1" ] && run_lifecycle + + restore_baseline + summary +} + +main "$@" diff --git a/tests/integration/scenarios.sh b/tests/integration/scenarios.sh new file mode 100644 index 0000000..1db4373 --- /dev/null +++ b/tests/integration/scenarios.sh @@ -0,0 +1,80 @@ +# shellcheck shell=bash +# +# Declarative config matrix for the integration suite (issue #54). +# +# Each scenario is a NAME and a set of `dotted.path=value` overrides applied to the box's +# baseline config.json (see lib.sh:render_scenario_config). Keeping the matrix as data — not +# code — means adding a case is a one-line edit, and selftest.sh can prove that every value +# of every axis is exercised at least once (an acceptance criterion of #54). +# +# The full cross-product is large; we cover the realistic combinations and guarantee each +# axis value appears once. Axes (from the issue): +# monero.mode .............. local | remote +# monero.prune ............. true (pruned) | false (full) +# monero.rpc_lan_access .... false (127.0.0.1) | true (LAN bind) +# p2pool.pool .............. main | mini | nano +# xvb.enabled .............. true | false +# dashboard.secure ......... true (Caddy TLS) | false +# dashboard.tari_required .. true (blocking) | false (non-blocking) +# +# Prerequisite-gated axes (skipped-with-a-loud-log, never silently, when the box can't host +# them — see run.sh): +# * monero.prune=false (full) and =true (pruned) are different on-disk DBs. We only flip +# prune when a matching synced data dir is available; otherwise the case is reported +# SKIPPED so we never silently drop coverage or mutate the canonical chain. +# * monero.mode=remote needs a reachable external node (REMOTE_MONERO_HOST); the natural +# choice is the box's own synced monerod on its LAN address. + +# Emit the matrix as `NAMEoverrides…`, one scenario per line. Lines starting with the +# canonical-first scenario are ordered so the cheapest, most-common config runs first. +scenario_matrix() { + cat <<'EOF' +local-pruned-main-secure-tari monero.mode=local monero.prune=true monero.rpc_lan_access=false p2pool.pool=main xvb.enabled=true dashboard.secure=true dashboard.tari_required=true +local-full-main-secure-tari monero.mode=local monero.prune=false p2pool.pool=main xvb.enabled=true dashboard.secure=true dashboard.tari_required=true +local-pruned-mini-secure-tari monero.mode=local monero.prune=true p2pool.pool=mini xvb.enabled=true dashboard.secure=true dashboard.tari_required=true +local-pruned-nano-insecure monero.mode=local monero.prune=true p2pool.pool=nano xvb.enabled=true dashboard.secure=false dashboard.tari_required=true +local-pruned-main-rpclan monero.mode=local monero.prune=true monero.rpc_lan_access=true p2pool.pool=main xvb.enabled=true dashboard.secure=true dashboard.tari_required=true +local-pruned-main-xvb-off monero.mode=local monero.prune=true p2pool.pool=main xvb.enabled=false dashboard.secure=true dashboard.tari_required=true +local-pruned-main-tari-optional monero.mode=local monero.prune=true p2pool.pool=main xvb.enabled=true dashboard.secure=true dashboard.tari_required=false +remote-main-secure-tari monero.mode=remote p2pool.pool=main xvb.enabled=true dashboard.secure=true dashboard.tari_required=true +EOF +} + +# The axis -> values map the matrix must cover. selftest.sh asserts every value below appears +# in at least one scenario's overrides (or is justified as prerequisite-gated). +axis_coverage() { + cat <<'EOF' +monero.mode=local +monero.mode=remote +monero.prune=true +monero.prune=false +monero.rpc_lan_access=true +monero.rpc_lan_access=false +p2pool.pool=main +p2pool.pool=mini +p2pool.pool=nano +xvb.enabled=true +xvb.enabled=false +dashboard.secure=true +dashboard.secure=false +dashboard.tari_required=true +dashboard.tari_required=false +EOF +} + +# Print the override string for a named scenario (empty if not found). +scenario_overrides() { + local want="$1" name rest + while IFS=$'\t' read -r name rest; do + [ "$name" = "$want" ] && { printf '%s' "$rest"; return 0; } + done < <(scenario_matrix) + return 1 +} + +# Print just the scenario names, one per line. +scenario_names() { + local name rest + while IFS=$'\t' read -r name rest; do + [ -n "$name" ] && printf '%s\n' "$name" + done < <(scenario_matrix) +} diff --git a/tests/integration/selftest.sh b/tests/integration/selftest.sh new file mode 100755 index 0000000..db1079d --- /dev/null +++ b/tests/integration/selftest.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Self-test for the integration harness's pure logic (config rendering, expectation +# derivation, redaction, matrix coverage, the SSH/local exec wrapper, JSON parsing). +# +# This runs anywhere — no real server needed — so it can gate every PR (unlike the live +# matrix in run.sh, which needs the test box). It dogfoods the very assertion helpers the +# harness ships. Run: tests/integration/selftest.sh +# +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tests/integration/lib.sh +source "$HERE/lib.sh" +# shellcheck source=tests/integration/scenarios.sh +source "$HERE/scenarios.sh" + +echo "== overrides_to_jq: value typing ==" +assert_contains "boolean stays unquoted" "$(overrides_to_jq monero.prune=false)" '.monero.prune=false' +assert_contains "string gets quoted" "$(overrides_to_jq monero.mode=remote)" '.monero.mode="remote"' +assert_contains "integer stays unquoted" "$(overrides_to_jq monero.remote.rpc_port=18081)" '.monero.remote.rpc_port=18081' +assert_contains "negative int unquoted" "$(overrides_to_jq foo=-5)" '.foo=-5' +assert_contains "dotted ip is a string" "$(overrides_to_jq monero.remote.host=10.0.0.5)" '.monero.remote.host="10.0.0.5"' + +echo "== render_scenario_config: applies overrides, stays valid JSON ==" +BASE='{"monero":{"mode":"local","prune":true,"wallet_address":"49keep"},"p2pool":{"pool":"main"}}' +RENDERED="$(render_scenario_config "$BASE" monero.mode=remote monero.prune=false p2pool.pool=mini)" +printf '%s' "$RENDERED" | jq empty 2>/dev/null && it_pass "rendered config is valid JSON" || it_fail "rendered config is valid JSON" "jq rejected it" +assert_eq "override: mode" "$(jq_get "$RENDERED" '.monero.mode')" "remote" +assert_eq "override: prune" "$(jq_get "$RENDERED" '.monero.prune')" "false" +assert_eq "override: pool" "$(jq_get "$RENDERED" '.p2pool.pool')" "mini" +assert_eq "preserved: wallet" "$(jq_get "$RENDERED" '.monero.wallet_address')" "49keep" + +echo "== expected/absent services: profile gating ==" +LOCAL='{"monero":{"mode":"local"}}' +REMOTE='{"monero":{"mode":"remote"}}' +assert_contains "local includes monerod" "$(expected_services "$LOCAL")" "monerod" +assert_contains "local includes p2pool" "$(expected_services "$LOCAL")" "p2pool" +case "$(expected_services "$REMOTE")" in *monerod*) it_fail "remote excludes monerod" "monerod present" ;; *) it_pass "remote excludes monerod" ;; esac +assert_eq "remote marks monerod absent" "$(absent_services "$REMOTE")" "monerod" +assert_eq "local marks nothing absent" "$(absent_services "$LOCAL")" "" +assert_eq "pool_label main" "$(pool_label main)" "Main" +assert_eq "pool_label nano" "$(pool_label nano)" "Nano" + +echo "== redact: secrets never leak into artifacts ==" +ONION="$(printf 'a%.0s' $(seq 1 56)).onion" +SECRETS="$(printf 'PROXY_AUTH_TOKEN=deadbeefcafe\nMONERO_NODE_PASSWORD=hunter2\nMONERO_ONION_ADDRESS=%s\nHOST_IP=box.lan\n' "$ONION")" +REDACTED="$(printf '%s' "$SECRETS" | redact)" +assert_contains "token redacted" "$REDACTED" "PROXY_AUTH_TOKEN=" +assert_contains "password redacted" "$REDACTED" "MONERO_NODE_PASSWORD=" +assert_contains "onion redacted" "$REDACTED" ".onion" +assert_contains "non-secret kept" "$REDACTED" "HOST_IP=box.lan" +case "$REDACTED" in *deadbeefcafe*) it_fail "raw token absent" "token leaked" ;; *) it_pass "raw token absent" ;; esac + +echo "== matrix: every axis value is covered ==" +CORPUS="$(scenario_matrix | cut -f2 | tr '\n' ' ')" +while IFS= read -r val; do + [ -z "$val" ] && continue + case " $CORPUS " in + *" $val "*) it_pass "axis covered: $val" ;; + *) it_fail "axis covered: $val" "no scenario sets $val" ;; + esac +done < <(axis_coverage) + +echo "== scenarios: lookup helpers ==" +assert_ne "scenario_names is non-empty" "$(scenario_names | head -n1)" "" +assert_contains "overrides lookup works" "$(scenario_overrides remote-main-secure-tari)" "monero.mode=remote" + +echo "== rx: local exec runs in the stack dir ==" +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +printf 'marker' > "$TMP/sentinel" +IT_MODE="local"; IT_REMOTE_DIR="$TMP" +assert_eq "rx runs command on target" "$(rx 'cat sentinel')" "marker" +assert_eq "rx cwd is the stack dir" "$(rx 'pwd')" "$TMP" + +echo "== api_state + jq_get: parse a fixture ==" +# Stub rx so api_state returns a representative /api/state payload. +FIXTURE='{"sync":{"monero":{"state":"done"},"tari":{"state":"done"}},"monero":{"mode":"Pruned"},"pool":{"type":"Main"},"proxy_workers":2,"stratum":{"conns":2,"total_hashes":12345}}' +rx() { printf '%s' "$FIXTURE"; } +ST="$(api_state)" +assert_eq "parse monero sync state" "$(jq_get "$ST" '.sync.monero.state')" "done" +assert_eq "parse pool type" "$(jq_get "$ST" '.pool.type')" "Main" +assert_eq "parse worker count" "$(jq_get "$ST" '.proxy_workers')" "2" +assert_eq "missing key -> empty" "$(jq_get "$ST" '.nope.nope')" "" + +echo "== assertion helpers: counters behave ==" +_p="$IT_PASS"; _f="$IT_FAIL" +assert_num_ge "num_ge passes when equal" 5 5 +assert_num_gt "num_gt passes when greater" 6 5 +[ "$IT_PASS" -gt "$_p" ] && it_pass "passing assertions increment IT_PASS" || it_fail "passing assertions increment IT_PASS" "no increment" + +# --- Tally ------------------------------------------------------------------ +echo "" +echo "selftest: $IT_PASS passed, $IT_FAIL failed" +[ "$IT_FAIL" -eq 0 ] || exit 1 From 863ef7b302aa3f935086fa36adc83135806f7726 Mon Sep 17 00:00:00 2001 From: Vijit Singh Date: Thu, 4 Jun 2026 01:45:51 -0500 Subject: [PATCH 02/18] Add 4-tier test strategy: fake daemons, mini-stack, fault injection, unit gaps (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simulate every runtime situation the live synced box can't show, at the cheapest honest tier. Documented in docs/testing-strategy.md with a full scenario catalog. Tier 1 (unit, every PR): backfill the genuine gaps the audit found — the required-Tari sync gate (monero synced but tari syncing → still held), the #35 one-way-latch × #31 failover interaction after release, and a simultaneous double outage. +3 dashboard tests; suite 381 green at 92.86% coverage. Tier 2 (contract, every PR, docker-free): controllable fake monerod (HTTP get_info) and fake Tari (gRPC BaseNode, via the vendored stubs) under tests/integration/fakes/, plus a contract test that points the REAL Monero/Tari clients at them and asserts they parse every state (synced/syncing/down). This is the verifiable proof the fakes speak the daemons' wire format. make test-fakes. Tier 3 (mini-stack, CI with docker): tests/integration/mini-stack/ runs the REAL dashboard + docker-control/-proxy against the fakes with lightweight p2pool/ xmrig-proxy containers, and a scenario runner asserting the control plane end-to-end — sync hold→release (#35) and node-down reject→readmit (#31) — actually stopping/starting real containers. Its own workflow (needs a docker daemon); compose validated with `docker compose config`. make test-mini-stack. Tier 4 (live matrix): add a --fault-injection phase to run.sh that deliberately breaks monerod (stop / SIGSTOP / remove) and asserts pithead status' verdicts (down / unhealthy / missing) and the full failover→recovery cycle, plus the service_state helper and parsing covered by the self-test (53 green). Enabler: UPDATE_INTERVAL is now env-configurable so the mini-stack loops fast in CI. CI wires the contract test into the dashboard job (every PR) and the docker mini-stack as its own paths-filtered workflow. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 6 +- .github/workflows/integration-mini-stack.yml | 27 ++++ CHANGELOG.md | 24 ++- Makefile | 13 +- .../mining_dashboard/config/config.py | 2 +- .../tests/service/test_data_service.py | 70 +++++++++ docs/README.md | 1 + docs/integration-testing.md | 10 +- docs/testing-strategy.md | 130 ++++++++++++++++ tests/integration/README.md | 11 +- tests/integration/fakes/fake_monerod.py | 145 ++++++++++++++++++ tests/integration/fakes/fake_tari.py | 128 ++++++++++++++++ tests/integration/fakes/test_contract.py | 90 +++++++++++ tests/integration/lib.sh | 5 + .../mini-stack/docker-compose.fake.yml | 108 +++++++++++++ .../integration/mini-stack/run-mini-stack.sh | 109 +++++++++++++ tests/integration/run.sh | 93 ++++++++++- tests/integration/selftest.sh | 7 + 18 files changed, 964 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/integration-mini-stack.yml create mode 100644 docs/testing-strategy.md create mode 100644 tests/integration/fakes/fake_monerod.py create mode 100644 tests/integration/fakes/fake_tari.py create mode 100644 tests/integration/fakes/test_contract.py create mode 100644 tests/integration/mini-stack/docker-compose.fake.yml create mode 100755 tests/integration/mini-stack/run-mini-stack.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96e2ff4..2e71b66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,10 @@ jobs: - name: Run pytest with coverage gate working-directory: build/dashboard run: python -m pytest --cov=mining_dashboard --cov-report=term-missing --cov-fail-under=80 + - name: Fake-daemon contract test (real clients vs controllable fakes) + # Points the real Monero/Tari clients at the integration fakes and asserts they parse + # every state (synced/syncing/down). Docker-free, so it runs on every PR (issue #54). + run: PYTHONPATH=build/dashboard python -m pytest tests/integration/fakes -q frontend: name: Frontend logic tests (node --test) @@ -53,7 +57,7 @@ jobs: # the job when one is briefly out of sync — see issue #64. - name: Lint pithead and test scripts # Gate on warnings+errors (real issues); info-level style nits vary by shellcheck version. - run: shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh tests/integration/*.sh + run: shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh tests/integration/*.sh tests/integration/mini-stack/*.sh - name: Run pithead test suite run: bash tests/stack/run.sh - name: Run integration harness self-test diff --git a/.github/workflows/integration-mini-stack.yml b/.github/workflows/integration-mini-stack.yml new file mode 100644 index 0000000..958a41c --- /dev/null +++ b/.github/workflows/integration-mini-stack.yml @@ -0,0 +1,27 @@ +name: Integration mini-stack + +# The fake-daemon docker mini-stack (issue #54, tier 3): brings up the REAL dashboard + +# docker-control proxy against controllable fake monerod/Tari and asserts the control plane +# (sync hold/release, node-down reject/readmit) end-to-end. It needs a Docker daemon, so it +# runs as its own job (not part of the always-on CI matrix), triggered on changes to the +# integration harness or the dashboard, and on demand. +on: + workflow_dispatch: + pull_request: + paths: + - "tests/integration/**" + - "build/dashboard/**" + - ".github/workflows/integration-mini-stack.yml" + +jobs: + mini-stack: + name: Fake-daemon mini-stack (docker) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # ubuntu-latest ships Docker with the Compose v2 plugin — no setup needed. + - name: Run the fake-daemon mini-stack + run: bash tests/integration/mini-stack/run-mini-stack.sh + - name: Dump dashboard logs on failure + if: failure() + run: docker compose -f tests/integration/mini-stack/docker-compose.fake.yml logs --no-color || true diff --git a/CHANGELOG.md b/CHANGELOG.md index e43fad8..e480730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,13 +13,23 @@ per the process in [`docs/releasing.md`](docs/releasing.md). ### Added -- End-to-end integration test suite (`tests/integration/`) that drives a real, already-synced - Pithead server through the config matrix and asserts the stack behaves — containers healthy, - nodes synced, miners mining, the dashboard reading correct live state, `status` exit codes, - and secrets preserved across re-applies. Runs over SSH or `--local`, reuses the synced chain - data dirs (never re-syncs), and is the blocking pre-release gate (#54). Surfaced as `make - test-integration`; a pure-logic `selftest` runs in CI on every PR. See - `docs/integration-testing.md`. +- A four-tier test strategy for simulating every runtime situation (#54), documented in + `docs/testing-strategy.md` with a full scenario catalog: + - **Live config-matrix suite** (`tests/integration/`, tier 4) that drives a real, synced + server through the config matrix and asserts the stack behaves — containers healthy, nodes + synced, miners mining, dashboard reading correct live state, `status` exit codes, secrets + preserved. Runs over SSH or `--local`; the blocking pre-release gate. A `--fault-injection` + phase deliberately breaks monerod (stop / SIGSTOP / remove) to assert `pithead status`' + down/unhealthy/missing verdicts and the failover→recovery cycle. `make test-integration`. + - **Controllable fake monerod/Tari + a contract test** (`tests/integration/fakes/`, tier 2) + that points the real dashboard clients at the fakes and asserts they parse every state — + docker-free, runs on every PR. `make test-fakes`. + - **Fake-daemon docker mini-stack** (`tests/integration/mini-stack/`, tier 3) running the real + dashboard + docker-control proxy against the fakes, asserting sync hold/release and + node-down reject/readmit end-to-end with real containers. `make test-mini-stack`. + - New dashboard unit tests for the required-Tari sync gate, the #35-latch × #31-failover + interaction, and simultaneous double outages. + - `UPDATE_INTERVAL` is now env-configurable (lets the mini-stack loop fast in CI). - Per-worker share stats in the dashboard's Workers table: accepted / rejected (with invalid folded in) counts per rig, a **⚠** flag on a high reject rate, and a **Proxy totals** footer (pool-wide accepted / rejected / invalid + best difficulty) collected from the xmrig-proxy diff --git a/Makefile b/Makefile index ae625ff..d9946c0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Local test entry points (mirror the GitHub Actions CI jobs). -.PHONY: test test-dashboard test-stack test-compose test-integration test-integration-selftest lint +.PHONY: test test-dashboard test-stack test-compose test-integration test-integration-selftest test-fakes test-mini-stack lint -test: lint test-dashboard test-stack test-compose test-integration-selftest ## Run everything that doesn't need a server +test: lint test-dashboard test-stack test-compose test-integration-selftest test-fakes ## Run everything that doesn't need a server/docker test-dashboard: ## Dashboard unit/component tests with coverage gate cd build/dashboard && PYTHONPATH=. python3 -m pytest \ @@ -16,6 +16,12 @@ test-compose: ## Validate docker-compose.yml interpolation test-integration-selftest: ## Integration harness pure-logic self-test (no server needed) bash tests/integration/selftest.sh +test-fakes: ## Fake-daemon contract test — real dashboard clients vs controllable fakes (no docker) + PYTHONPATH=build/dashboard python3 -m pytest tests/integration/fakes -q + +test-mini-stack: ## Fake-daemon docker mini-stack end-to-end (needs docker; CI) + bash tests/integration/mini-stack/run-mini-stack.sh + # End-to-end matrix against a REAL test server (issue #54). Needs a provisioned box; pass # connection + options through ARGS, e.g.: # make test-integration ARGS="--host miner@10.0.0.5 --dir pithead --lifecycle" @@ -24,4 +30,5 @@ test-integration: ## Run the live config-matrix integration suite (requires a te bash tests/integration/run.sh $(ARGS) lint: ## shellcheck the stack scripts - shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh tests/integration/*.sh + shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh \ + tests/integration/*.sh tests/integration/mini-stack/*.sh diff --git a/build/dashboard/mining_dashboard/config/config.py b/build/dashboard/mining_dashboard/config/config.py index d4f8093..9c3e168 100644 --- a/build/dashboard/mining_dashboard/config/config.py +++ b/build/dashboard/mining_dashboard/config/config.py @@ -24,7 +24,7 @@ # XMRig Worker API Configuration XMRIG_API_PORT = 8080 API_TIMEOUT = 1 # Connection timeout (seconds) for worker API calls -UPDATE_INTERVAL = 30 # Frequency (seconds) of the main data aggregation loop +UPDATE_INTERVAL = int(os.environ.get("UPDATE_INTERVAL", 30)) # main data-loop period (s); lowered in integration tests # --- XvB Algorithm Constants --- # Duration of the donation switching cycle (10 minutes) diff --git a/build/dashboard/tests/service/test_data_service.py b/build/dashboard/tests/service/test_data_service.py index 304c681..ee881d7 100644 --- a/build/dashboard/tests/service/test_data_service.py +++ b/build/dashboard/tests/service/test_data_service.py @@ -580,3 +580,73 @@ async def test_iteration_survives_collector_error(self): # The error is caught inside the loop; the sleep after it raises to stop us. with pytest.raises(StopAsyncIteration): await svc.run() + + +class TestControlPlaneComposition: + """Compositions of the sync-gate (#35) and failover (#31) the per-feature tests don't + cover on their own: the required-Tari hold, and the two features coexisting after release.""" + + async def test_run_holds_when_tari_required_and_only_monero_synced(self): + # Monero synced, Tari still syncing, Tari REQUIRED: the gate condition + # `monero_synced AND (tari_synced OR NOT TARI_REQUIRED)` is NOT satisfied, so the + # miner stays held until Tari also finishes — the mirror of the non-blocking case. + svc, sm, proxy = _make_service() + proxy.get_workers.return_value = {"workers": []} + svc._apply_worker_rejection = AsyncMock() + + worker_client = MagicMock() + worker_client.get_stats = AsyncMock(return_value={}) + tari_client = MagicMock() + tari_client.get_sync_status = AsyncMock( + return_value={"is_syncing": True, "reachable": True, "percent": 80, "current": 80, "target": 100}) + tari_client.close = AsyncMock() + + with patch.object(ds_mod, "ClientSession", _FakeClientSession), \ + patch.object(ds_mod, "XMRigWorkerClient", return_value=worker_client), \ + patch.object(ds_mod, "TariClient", return_value=tari_client), \ + patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool", "xmrig-proxy"]), \ + patch.object(ds_mod, "TARI_REQUIRED", True), \ + patch.object(ds_mod, "get_stratum_stats", return_value=({}, [])), \ + patch.object(ds_mod, "get_network_stats", return_value={"height": 100}), \ + patch.object(ds_mod, "get_tari_stats", return_value={"active": True, "status": "OK", "height": 3}), \ + patch.object(ds_mod, "get_p2pool_stats", return_value={"pool": {"last_share_time": 0, "difficulty": 0}}), \ + patch.object(ds_mod, "get_monero_sync_status", AsyncMock(return_value={"is_syncing": False, "reachable": True})), \ + patch.object(ds_mod, "get_disk_usage", return_value={}), \ + patch.object(ds_mod, "get_hugepages_status", return_value=("Enabled", "ok", "1/2")), \ + patch.object(ds_mod, "get_memory_usage", return_value={}), \ + patch.object(ds_mod, "get_load_average", return_value="0"), \ + patch.object(ds_mod, "get_cpu_usage", return_value="0%"), \ + patch("asyncio.sleep", AsyncMock(side_effect=StopAsyncIteration)): + with pytest.raises(StopAsyncIteration): + await svc.run() + + stopped = {c.args[0] for c in svc.docker_control.stop.await_args_list} + assert stopped == {"p2pool", "xmrig-proxy"} + svc.docker_control.start.assert_not_called() + assert svc.miner_released is False + assert svc.latest_data["miner_held"] is True + + async def test_post_release_blip_lets_failover_act_without_rehold(self): + # After release, a node-down event must NOT be re-held by the sync gate (the #35 + # one-way latch), yet #31 failover must still stop the proxy so workers fail over. + # The two coexist: gate no-ops, rejection acts on the proxy only. + svc, _sm, _proxy = _make_service() + svc.miner_released = True + with patch.object(ds_mod, "SYNC_GATE_CONTAINERS", ["p2pool", "xmrig-proxy"]), \ + patch.object(ds_mod, "REJECT_WORKERS_CONTAINER", "xmrig-proxy"), \ + patch.object(ds_mod, "TARI_REQUIRED", True): + await svc._apply_sync_gate(gate_satisfied=False) # latch → no-op + await svc._apply_worker_rejection(monero_down=True, tari_down=False) + stopped = [c.args[0] for c in svc.docker_control.stop.await_args_list] + assert stopped == ["xmrig-proxy"] # p2pool was NOT re-held + svc.docker_control.start.assert_not_called() + assert svc.workers_rejected is True + + async def test_both_nodes_down_rejects_once(self): + # A simultaneous Monero+Tari outage (both required) is a single rejection, not two. + svc, _sm, _proxy = _make_service() + with patch.object(ds_mod, "REJECT_WORKERS_CONTAINER", "xmrig-proxy"), \ + patch.object(ds_mod, "TARI_REQUIRED", True): + await svc._apply_worker_rejection(monero_down=True, tari_down=True) + svc.docker_control.stop.assert_awaited_once_with("xmrig-proxy") + assert svc.workers_rejected is True diff --git a/docs/README.md b/docs/README.md index 6885d40..d68c933 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ deeper on individual topics once you're up and running. | [Connecting Miners](workers.md) | Pointing any existing rig at the stack, plus [RigForge](https://github.com/p2pool-starter-stack/rigforge) for setting up new miners. | | [Architecture](architecture.md) | The nine services, how they fit together, the privacy model, and the algorithmic XvB switching engine. | | [Operations & Maintenance](operations.md) | The full `pithead` command reference, upgrades, backups, and troubleshooting. | +| [Testing Strategy](testing-strategy.md) | The four test tiers (unit → contract → fake-daemon mini-stack → live matrix), the full scenario catalog, and which tier proves each situation. | | [Integration Testing](integration-testing.md) | The end-to-end config-matrix suite that validates the stack against real Monero + Tari nodes — the blocking pre-release gate. | | [Releasing](releasing.md) | How Pithead is versioned and released — one product, one version, the `VERSION` source of truth, and the GHCR stage→promote pipeline. | | [FAQ](faq.md) | Common questions, plus why Pithead vs. doing it yourself or Gupax. | diff --git a/docs/integration-testing.md b/docs/integration-testing.md index 82c1316..a42276c 100644 --- a/docs/integration-testing.md +++ b/docs/integration-testing.md @@ -11,6 +11,13 @@ dashboard pytest mocks its clients. They prove the *code* is correct; they can't real `apply → sync-gate → mine → status` flow works on a real host. That's what this suite is for. +> This live matrix is **tier 4** of a four-tier plan. The runtime *situations* a healthy box +> can't show (cold sync, node-down, unhealthy containers, XvB tiers) are simulated more cheaply +> at lower tiers — unit tests, a client **contract test** against controllable fakes +> ([`tests/integration/fakes/`](../tests/integration/fakes/)), and a **fake-daemon docker +> mini-stack** ([`tests/integration/mini-stack/`](../tests/integration/mini-stack/)). See +> [Testing Strategy](testing-strategy.md) for the full picture and scenario catalog. + The lives under [`tests/integration/`](../tests/integration/): | File | Role | @@ -118,7 +125,8 @@ Useful flags (full list in `run.sh --help`): | `--workers ` | Miners expected online while mining (default `2`). | | `--remote-monero-host ` | External node endpoint for the `remote` scenario. | | `--pruned-data-dir` / `--full-data-dir` | Synced alt DB to enable the opposite prune mode. | -| `--lifecycle` | Also run the lifecycle + node-down failover phase. | +| `--lifecycle` | Also run the lifecycle phase (restart, apply secret-preservation). | +| `--fault-injection` | Also break monerod (stop / SIGSTOP / remove) and assert `status`' down/unhealthy/missing verdicts and the failover→recovery cycle. Destructive-then-restored; local mode only; slow. | | `--keep` | Don't restore the original config (leave the box on the last scenario). | | `--out ` | Where to write the manifest and failure artifacts. | | `--list` | Print the matrix and axis coverage and exit. | diff --git a/docs/testing-strategy.md b/docs/testing-strategy.md new file mode 100644 index 0000000..90ed878 --- /dev/null +++ b/docs/testing-strategy.md @@ -0,0 +1,130 @@ +# Testing Strategy + +How Pithead simulates **every situation the stack can be in** — and which layer proves each +one. This is the map behind the [integration suite](integration-testing.md); read that for how +to run the live matrix, and this for *what we test where, and why*. + +The guiding idea: the stack's runtime behaviour is a **state machine** (syncing → held → +released; healthy → down → rejected → recovered → readmitted; XvB tiers; container health), +and a healthy, already-synced box only ever shows you one corner of it. So we simulate the rest +— at the cheapest layer that can prove each situation honestly. + +## The four tiers + +| Tier | What it is | Simulates | Where it runs | +|---|---|---|---| +| **1 — Unit** | `build/dashboard/tests/` (pytest, mocked clients) and `tests/stack/` (shell, `docker`/`sudo` stubbed) | Decision logic & field mapping: sync-gate, failover, node-health debounce, XvB engine, `/api/state` shapes, `pithead` config/status logic | Every PR (`make test`) | +| **2 — Contract** | `tests/integration/fakes/test_contract.py` | The real Monero/Tari **clients** parsing the real daemons' wire format — points the actual clients at controllable fakes | Every PR (docker-free) | +| **3 — Mini-stack** | `tests/integration/mini-stack/` (real dashboard + docker-control vs fake daemons) | The control plane **end-to-end with real containers**: hold/release and reject/readmit actually stopping/starting `p2pool`/`xmrig-proxy`, driven deterministically | CI with Docker (`make test-mini-stack`) | +| **4 — Live matrix** | `tests/integration/run.sh` against a real, synced box | What only reality proves: real merge-mining, prune/full DB size, Caddy TLS, Tor onions, HugePages, plus fault injection for real container health verdicts | Manual / release gate (`make test-integration`) | + +**Why this shape, and the answer to "should we use stubs?"** Stubs already do the heavy +lifting — the dashboard has ~140 unit tests that exhaustively drive the hard runtime states with +mocked clients. Adding *more* mocks for the same logic would be duplication. What stubs **can't** +prove is wiring: that the real clients parse real daemon output (tier 2), that the dashboard's +stop/start actually moves real containers (tier 3), and that real daemons sync/merge-mine and +real containers go unhealthy (tier 4). So the strategy is **stubs for logic, controllable fake +daemons for the control-plane wiring, and the real box for the irreducibly-real** — each +situation tested once, at the lowest tier that's honest. + +The fakes are the key enabler: because the whole control plane is env-configurable +(`MONERO_RPC_URL`, `TARI_GRPC_ADDRESS`, `DOCKER_CONTROL_URL`, `NODE_DOWN_AFTER_SEC`, +`UPDATE_INTERVAL`, …), we can point the real code at tiny controllable servers and drive the +entire state machine in seconds, in CI, with no chain and no test box. + +## Scenario catalog + +Every situation we care about, what triggers it, and the tier(s) that cover it. ✅ = covered +today; ▶ = exercised by the live matrix / mini-stack when run. + +### A. Configuration permutations +The deploy-time axes — each changes a real runtime path. Full table and assertions in +[Integration Testing › The config matrix](integration-testing.md#the-config-matrix). + +| Situation | Trigger | Tier | +|---|---|---| +| `monero.mode` local vs remote (monerod present/absent, profile gating) | config | 4 ▶ | +| `monero.prune` pruned vs full (DB size, #32 display) | config | 1 ✅ (display) · 4 ▶ (real DB) | +| `monero.rpc_lan_access`, `dashboard.secure`, `xvb.enabled`, `dashboard.tari_required` | config → `.env`/Caddyfile | 4 ▶ | +| `p2pool.pool` main / mini / nano (sidechain, flags) | config | 4 ▶ | + +### B. Sync lifecycle (#35) +| Situation | Trigger | Tier | +|---|---|---| +| Cold start, chains syncing → **hold** `p2pool`+`xmrig-proxy` | both `is_syncing` | 1 ✅ · 3 ▶ | +| Monero synced, Tari **required** but still syncing → keep holding | `monero_synced ∧ ¬tari_synced ∧ TARI_REQUIRED` | 1 ✅ (added) · 3 ▶ | +| Monero synced, Tari **non-blocking** → release, passive Tari badge (#51) | `¬TARI_REQUIRED` | 1 ✅ · 4 ▶ | +| Both synced → **release** (one-way latch) | gate satisfied | 1 ✅ · 3 ▶ | +| Network-height UI override doesn't deadlock the gate | p2pool held → height 0 | 1 ✅ | +| Restart mid-sync / post-release (latch persisted) | snapshot reload | 1 ✅ | + +### C. Node health & failover (#31) +| Situation | Trigger | Tier | +|---|---|---| +| monerod down → **reject workers** (stop `xmrig-proxy`) | unreachable ≥ `NODE_DOWN_AFTER_SEC` | 1 ✅ · 3 ▶ · 4 ▶ | +| Tari down + required → reject; Tari down + non-blocking → **ignore** | `tari_down ∧ TARI_REQUIRED?` | 1 ✅ | +| Recovery hysteresis — readmit only after stable `NODE_RECOVERY_AFTER_SEC` | reachable again | 1 ✅ | +| Transient blip / never-reachable → **no** false reject | debounce / `ever_up` | 1 ✅ | +| Double outage; readmit only when **both** healthy | both down → both up | 1 ✅ (added) | +| #35 latch × #31 failover coexist after release | down post-release | 1 ✅ (added) · 3 ▶ | +| Stop/start fails → retry next cycle (idempotent) | docker error | 1 ✅ | + +### D. Container health verdicts (`pithead status`) +| Situation | Trigger | Tier | +|---|---|---| +| All healthy → exit 0 | steady state | 1 ✅ · 4 ▶ | +| Required node **down** / **missing** → exit 1 | stop / `rm` monerod | 1 ✅ (node-down) · 4 ▶ (`--fault-injection`) | +| Running but **unhealthy** → exit 1 | healthcheck fails (SIGSTOP) | 4 ▶ (`--fault-injection`) | +| Miner stopped under sync-hold / failover → exit **0** (intentional) | held / rejected | 1 ✅ · 4 ▶ | +| Remote mode ignores monerod | profile off | 1 ✅ · 4 ▶ | + +### E. XvB switching engine +| Situation | Trigger | Tier | +|---|---|---| +| Disabled / zero shares / `fail_count ≥ 3` / no sustainable tier → P2POOL | guards | 1 ✅ | +| Closed-loop ramp/back-off, cold-start seed, VIP-reserve anti-overshoot (#70) | controller | 1 ✅ | +| P2POOL / XVB / SPLIT modes, tiers, smart-sleep early exit | decision | 1 ✅ | +| Real XvB endpoint reachable / failing | network | 4 (real endpoint) | + +### F. Dashboard `/api/state` field states +| Situation | Trigger | Tier | +|---|---|---| +| sync state loading/syncing/done; pruned/full/unknown; db_size | metrics | 1 ✅ | +| badges (node-down, workers-rejected, miner-held, passive-Tari, pruned/full, low-HR) | metrics | 1 ✅ | +| system levels (cpu/mem/disk/hugepages), worker pool/online, chart outage breaks | metrics | 1 ✅ | +| Dashboard reads correct live state on a real stack | real daemons | 4 ▶ | + +### G. CLI lifecycle (`pithead`) +| Situation | Trigger | Tier | +|---|---|---| +| Config validation, secret preservation, `apply` no-op/destructive guards | sourced fns | 1 ✅ | +| `setup`→`up`→`status`→`apply`→`restart`→`down`; idempotency; secret preservation | real box | 4 ▶ (`--lifecycle`) | +| `upgrade` (image pull/rebuild) | real box | release staging smoke (docs) | +| `backup`/`restore`, `reset-dashboard`, `doctor` | real box | 1 ✅ (partial) · 4 (future) | + +### H. Host / infrastructure (real-only) +| Situation | Trigger | Tier | +|---|---|---| +| Real merge-mining share lands; real hashrate on dashboard | live mining | 4 ▶ | +| Caddy TLS scheme; Tor onion provisioning; HugePages/AVX2; real disk pressure; prune DB size | real host | 4 ▶ | + +## Running each tier + +```bash +make test # tiers 1 + 2 (+ harness self-test) — every-PR, no docker/server +make test-fakes # tier 2 contract test on its own +make test-mini-stack # tier 3 — needs docker +make test-integration ARGS="--host user@box --dir pithead --lifecycle --fault-injection" # tier 4 +``` + +## Adding a scenario + +- **Logic** (a new decision/branch) → a unit test (tier 1). Cheapest, fastest. +- **A new daemon state** the clients must parse → extend the fakes + the contract test (tier 2), + and it becomes drivable in the mini-stack (tier 3). +- **A config axis** → one row in `tests/integration/scenarios.sh` (tier 4). The self-test + enforces every axis value is covered. +- **A failure mode needing real containers** → a fault in `run.sh`'s fault-injection phase + (tier 4) and/or a mini-stack scenario (tier 3). + +Keep each situation at the lowest honest tier; don't re-prove logic with a heavier harness. diff --git a/tests/integration/README.md b/tests/integration/README.md index 3c32c58..760aeba 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -5,12 +5,21 @@ config matrix and asserts the stack behaves (issue [#54](https://github.com/p2pool-starter-stack/pithead/issues/54)). ``` -run.sh entry point — connects (SSH or --local) and runs the matrix +run.sh entry point — connects (SSH or --local) and runs the matrix (+ --lifecycle, + --fault-injection) scenarios.sh the declarative config matrix (data, not code) lib.sh shared helpers: target I/O, assertions, readiness waiters, redaction selftest.sh pure-logic self-test (no server) — runs in CI on every PR +fakes/ controllable fake monerod/Tari + a contract test pointing the REAL clients at + them (tier 2; runs in CI, no docker) +mini-stack/ docker overlay running the real dashboard + docker-control vs the fakes, with a + scenario runner for hold/release + reject/readmit (tier 3; needs docker) ``` +The live matrix here is **tier 4** of the broader plan — see +[`docs/testing-strategy.md`](../../docs/testing-strategy.md) for all four tiers and the full +scenario catalog. + Quick start: ```bash diff --git a/tests/integration/fakes/fake_monerod.py b/tests/integration/fakes/fake_monerod.py new file mode 100644 index 0000000..e516180 --- /dev/null +++ b/tests/integration/fakes/fake_monerod.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +""" +Controllable fake monerod for the integration mini-stack (issue #54, tier 3). + +Speaks just enough of monerod's `get_info` RPC for the dashboard's MoneroClient to read it, +plus a `/control` endpoint to drive its state from a test. Lets us reproduce the whole Monero +side of the runtime state machine — syncing %, synced, unreachable, pruned/full DB size — +deterministically, with no real chain. + +Run standalone (in the docker mini-stack): + python3 fake_monerod.py --port 18081 + +Drive it: + curl -s localhost:18081/control -d '{"mode":"syncing","height":1500,"target_height":3000}' + curl -s localhost:18081/control -d '{"mode":"down"}' + curl -s localhost:18081/get_info + +Use in-process (the contract test): + with FakeMonerod() as m: + m.set(mode="syncing", height=1500, target_height=3000) + ...point a real MoneroClient at m.url... +""" +import argparse +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# mode ∈ {"synced", "syncing", "down"}. height/target_height/database_size are the figures +# get_info returns; the client derives sync %/DB size from them (MoneroClient.get_sync_status). +DEFAULT_STATE = { + "mode": "synced", + "height": 3_000_000, + "target_height": 3_000_000, + "database_size": 85 * 10**9, +} + + +class _Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): # keep the test output clean + pass + + def _send(self, code, payload): + body = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path.rstrip("/") != "/get_info": + self._send(404, {"status": "NOT_FOUND"}) + return + st = self.server.state + # "down" → unreachable: monerod's RPC not answering. A non-200 makes MoneroClient + # treat the node as unreachable (get_info returns None), which is what we want. + if st["mode"] == "down": + self._send(503, {"status": "BUSY"}) + return + if st["mode"] == "syncing": + payload = { + "status": "OK", + "synchronized": False, + "height": st["height"], + "target_height": st["target_height"], + "database_size": st["database_size"], + } + else: # synced — monerod reports synchronized and target_height 0 once caught up + payload = { + "status": "OK", + "synchronized": True, + "height": st["height"], + "target_height": 0, + "database_size": st["database_size"], + } + self._send(200, payload) + + def do_POST(self): + if self.path.rstrip("/") != "/control": + self._send(404, {"status": "NOT_FOUND"}) + return + length = int(self.headers.get("Content-Length", 0)) + try: + data = json.loads(self.rfile.read(length) or b"{}") + except ValueError: + self._send(400, {"error": "bad json"}) + return + self.server.state.update(data) + self._send(200, self.server.state) + + +class _Server(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, addr, state): + super().__init__(addr, _Handler) + self.state = state + + +class FakeMonerod: + """Context manager that runs the fake on an ephemeral port in a background thread.""" + + def __init__(self, port=0, host="127.0.0.1", **state): + self.state = {**DEFAULT_STATE, **state} + self._srv = _Server((host, port), self.state) + self.host, self.port = self._srv.server_address + + @property + def url(self): + return f"http://{self.host}:{self.port}" + + def set(self, **kwargs): + self.state.update(kwargs) + + def __enter__(self): + self._thread = threading.Thread(target=self._srv.serve_forever, daemon=True) + self._thread.start() + return self + + def __exit__(self, *_exc): + self._srv.shutdown() + self._srv.server_close() + + +def main(): + ap = argparse.ArgumentParser(description="Controllable fake monerod") + ap.add_argument("--port", type=int, default=18081) + ap.add_argument("--host", default="0.0.0.0") # noqa: S104 — test-only container + ap.add_argument("--mode", default="synced", choices=["synced", "syncing", "down"], + help="initial state (the mini-stack boots 'syncing' to exercise the hold)") + args = ap.parse_args() + state = dict(DEFAULT_STATE, mode=args.mode) + # "syncing" needs height < target_height to read as syncing (else it looks caught up). + if args.mode == "syncing" and state["height"] >= state["target_height"]: + state["height"], state["target_height"] = 1_500_000, 3_000_000 + srv = _Server((args.host, args.port), state) + print(f"fake-monerod listening on {args.host}:{args.port} (mode={args.mode})", flush=True) + try: + srv.serve_forever() + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/tests/integration/fakes/fake_tari.py b/tests/integration/fakes/fake_tari.py new file mode 100644 index 0000000..e6d3ce6 --- /dev/null +++ b/tests/integration/fakes/fake_tari.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Controllable fake Tari base node for the integration mini-stack (issue #54, tier 3). + +Implements just the two BaseNode gRPC methods the dashboard's TariClient calls — GetTipInfo +and GetSyncProgress — against the project's own vendored protobuf stubs, so the real client +talks to it unchanged (the client uses an insecure channel, so there's no auth to fake). A +small HTTP `/control` side-channel drives its state. + +Run standalone (in the docker mini-stack): + python3 fake_tari.py --grpc-port 18142 --control-port 18152 + +Drive it: + curl -s localhost:18152/control -d '{"mode":"syncing","height":500,"target_height":2000}' + curl -s localhost:18152/control -d '{"mode":"down"}' + +Use in-process (the contract test) via start_server(). +""" +import argparse +import asyncio +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import grpc + +from mining_dashboard.client.tari.generated import base_node_pb2 as bn +from mining_dashboard.client.tari.generated import base_node_pb2_grpc as bn_grpc + +# mode ∈ {"synced", "syncing", "down"}. +DEFAULT_STATE = {"mode": "synced", "height": 2_000_000, "target_height": 2_000_000} + + +class FakeBaseNode(bn_grpc.BaseNodeServicer): + def __init__(self, state): + self.state = state + + async def GetTipInfo(self, request, context): + st = self.state + if st["mode"] == "down": + await context.abort(grpc.StatusCode.UNAVAILABLE, "fake node down") + resp = bn.TipInfoResponse() + resp.metadata.best_block_height = st["height"] + # initial_sync_achieved is the authoritative "fully synced" flag the client trusts. + resp.initial_sync_achieved = st["mode"] == "synced" + return resp + + async def GetSyncProgress(self, request, context): + st = self.state + if st["mode"] == "down": + await context.abort(grpc.StatusCode.UNAVAILABLE, "fake node down") + resp = bn.SyncProgressResponse() + resp.local_height = st["height"] + resp.tip_height = st["target_height"] + return resp + + +async def start_server(port, state): + """Start a gRPC server on `port` (0 = ephemeral). Returns (server, bound_port).""" + server = grpc.aio.server() + bn_grpc.add_BaseNodeServicer_to_server(FakeBaseNode(state), server) + bound = server.add_insecure_port(f"127.0.0.1:{port}") + await server.start() + return server, bound + + +# --- standalone HTTP control side-channel (docker mini-stack only) ---------- +class _ControlHandler(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + if self.path.rstrip("/") != "/control": + self.send_response(404) + self.end_headers() + return + length = int(self.headers.get("Content-Length", 0)) + try: + data = json.loads(self.rfile.read(length) or b"{}") + except ValueError: + self.send_response(400) + self.end_headers() + return + self.server.state.update(data) + body = json.dumps(self.server.state).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + + +class _ControlServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, addr, state): + super().__init__(addr, _ControlHandler) + self.state = state + + +async def _main_async(args, state): + server, _ = await start_server(args.grpc_port, state) + ctrl = _ControlServer(("0.0.0.0", args.control_port), state) # noqa: S104 — test-only + threading.Thread(target=ctrl.serve_forever, daemon=True).start() + print( + f"fake-tari gRPC on :{args.grpc_port}, control on :{args.control_port}", + flush=True, + ) + await server.wait_for_termination() + + +def main(): + ap = argparse.ArgumentParser(description="Controllable fake Tari base node") + ap.add_argument("--grpc-port", type=int, default=18142) + ap.add_argument("--control-port", type=int, default=18152) + ap.add_argument("--mode", default="synced", choices=["synced", "syncing", "down"], + help="initial state (the mini-stack boots 'syncing' to exercise the hold)") + args = ap.parse_args() + state = dict(DEFAULT_STATE, mode=args.mode) + if args.mode == "syncing" and state["height"] >= state["target_height"]: + state["height"], state["target_height"] = 1_000_000, 2_000_000 + try: + asyncio.run(_main_async(args, state)) + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/tests/integration/fakes/test_contract.py b/tests/integration/fakes/test_contract.py new file mode 100644 index 0000000..2d83d85 --- /dev/null +++ b/tests/integration/fakes/test_contract.py @@ -0,0 +1,90 @@ +""" +Contract test: point the REAL dashboard clients at the controllable fakes and assert they +parse every state we need to drive in the mini-stack (issue #54, tier 3 / tier 2 seam). + +This is the proof that the fakes speak the daemons' wire format closely enough for the real +MoneroClient / TariClient — and it runs anywhere (no docker, no real chain). If a future +monerod/Tari change breaks the parser, this goes red here instead of only on the live box. + +Run: PYTHONPATH=build/dashboard python3 -m pytest tests/integration/fakes -q +""" +import asyncio +import pathlib +import sys + +import requests +from unittest.mock import MagicMock + +_HERE = pathlib.Path(__file__).resolve().parent +_REPO = _HERE.parents[2] +# Make the dashboard package and the fakes importable regardless of how pytest is invoked. +sys.path.insert(0, str(_REPO / "build" / "dashboard")) +sys.path.insert(0, str(_HERE)) + +from fake_monerod import FakeMonerod # noqa: E402 +from fake_tari import start_server # noqa: E402 +from mining_dashboard.client.monero.monero_client import MoneroClient # noqa: E402 +from mining_dashboard.client.tari.tari_client import TariClient # noqa: E402 + + +# --- Monero (HTTP get_info) ------------------------------------------------- +def test_monero_synced_reads_no_sync_and_db_size(): + with FakeMonerod(database_size=85 * 10**9) as m: + client = MoneroClient(url=m.url, username="") + st = client.get_sync_status() + assert st == {"is_syncing": False, "db_size": 85 * 10**9} + + +def test_monero_syncing_reports_percent(): + with FakeMonerod() as m: + m.set(mode="syncing", height=1500, target_height=3000, database_size=40 * 10**9) + client = MoneroClient(url=m.url, username="") + st = client.get_sync_status() + assert st["is_syncing"] is True + assert st["current"] == 1500 and st["target"] == 3000 and st["percent"] == 50 + assert st["db_size"] == 40 * 10**9 + + +def test_monero_down_is_unreachable(): + with FakeMonerod() as m: + m.set(mode="down") + client = MoneroClient(url=m.url, username="") + assert client.get_sync_status() is None + + +def test_monero_http_control_mutates_state(): + # Validates the /control path the docker mini-stack drives over the network. + with FakeMonerod() as m: + requests.post(m.url + "/control", json={"mode": "syncing", "height": 10, "target_height": 100}, timeout=5) + info = requests.get(m.url + "/get_info", timeout=5).json() + assert info["synchronized"] is False and info["height"] == 10 and info["target_height"] == 100 + + +# --- Tari (gRPC BaseNode) --------------------------------------------------- +# Driven via asyncio.run so they don't depend on pytest-asyncio being active (the dashboard's +# asyncio_mode=auto only applies when pytest's rootdir is build/dashboard). +async def _tari_get_status(state): + server, bound = await start_server(0, state) + client = TariClient(MagicMock()) + client.grpc_address = f"127.0.0.1:{bound}" + try: + return await client.get_sync_status() + finally: + await client.close() + await server.stop(None) + + +def test_tari_synced_reads_done(): + st = asyncio.run(_tari_get_status({"mode": "synced", "height": 2000, "target_height": 2000})) + assert st["is_syncing"] is False and st["reachable"] is True and st["percent"] == 100 + + +def test_tari_syncing_reports_percent(): + st = asyncio.run(_tari_get_status({"mode": "syncing", "height": 500, "target_height": 2000})) + assert st["is_syncing"] is True and st["percent"] == 25 and st["reachable"] is True + + +def test_tari_down_is_unreachable_with_no_cache(): + # No prior good reading to cache, so a down node is reported unreachable immediately. + st = asyncio.run(_tari_get_status({"mode": "down", "height": 0, "target_height": 0})) + assert st["reachable"] is False diff --git a/tests/integration/lib.sh b/tests/integration/lib.sh index 9fd4796..2d6d2d3 100644 --- a/tests/integration/lib.sh +++ b/tests/integration/lib.sh @@ -148,6 +148,11 @@ pithead() { rx "$IT_PITHEAD $*"; } # network). Empty output on failure so callers can detect unreachable. api_state() { rx "curl -fsS --max-time 10 http://127.0.0.1:8000/api/state" 2>/dev/null; } +# Split a " " string (from service_state) into its two fields. Pure helpers so +# the self-test can verify the fault-injection predicates classify correctly. +svc_state_of() { printf '%s' "${1%% *}"; } +svc_health_of() { printf '%s' "${1##* }"; } + # Pull a jq path out of a JSON blob, printing nothing for an absent/null value. The `?` # swallows "cannot index null" on a missing parent, and `values` drops nulls — but NOT # boolean false (so `.monero.prune == false` reads as "false", not ""; `// empty` would diff --git a/tests/integration/mini-stack/docker-compose.fake.yml b/tests/integration/mini-stack/docker-compose.fake.yml new file mode 100644 index 0000000..64ef2e8 --- /dev/null +++ b/tests/integration/mini-stack/docker-compose.fake.yml @@ -0,0 +1,108 @@ +# Integration mini-stack (issue #54, tier 3). +# +# Runs the REAL dashboard + the REAL docker-control/-proxy socket proxies against CONTROLLABLE +# fake monerod/Tari, with lightweight p2pool/xmrig-proxy containers the dashboard can actually +# stop/start. This reproduces the runtime control plane end-to-end — sync-hold/release (#35) and +# node-down → reject → readmit (#31) — deterministically, in CI, with no real chain or test box. +# +# Driven by run-mini-stack.sh. The dashboard and the fakes share one image (the dashboard's, +# which already has mining_dashboard + grpc installed, so fake_tari can use the vendored stubs). +name: pithead-itest + +x-fake-image: &fake_image pithead-dashboard:itest + +networks: + itestnet: + driver: bridge + +volumes: + dashboard_data: + dashboard_stats: + +services: + # The real dashboard, pointed at the fakes and the socket proxies. Fast loop + short debounce + # so scenarios converge in seconds. Binds 127.0.0.1:8000 inside the container; the runner + # reads /api/state via `compose exec`, so no published port is needed. + dashboard: + build: ../../../build/dashboard + image: *fake_image + container_name: itest-dashboard + networks: [itestnet] + volumes: + - dashboard_data:/data + - dashboard_stats:/app/stats:ro + environment: + HOST_IP: "127.0.0.1" + TZ: "Etc/UTC" + MONERO_RPC_URL: "http://fake-monerod:18081" + MONERO_NODE_USERNAME: "" + MONERO_NODE_PASSWORD: "" + MONERO_NODE_HOST: "fake-monerod" + MONERO_PRUNE: "true" + TARI_GRPC_ADDRESS: "fake-tari:18142" + DOCKER_PROXY_URL: "tcp://docker-proxy:2375" + DOCKER_CONTROL_URL: "tcp://docker-control:2375" + SYNC_GATE_CONTAINERS: "p2pool,xmrig-proxy" + REJECT_WORKERS_CONTAINER: "xmrig-proxy" + TARI_REQUIRED: "true" + XVB_ENABLED: "false" + XVB_POOL_URL: "" + XVB_DONOR_ID: "" + P2POOL_URL: "p2pool:3333" + MONERO_WALLET_ADDRESS: "49iTestWalletPlaceholderXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + PROXY_HOST: "xmrig-proxy" + PROXY_API_PORT: "3344" + PROXY_AUTH_TOKEN: "itest" + UPDATE_INTERVAL: "2" + NODE_DOWN_AFTER_SEC: "4" + NODE_RECOVERY_AFTER_SEC: "3" + depends_on: [fake-monerod, fake-tari, docker-control, docker-proxy, p2pool, xmrig-proxy] + + fake-monerod: + image: *fake_image + container_name: itest-fake-monerod + networks: [itestnet] + entrypoint: [] + # Boot mid-sync so the dashboard holds the miner; the runner flips it to synced/down. + command: ["python3", "/fakes/fake_monerod.py", "--port", "18081", "--mode", "syncing"] + ports: ["18081:18081"] # control + get_info, reachable from the runner + volumes: ["../fakes:/fakes:ro"] + + fake-tari: + image: *fake_image + container_name: itest-fake-tari + networks: [itestnet] + entrypoint: [] + command: ["python3", "/fakes/fake_tari.py", "--grpc-port", "18142", "--control-port", "18152", "--mode", "syncing"] + ports: ["18152:18152"] # HTTP control side-channel, reachable from the runner + volumes: ["../fakes:/fakes:ro"] + + # Stand-ins for the miner containers: real, named containers the dashboard genuinely + # stops/starts via docker-control. They just idle. + p2pool: + image: busybox:1.36 + container_name: p2pool + networks: [itestnet] + command: ["sh", "-c", "while true; do sleep 30; done"] + + xmrig-proxy: + image: busybox:1.36 + container_name: xmrig-proxy + networks: [itestnet] + command: ["sh", "-c", "while true; do sleep 30; done"] + + # Read-only socket proxy (stats/logs) — mirrors the production docker-proxy. + docker-proxy: + image: tecnativa/docker-socket-proxy:v0.4.2 + container_name: itest-docker-proxy + networks: [itestnet] + environment: ["CONTAINERS=1", "LOGS=1"] + volumes: ["/var/run/docker.sock:/var/run/docker.sock:ro"] + + # Write proxy scoped to start/stop only — mirrors the production docker-control. + docker-control: + image: tecnativa/docker-socket-proxy:v0.4.2 + container_name: itest-docker-control + networks: [itestnet] + environment: ["POST=1", "ALLOW_START=1", "ALLOW_STOP=1"] + volumes: ["/var/run/docker.sock:/var/run/docker.sock:ro"] diff --git a/tests/integration/mini-stack/run-mini-stack.sh b/tests/integration/mini-stack/run-mini-stack.sh new file mode 100755 index 0000000..4753782 --- /dev/null +++ b/tests/integration/mini-stack/run-mini-stack.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# +# Drive the integration mini-stack (issue #54, tier 3) through the control-plane state machine +# and assert the REAL dashboard holds/releases and rejects/readmits the REAL miner containers, +# driven by the controllable fakes. Needs docker (compose v2). Runs in CI; also `make +# test-mini-stack`. +# +# Scenarios: +# 1. boot syncing → dashboard HOLDS p2pool + xmrig-proxy (#35) +# 2. both chains synced → dashboard RELEASES them +# 3. monerod down → dashboard REJECTS workers (stops xmrig-proxy) (#31) +# 4. monerod back → dashboard READMITS workers +# +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="$HERE/docker-compose.fake.yml" +PASS=0 +FAIL=0 + +c_ok() { PASS=$((PASS + 1)); printf ' \033[1;32m✓\033[0m %s\n' "$1"; } +c_bad() { FAIL=$((FAIL + 1)); printf ' \033[1;31m✗\033[0m %s\n %s\n' "$1" "${2:-}"; } +log() { printf '\033[1;36m[mini-stack]\033[0m %s\n' "$1"; } + +if ! docker compose version >/dev/null 2>&1; then + echo "SKIP: docker compose not available" + exit 0 +fi + +compose() { docker compose -f "$COMPOSE_FILE" "$@"; } +cstate() { docker inspect -f '{{.State.Status}}' "$1" 2>/dev/null || echo "missing"; } +ctl() { curl -fsS --max-time 5 "$1" -d "$2" >/dev/null; } # POST JSON to a fake /control + +# Poll a container until it reaches an expected state, or time out. +wait_state() { # wait_state [timeout_s] + local c="$1" want="$2" timeout="${3:-60}" end + end=$(( $(date +%s) + timeout )) + while :; do + [ "$(cstate "$c")" = "$want" ] && return 0 + [ "$(date +%s)" -ge "$end" ] && return 1 + sleep 1 + done +} + +assert_state() { # assert_state