Add end-to-end integration test harness against a real server (#54)

.github/workflows/ci.yml

@@ -19,6 +19,10 @@ jobs:
       - name: Run pytest with coverage gate
         working-directory: build/dashboard
         run: python -m pytest --cov=mining_dashboard --cov-report=term-missing --cov-fail-under=80
+      - name: Fake-daemon contract test (real clients vs controllable fakes)
+        # Points the real Monero/Tari clients at the integration fakes and asserts they parse
+        # every state (synced/syncing/down). Docker-free, so it runs on every PR (issue #54).
+        run: PYTHONPATH=build/dashboard python -m pytest tests/integration/fakes -q
 
   frontend:
     name: Frontend logic tests (node --test)
@@ -53,14 +57,23 @@ jobs:
       # the job when one is briefly out of sync — see issue #64.
       - name: Lint pithead and test scripts
         # Gate on warnings+errors (real issues); info-level style nits vary by shellcheck version.
-        run: shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh
+        run: shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh tests/inventory.sh tests/integration/*.sh tests/integration/mini-stack/*.sh
       - name: Run pithead test suite
         run: bash tests/stack/run.sh
+      - name: Run integration harness self-test
+        # Pure-logic checks for the tests/integration/ harness (config rendering, matrix
+        # coverage, redaction). The LIVE matrix (tests/integration/run.sh) needs a real test
+        # server and runs as a gated/manual release gate (#54), not on every PR.
+        run: bash tests/integration/selftest.sh
+      - name: Check the test inventory is up to date
+        # docs/test-inventory.md is generated from the suites; fail if a test was added/removed
+        # without regenerating it (run `make test-inventory`).
+        run: make test-inventory-check
 
   compose:
-    name: Compose config validation
+    name: Compose config + security hardening
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Validate docker-compose.yml interpolation
+      - name: Validate docker-compose.yml interpolation + hardening invariants (#90)
         run: bash tests/stack/test_compose.sh

.github/workflows/integration-mini-stack.yml

@@ -0,0 +1,27 @@
+name: Integration mini-stack
+
+# The fake-daemon docker mini-stack (issue #54, tier 3): brings up the REAL dashboard +
+# docker-control proxy against controllable fake monerod/Tari and asserts the control plane
+# (sync hold/release, node-down reject/readmit) end-to-end. It needs a Docker daemon, so it
+# runs as its own job (not part of the always-on CI matrix), triggered on changes to the
+# integration harness or the dashboard, and on demand.
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "tests/integration/**"
+      - "build/dashboard/**"
+      - ".github/workflows/integration-mini-stack.yml"
+
+jobs:
+  mini-stack:
+    name: Fake-daemon mini-stack (docker)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # ubuntu-latest ships Docker with the Compose v2 plugin — no setup needed.
+      - name: Run the fake-daemon mini-stack
+        run: bash tests/integration/mini-stack/run-mini-stack.sh
+      - name: Dump dashboard logs on failure
+        if: failure()
+        run: docker compose -f tests/integration/mini-stack/docker-compose.fake.yml logs --no-color || true

.github/workflows/release-gate.yml

@@ -0,0 +1,72 @@
+name: Release gate (self-hosted)
+
+# Tier-4 end-to-end validation against the REAL synced Monero + Tari nodes — the pre-release
+# gate (#54). It runs on the dedicated, self-hosted release server (which holds real wallet /
+# onion keys), so it MUST only ever run code we trust.
+#
+# SECURITY: there is deliberately NO `pull_request` trigger. A fork PR's code running on this
+# runner could steal the box's keys or persist a backdoor (GitHub recommends against self-hosted
+# runners on public repos for exactly this reason). The gate runs only on:
+#   - workflow_dispatch — a maintainer manually runs it on a ref they've reviewed, OR
+#   - push to main      — post-merge, on trusted code.
+# To end-to-end a specific fork PR, review it first, then dispatch this workflow on that ref.
+# See docs/release-server.md.
+on:
+  workflow_dispatch:
+    inputs:
+      stack_dir:
+        description: "Path to the deployed Pithead stack on the runner (absolute; default $HOME/code/pithead)"
+        required: false
+        default: ""
+      mode:
+        description: "check = non-destructive; matrix = full destructive config matrix (with a safety backup + auto-rollback)"
+        required: false
+        default: "check"
+        type: choice
+        options: [check, matrix]
+  push:
+    branches: [main]
+
+# Never run two gates against the one shared box at the same time.
+concurrency:
+  group: release-gate
+  cancel-in-progress: false
+
+jobs:
+  release-gate:
+    name: Tier-4 live matrix (real nodes)
+    # Register the server with these labels: `pithead-release` scopes the gate to the dedicated
+    # box; prefer an ephemeral / just-in-time runner in its own runner group.
+    runs-on: [self-hosted, pithead-release]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate against the real synced nodes
+        # Inputs go through env (not interpolated into the script) to avoid shell injection.
+        env:
+          STACK_DIR_INPUT: ${{ github.event.inputs.stack_dir }}
+          MODE_INPUT: ${{ github.event.inputs.mode }}
+        run: |
+          set -euo pipefail
+          DIR="${STACK_DIR_INPUT:-$HOME/code/pithead}"
+          MODE="${MODE_INPUT:-check}"
+          echo "Release gate: stack dir=$DIR, mode=$MODE"
+
+          # Always assess fitness + the non-destructive live state first.
+          bash tests/integration/run.sh --local --dir "$DIR" --readiness
+          bash tests/integration/run.sh --local --dir "$DIR" --check
+
+          # The full destructive config matrix is opt-in; --safety-backup rolls the box back if
+          # anything fails, so a red run leaves the server as it found it.
+          if [ "$MODE" = "matrix" ]; then
+            bash tests/integration/run.sh --local --dir "$DIR" --workers 2 --safety-backup --lifecycle
+          fi
+
+      - name: Upload artifacts (redacted)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-gate-results
+          path: tests/integration/results/
+          if-no-files-found: ignore
+          retention-days: 14

.gitignore

@@ -21,5 +21,8 @@ htmlcov/
 *.egg-info/
 .eggs/
 
+# Integration test artifacts (manifest, per-scenario logs, captured state)
+/tests/integration/results/
+
 # OS
 .DS_Store

CHANGELOG.md

@@ -13,6 +13,53 @@ per the process in [`docs/releasing.md`](docs/releasing.md).
 
 ### Added
 
+- A four-tier test strategy for simulating every runtime situation (#54), documented in
+  `docs/testing-strategy.md` with a full scenario catalog:
+  - **Live config-matrix suite** (`tests/integration/`, tier 4) that drives a real, synced
+    server through the config matrix and asserts the stack behaves — containers healthy, nodes
+    synced, miners mining, dashboard reading correct live state, `status` exit codes, secrets
+    preserved. Runs over SSH or `--local`; the blocking pre-release gate. A `--fault-injection`
+    phase deliberately breaks monerod (stop / SIGSTOP / remove) to assert `pithead status`'
+    down/unhealthy/missing verdicts and the failover→recovery cycle. `make test-integration`.
+  - **Controllable fake monerod/Tari + a contract test** (`tests/integration/fakes/`, tier 2)
+    that points the real dashboard clients at the fakes and asserts they parse every state —
+    docker-free, runs on every PR. `make test-fakes`.
+  - **Fake-daemon docker mini-stack** (`tests/integration/mini-stack/`, tier 3) running the real
+    dashboard + docker-control proxy against the fakes, asserting sync hold/release and Tari
+    reject/readmit end-to-end with real containers (`make test-mini-stack`). Validated green
+    (11/11) on a real Docker host, and isolated (namespaced container names + non-colliding
+    ports) so it can run safely beside a live deployment.
+  - New dashboard unit tests for the required-Tari sync gate, the #35-latch × #31-failover
+    interaction, and simultaneous double outages.
+  - A generated **test inventory** (`docs/test-inventory.md`, `make test-inventory`) listing
+    every test/scenario across all suites, kept honest by a CI drift check.
+  - A non-destructive **`--check`** mode for the live harness (assert the box's current state —
+    no config change/apply/restore); the safe first run / ongoing health check. Validated with
+    a 22/22 green run against a real synced, mining box, which calibrated the harness to trust
+    monerod's own sync flag (a synced local node's dashboard sync panel reads "loading") and
+    `proxy_workers` for mining liveness (`stratum.conns` can read 0 while mining).
+  - A developer testing guide (`docs/testing-guide.md`): per-change recipes, conventions, and
+    the calibration gotchas learned on real hardware.
+  - Regression guards for past bugs/security fixes: extended the #90 hardening section of
+    `tests/stack/test_compose.sh` with per-service least-privilege checks for the Docker socket
+    proxies (the read proxy can't POST; the control proxy is start/stop-only; both mount the
+    socket read-only) and the Tari `[m]inotari` self-match guard — alongside the existing
+    no-new-privileges / cap_drop / credential-free-healthcheck assertions. Plus a
+    `dashboard.host` "auto"-revert test and the schema-migration test that caught the DB upgrade
+    bug above.
+  - Release/validation-server tooling: a `--readiness` mode for the live harness (non-destructive
+    assessment that a box is fit to be a release server — synced chains reusable, snapshot-capable
+    filesystem, disk headroom, secrets owner-only, dashboard localhost-only), a
+    `docs/release-server.md` guide (why end-to-end validation needs a dedicated server vs. what
+    GitHub Actions runs free on every PR, the hardening checklist, and the **safe** self-hosted-
+    runner setup), and a `release-gate.yml` workflow that runs the tier-4 matrix on a self-hosted
+    runner only on trusted code (manual dispatch / push to main — never on a fork PR).
+  - A `--safety-backup` rollback net for the live harness: takes a real `pithead backup` before
+    the destructive scenarios and automatically rolls the box back (down → restore → up) if
+    anything fails, removing the archive on success — so the destructive matrix can run on a
+    precious box. The `--lifecycle` phase also does a `backup` → `restore` round-trip (assert the
+    pool reverts and secrets survive), exercising both verbs end-to-end.
+  - `UPDATE_INTERVAL` is now env-configurable (lets the mini-stack loop fast in CI).
 - Dashboard header shows the host's **IP address** next to the hostname when the configured
   `dashboard.host` is a name, as `hostname @ ip` (e.g. `pithead.local @ 192.168.1.42`), so you can still reach the
   dashboard when the hostname doesn't resolve from your phone or another machine on the LAN. The
@@ -62,6 +109,14 @@ per the process in [`docs/releasing.md`](docs/releasing.md).
 
 ### Changed
 
+- The Compose **project name is now pinned to `pithead`** (`name:` in `docker-compose.yml`), so
+  the stack's images, network and volumes are prefixed `pithead*` regardless of the checkout
+  directory — instead of inheriting the directory's name (which left older checkouts named after
+  the repo's previous name). `pithead up`/`apply`/`upgrade` detect a stack still running under
+  the old, directory-derived project name and migrate it automatically (only that project's
+  containers are removed so the renamed project can take over — bind-mounted chain data and the
+  Tor onion keys are untouched). One-time after the rename, Caddy re-issues its local TLS cert
+  under the new project, so re-trust the dashboard cert if you'd installed the old one.
 - Hardened the leaf containers (caddy, xmrig-proxy, dashboard, docker-proxy, docker-control)
   with `no-new-privileges`. All except the dashboard also `cap_drop: [ALL]` (caddy keeps
   `NET_BIND_SERVICE` for `:80`/`:443`); the dashboard keeps its default capabilities because it
@@ -88,6 +143,16 @@ per the process in [`docs/releasing.md`](docs/releasing.md).
   before resetting (without an `apply`) can no longer wipe a directory the stack never used. It
   also refuses to run rather than guess if `.env` doesn't name them (#139).
 
+### Fixed
+
+- Dashboard pruned/full label (#32) always showed **Full** on local nodes: the dashboard parsed
+  `MONERO_PRUNE` with `== "true"`, but pithead writes it as `1`/`0`, so a pruned node read as
+  Full. Now accepts `1`/`true`/`yes`/`on`. Found by the live integration harness on a real box.
+- Dashboard DB upgrade path: opening a database created by an early (pre-`timestamp`) schema
+  threw `no such column: timestamp` and aborted the migration, leaving the DB half-upgraded —
+  `_create_tables` built the `idx_ts` index on a column `_migrate_db` hadn't added yet. Indexes
+  are now created after migrations. Found by a new schema-migration intent test.
+
 ### Security
 
 - The monerod RPC credentials are no longer interpolated into the compose healthcheck command

CONTRIBUTING.md

@@ -21,17 +21,26 @@ whole new feature, contributions are very welcome. This guide covers the workflo
    make test
    ```
 
-   This runs everything CI does:
+   This runs everything CI does without a server or Docker:
 
-   - **lint** — `shellcheck` over `pithead` and the test scripts. Keep `pithead`
-     shellcheck-clean (no new warnings).
-   - **test-dashboard** — the dashboard `pytest` suite, which must stay at or above the
-     **80% coverage gate**.
+   - **lint** — `shellcheck` over `pithead` and the test scripts (keep them
+     `--severity=warning` clean).
+   - **test-dashboard** — the dashboard `pytest` suite (must stay ≥ the **80% coverage gate**).
    - **test-stack** — the `pithead` shell test suite.
    - **test-compose** — `docker-compose.yml` interpolation validation.
-
-4. Update the docs in [`docs/`](docs/) (and the README, if relevant) for any
-   user-facing change.
+   - **test-integration-selftest** — the integration harness's own pure logic.
+   - **test-fakes** — the tier-2 contract test (real dashboard clients vs controllable fakes).
+   - the **test-inventory drift check** — fails if a test was added/removed without
+     regenerating [`docs/test-inventory.md`](docs/test-inventory.md) (`make test-inventory`).
+
+   Bigger, infra-dependent suites run separately: `make test-mini-stack` (tier-3 docker) and
+   `make test-integration` (tier-4 live, against a real box — start with `--check`).
+
+4. **Add or update tests** for your change — cover the *intent* (a behavior/contract), not just
+   the line. The [Testing Guide](docs/testing-guide.md) has per-change recipes; the
+   [Testing Strategy](docs/testing-strategy.md) explains the tiers.
+5. Update the docs in [`docs/`](docs/) (and the README, if relevant) for any
+   user-facing change, and run `make test-inventory` if you touched the test suites.
 
 ## Opening a pull request
 

Makefile

@@ -1,7 +1,7 @@
 # Local test entry points (mirror the GitHub Actions CI jobs).
-.PHONY: test test-dashboard test-stack test-compose lint
+.PHONY: test test-dashboard test-stack test-compose test-integration test-integration-selftest test-fakes test-mini-stack lint
 
-test: lint test-dashboard test-stack test-compose ## Run everything
+test: lint test-dashboard test-stack test-compose test-integration-selftest test-fakes ## Run everything that doesn't need a server/docker
 
 test-dashboard: ## Dashboard unit/component tests with coverage gate
 	cd build/dashboard && PYTHONPATH=. python3 -m pytest \
@@ -10,8 +10,33 @@ test-dashboard: ## Dashboard unit/component tests with coverage gate
 test-stack: ## pithead shell test suite
 	bash tests/stack/run.sh
 
-test-compose: ## Validate docker-compose.yml interpolation
+test-compose: ## Validate docker-compose.yml interpolation + hardening invariants (#90)
 	bash tests/stack/test_compose.sh
 
+test-integration-selftest: ## Integration harness pure-logic self-test (no server needed)
+	bash tests/integration/selftest.sh
+
+test-fakes: ## Fake-daemon contract test — real dashboard clients vs controllable fakes (no docker)
+	PYTHONPATH=build/dashboard python3 -m pytest tests/integration/fakes -q
+
+test-mini-stack: ## Fake-daemon docker mini-stack end-to-end (needs docker; CI)
+	bash tests/integration/mini-stack/run-mini-stack.sh
+
+test-inventory: ## Regenerate the test coverage inventory (docs/test-inventory.md)
+	bash tests/inventory.sh > docs/test-inventory.md
+
+test-inventory-check: ## Fail if docs/test-inventory.md is stale (CI drift guard)
+	@bash tests/inventory.sh | diff -u docs/test-inventory.md - \
+		&& echo "test-inventory is up to date" \
+		|| { echo "docs/test-inventory.md is stale — run 'make test-inventory'"; exit 1; }
+
+# End-to-end matrix against a REAL test server (issue #54). Needs a provisioned box; pass
+# connection + options through ARGS, e.g.:
+#   make test-integration ARGS="--host miner@10.0.0.5 --dir pithead --lifecycle"
+# See docs/integration-testing.md.
+test-integration: ## Run the live config-matrix integration suite (requires a test box; pass ARGS=...)
+	bash tests/integration/run.sh $(ARGS)
+
 lint: ## shellcheck the stack scripts
-	shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh
+	shellcheck --severity=warning pithead tests/stack/run.sh tests/stack/test_compose.sh \
+		tests/inventory.sh tests/integration/*.sh tests/integration/mini-stack/*.sh

build/dashboard/mining_dashboard/config/config.py

@@ -31,7 +31,12 @@
 # XMRig Worker API Configuration
 XMRIG_API_PORT = 8080
 API_TIMEOUT = 1         # Connection timeout (seconds) for worker API calls
-UPDATE_INTERVAL = 30    # Frequency (seconds) of the main data aggregation loop
+try:
+    # main data-loop period (s); lowered in integration tests. Tolerate a malformed override
+    # rather than crashing the dashboard at import.
+    UPDATE_INTERVAL = int(float(os.environ.get("UPDATE_INTERVAL", "30")))
+except (TypeError, ValueError):
+    UPDATE_INTERVAL = 30
 
 # --- XvB Algorithm Constants ---
 # Duration of the donation switching cycle (10 minutes)
@@ -127,7 +132,10 @@
 # Whether the bundled monerod is configured to prune the blockchain (config.json
 # monero.prune → MONERO_PRUNE). Used to label the node Pruned/Full in the UI (Issue #32);
 # only meaningful for a local node (we don't control a remote node's pruning).
-MONERO_PRUNE = os.environ.get("MONERO_PRUNE", "true").strip().lower() == "true"
+# pithead renders this as 1/0 (the form monerod's CLI wants), so accept the numeric/boolean
+# truthy forms — not just "true", which silently read pruned nodes as Full before (the
+# pruned/full label is purely display, #32).
+MONERO_PRUNE = os.environ.get("MONERO_PRUNE", "true").strip().lower() in ("true", "1", "yes", "on")
 
 # --- Tari Configuration ---
 # Connection details for the Tari Base Node and Block Explorer