From f26d1882a5eb46dd94747e0549a1ba86540e01ab Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Thu, 11 Jun 2026 15:29:14 -0400 Subject: [PATCH 1/8] Potential drafts for an infra-focused quickstart. --- content/introduction/quickstart/draft-a.mdx | 289 ++++++++++++++++++++ content/introduction/quickstart/draft-b.mdx | 161 +++++++++++ start-stack.sh | 225 +++++++++++++++ 3 files changed, 675 insertions(+) create mode 100644 content/introduction/quickstart/draft-a.mdx create mode 100644 content/introduction/quickstart/draft-b.mdx create mode 100755 start-stack.sh diff --git a/content/introduction/quickstart/draft-a.mdx b/content/introduction/quickstart/draft-a.mdx new file mode 100644 index 0000000..62000bb --- /dev/null +++ b/content/introduction/quickstart/draft-a.mdx @@ -0,0 +1,289 @@ +--- +title: Quickstart +description: Bring up an Indexer and a Host on one machine, peer them over libp2p, and query the result. +--- + +# Quickstart + +Run a Shinzo Indexer and a Host on the same machine, peer them over libp2p, and query indexed Ethereum data through the Host's GraphQL API. If your Geth node is already reachable, the whole thing takes about ten minutes. + +When you're done you'll have: + +- A running Indexer pulling blocks from a Geth node and signing them. +- A running Host receiving those blocks over P2P and serving them. +- A GraphQL query returning real Ethereum data through the Host. +- A understanding of how the two main Shinzo infrastructure pieces fit together. + +## What you're building + +```mermaid +flowchart LR + Geth["Geth (your node)"] -->|RPC + WS| Indexer + Indexer -->|libp2p| Host + Host -->|GraphQL| You["You (curl)"] +``` + +Two containers through one shared Docker bridge. Both containers run on the same VM. The Host dials the Indexer's libp2p port directly over the bridge using the Indexer's published Peer ID. + +## Prerequisites + +- Docker. +- Both `curl` and `jq`. +- A reachable Ethereum execution node (Geth or compatible) exposing JSON-RPC and WebSocket. The Indexer reads from this node; it does not run one for you. Acceptable sources include a node you self-host, a node co-located with a validator, GCP Blockchain Node Engine, or a managed provider like Alchemy or QuickNode. If your node is behind authentication, see the [Indexer install guide's notes on API keys](/indexers/install#do-you-need-an-api-key). + +You don't need a wallet, funds, or a ShinzoHub registration for the quickstart. Registration is what lets your operators participate in the network and earn rewards. It's covered on the [Indexer registration](/indexers/register) and [Host registration](/hosts/register) pages. + +## Set your Geth endpoint + +Export the URL and (optionally) the API key for your Geth node. The rest of the quickstart references these variables. + +```shell +export GETH_RPC_URL="" +export GETH_WS_URL="" +export GETH_API_KEY="" # leave empty if your node has no auth +``` + +:::tip +The Indexer auto-detects the right header (`X-goog-api-key` for GCP Blockchain Node Engine, `X-Api-Key` for most self-hosted Nginx setups) based on the URL. If the node is on your private network and unauthenticated, leave `GETH_API_KEY` empty. +::: + +## Start the Indexer + +The Indexer sits next to a blockchain node, subscribes to new blocks, and turns them into signed structured documents. Run it as a single container with three ports exposed: + +| Host port | Container port | What it is | +| --- | --- | --- | +| `9181` | `9181` | DefraDB GraphQL API. The query interface for raw indexed data. | +| `9171` | `9171` | libp2p P2P port. How Hosts subscribe to this Indexer. | +| `8080` | `8080` | Health, metrics, and registration endpoints. | + +Run everything with: + +```shell +docker pull ghcr.io/shinzonetwork/shinzo-indexer-client:standard + +docker run -d \ + --name shinzo-indexer \ + -e GETH_RPC_URL="$GETH_RPC_URL" \ + -e GETH_WS_URL="$GETH_WS_URL" \ + -e GETH_API_KEY="$GETH_API_KEY" \ + -e INDEXER_START_HEIGHT=0 \ + -e DEFRADB_KEYRING_SECRET=devnet-secret \ + -e DEFRADB_PLAYGROUND=true \ + -e DEFRADB_P2P_ENABLED=true \ + -e DEFRADB_P2P_LISTEN_ADDR=/ip4/0.0.0.0/tcp/9171 \ + -e LOGGER_DEBUG=true \ + -p 9181:9181 \ + -p 9171:9171 \ + -p 8080:8080 \ + ghcr.io/shinzonetwork/shinzo-indexer-client:standard +``` + +`DEFRADB_KEYRING_SECRET` is the password that protects the Indexer's signing key. The Indexer uses this key to sign every document it produces, which gives downstream consumers a way to verify the data came from a real Indexer. `devnet-secret` is fine here. Use something real for anything beyond a throwaway. + +`DEFRADB_P2P_LISTEN_ADDR` tells DefraDB which interface and port to bind libp2p to inside the container. Binding to `0.0.0.0:9171` means the Host container, running on the same docker bridge, can reach it. + +`INDEXER_START_HEIGHT=0` starts indexing from genesis. For Ethereum Mainnet that's a lot of history, but the quickstart only needs a few committed blocks to verify the pipeline. To start closer to head, set this to a recent block height. + +`DEFRADB_PLAYGROUND=true` enables a browser GraphQL playground on the API port. + +## Read the Indexer's P2P address + +The Host needs two things to connect: + +1. The Indexer's libp2p Peer ID +1. A multiaddr it can dial. + +Both come from the Indexer's `/health` endpoint once it finishes starting up. + +```shell +until curl -fsS http://localhost:8080/health | jq -e '.p2p.self.id' >/dev/null 2>&1; do + sleep 2 +done + +curl -s http://localhost:8080/health | jq '.p2p.self' +``` + +The response looks something like: + +```json +{ + "id": "12D3KooWMYhYNBo4zAi9j7TpyGQJBSvbwSSNkgsMrLs6vHUnFUzY", + "addresses": [ + "/ip4/127.0.0.1/tcp/9171/p2p/12D3KooW...", + "/ip4/172.17.0.2/tcp/9171/p2p/12D3KooW..." + ] +} +``` + +`id` is the Indexer's libp2p Peer ID, derived from its keyring secret. It's stable across restarts as long as the secret stays the same. + +`addresses` are the multiaddrs the Indexer is actually listening on. The first one (`127.0.0.1`) is loopback and useless to other containers. The second (`172.17.0.2` here) is the Indexer container's IP on the docker bridge. That's the one the Host will dial. + +Capture both into shell variables: + +```shell +PEER_ID=$(curl -s http://localhost:8080/health | jq -r '.p2p.self.id') +INDEXER_IP=$(curl -s http://localhost:8080/health \ + | jq -r '[.p2p.self.addresses[] | capture("/ip4/(?[0-9.]+)/").ip + | select(. != "127.0.0.1" and . != "0.0.0.0")][0]') + +BOOTSTRAP_PEER="/ip4/${INDEXER_IP}/tcp/9171/p2p/${PEER_ID}" +echo "$BOOTSTRAP_PEER" +``` + +`BOOTSTRAP_PEER` is a libp2p multiaddr: "speak IPv4 to this address on this TCP port, then handshake with this Peer ID." If the Peer ID doesn't match, the Host refuses the connection. That's what makes the connection authenticated end to end. + +## Write the Host config + +The Host reads its configuration from a YAML file mounted into the container. The two values that matter here are the bootstrap peer (the multiaddr we just built) and the keyring secret. + +```shell +cat > ~/host-config.yaml </dev/null 2>&1; do + sleep 2 +done + +curl -s http://localhost:8081/health \ + | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' +``` + +A connected response looks like: + +```json +{ + "status": "ok", + "current_block": 25071330, + "p2p": { + "self": "12D3KooW...HostID", + "peers": ["12D3KooW...IndexerID"] + } +} +``` + +Check the Indexer's side too. Its peer list should now contain the Host: + +```shell +curl -s http://localhost:8080/health | jq '[.p2p.peers[].id]' +``` + +If both list each other, libp2p is connected and DefraDB is replicating between them. Data the Indexer commits lands in the Host within a few seconds. + +## Query the Host + +The Host runs an embedded DefraDB with a GraphQL API on the remapped port `9182`. Once `current_block` is non-zero on the Host's `/health`, query it: + +```shell +curl -s -X POST http://localhost:9182/api/v0/graphql \ + -H 'Content-Type: application/json' \ + -d '{"query":"query { Ethereum__Mainnet__Log(order: { blockNumber: DESC }, limit: 5) { address topics blockNumber transactionHash logIndex } }"}' \ + | jq +``` + +The rows that come back started life as logs on Ethereum Mainnet, were pulled in by the Indexer over the Geth WebSocket, signed, gossiped over libp2p to the Host, and are now being served back to you over GraphQL. More queries are on the [Host examples](/hosts/examples) page. + +## Undo everything + +If you want burn everything down and start again, run: + +```shell +docker rm -f shinzo-indexer shinzo-host +rm ~/host-config.yaml +``` + +The Indexer's DefraDB state lives inside the container and is destroyed with it. If you re-run the quickstart with the same `DEFRADB_KEYRING_SECRET`, the Indexer recovers the same Peer ID and the Host's saved bootstrap peer continues to work. + +## Troubleshooting + +### The Indexer's `/health` never returns a Peer ID + +```shell +docker logs --tail 50 shinzo-indexer +``` + +The most common cause is the Indexer can't reach Geth. Confirm `GETH_RPC_URL` and `GETH_WS_URL` are correct and reachable from the container, and that the API key (if any) is right. The Indexer can fall back to HTTP-only mode if WebSocket is unavailable, and will log that it did. + +### The Host never lists the Indexer as a peer + +Confirm the bootstrap peer multiaddr in the config uses the Indexer's non-loopback container IP, not `127.0.0.1`. From inside the Host container, the Indexer's loopback is unreachable. Verify with: + +```shell +docker exec shinzo-host getent hosts $(echo "$BOOTSTRAP_PEER" | grep -oP '/ip4/\K[0-9.]+') +``` + +Also check that the Peer ID portion of the multiaddr matches what `curl localhost:8080/health | jq .p2p.self.id` returns. + +### `Permission denied` on `.defra/keys` + +The container runs as UID `1001`. If you mounted a host directory for persistence (this quickstart does not), make sure it's owned by `1001:1001`. + +### Port already in use + +The Indexer or the Host is colliding with something else on the machine. Change the published ports in the `docker run` commands. Container-side ports stay the same. + +## Next steps + +- [Indexer install guide](/indexers/install) and [Host install guide](/hosts/install) for production setups: Nginx termination, proper data volumes, resource sizing. +- [Indexer registration](/indexers/register) and [Host registration](/hosts/register) to participate in the network and earn rewards. +- [Building apps with Shinzo](/guides/building-apps-with-shinzo) to use the Host's GraphQL API from an application. +- [Architecture overview](/reference/architecture-overview) for how Indexers, Hosts, ShinzoHub, and Outposts fit together. diff --git a/content/introduction/quickstart/draft-b.mdx b/content/introduction/quickstart/draft-b.mdx new file mode 100644 index 0000000..fc5aa8a --- /dev/null +++ b/content/introduction/quickstart/draft-b.mdx @@ -0,0 +1,161 @@ +--- +title: Quickstart +description: Bring up an Indexer and a Host on one machine, peer them over libp2p, and query the result. +--- + +# Quickstart + +Run a Shinzo Indexer and Host on the same machine, peer them, and query indexed Ethereum data through the Host's GraphQL API. About ten minutes. + +For background on what these components are, see [How it works](/introduction/how-it-works) and the [Architecture overview](/reference/architecture-overview). + +## Prerequisites + +- Docker, `curl`, `jq`. +- An Ethereum execution node (Geth or compatible) reachable over JSON-RPC and WebSocket. Self-hosted, GCP Blockchain Node Engine, Alchemy, QuickNode, etc. all work. If your node uses an API key, the Indexer auto-detects the right header. See [Do you need an API key?](/indexers/install#do-you-need-an-api-key). + +No wallet, funds, or ShinzoHub registration required. + +## Set your Geth endpoint + +```shell +export GETH_RPC_URL="" +export GETH_WS_URL="" +export GETH_API_KEY="" # leave empty if unauthenticated +``` + +## Start the Indexer + +```shell +docker run -d \ + --name shinzo-indexer \ + -e GETH_RPC_URL="$GETH_RPC_URL" \ + -e GETH_WS_URL="$GETH_WS_URL" \ + -e GETH_API_KEY="$GETH_API_KEY" \ + -e INDEXER_START_HEIGHT=0 \ + -e DEFRADB_KEYRING_SECRET=devnet-secret \ + -e DEFRADB_PLAYGROUND=true \ + -e DEFRADB_P2P_ENABLED=true \ + -e DEFRADB_P2P_LISTEN_ADDR=/ip4/0.0.0.0/tcp/9171 \ + -e LOGGER_DEBUG=true \ + -p 9181:9181 -p 9171:9171 -p 8080:8080 \ + ghcr.io/shinzonetwork/shinzo-indexer-client:standard +``` + +Ports: `9181` GraphQL API, `9171` libp2p, `8080` health and metrics. See the [Indexer install reference](/indexers/install#exposed-ports) for details. + +## Capture the Indexer's bootstrap peer + +Wait for the Indexer to publish its libp2p self-info on `/health`, then build the multiaddr the Host will dial. + +```shell +until curl -fsS http://localhost:8080/health | jq -e '.p2p.self.id' >/dev/null 2>&1; do + sleep 2 +done + +PEER_ID=$(curl -s http://localhost:8080/health | jq -r '.p2p.self.id') +INDEXER_IP=$(curl -s http://localhost:8080/health \ + | jq -r '[.p2p.self.addresses[] | capture("/ip4/(?[0-9.]+)/").ip + | select(. != "127.0.0.1" and . != "0.0.0.0")][0]') +BOOTSTRAP_PEER="/ip4/${INDEXER_IP}/tcp/9171/p2p/${PEER_ID}" +echo "$BOOTSTRAP_PEER" +``` + +## Write the Host config + +```shell +cat > ~/host-config.yaml </dev/null 2>&1; do + sleep 2 +done + +curl -s http://localhost:8081/health \ + | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' +``` + +Check the Indexer's side too: + +```shell +curl -s http://localhost:8080/health | jq '[.p2p.peers[].id]' +``` + +If each lists the other's Peer ID, libp2p is connected and DefraDB is replicating. + +## Query the Host + +Once `current_block` is non-zero on the Host's `/health`, query through the Host's GraphQL API on port `9182`: + +```shell +curl -s -X POST http://localhost:9182/api/v0/graphql \ + -H 'Content-Type: application/json' \ + -d '{"query":"query { Ethereum__Mainnet__Log(order: { blockNumber: DESC }, limit: 5) { address topics blockNumber transactionHash logIndex } }"}' \ + | jq +``` + +More queries: [Host examples](/hosts/examples). + +## Teardown + +```shell +docker rm -f shinzo-indexer shinzo-host +rm ~/host-config.yaml +``` + +## Troubleshooting + +| Symptom | Likely cause | +| --- | --- | +| Indexer `/health` never returns a Peer ID | Indexer can't reach Geth. Check `docker logs shinzo-indexer`. | +| Host never lists the Indexer peer | Bootstrap multiaddr points at `127.0.0.1` instead of the bridge IP, or the Peer ID has drifted. Rebuild `BOOTSTRAP_PEER`. | +| `Permission denied` on `.defra/keys` | Volume not owned by UID `1001`. See [Indexer troubleshooting](/indexers/install#permission-denied-on-defrakeys). | +| Port already in use | Bump the published ports in the `docker run` flags. Container-side ports stay the same. | + +## Next steps + +- Production install on a VM: [Indexer](/indexers/install#using-docker), [Host](/hosts/install#vm-deployment). +- Join the network: [Indexer registration](/indexers/register), [Host registration](/hosts/register). +- Build against it: [Building apps with Shinzo](/guides/building-apps-with-shinzo). +- Component internals: [Architecture overview](/reference/architecture-overview). diff --git a/start-stack.sh b/start-stack.sh new file mode 100755 index 0000000..57407b6 --- /dev/null +++ b/start-stack.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# +# start-stack.sh — bring up a Shinzo Indexer and a Shinzo Host on the SAME VM, +# connect the Host to the Indexer over P2P, and verify the connection. +# +# Runs end-to-end with no human interaction: +# 1. (re)starts a fresh Indexer container +# 2. waits for the Indexer health endpoint + P2P self-info +# 3. writes the Host config pointing at the Indexer's real libp2p address +# 4. (re)starts the Host container on remapped ports +# 5. verifies both peers see each other +# +# Requirements: docker, curl, jq. +# +set -euo pipefail + +# ---- Indexer Geth connection (override via env) ----------------------------- +GETH_RPC_URL="${GETH_RPC_URL:-http://35.193.228.182:8080}" +GETH_WS_URL="${GETH_WS_URL:-ws://35.193.228.182:8080}" +GETH_API_KEY="${GETH_API_KEY:-df7fc1e2c686d61a26d390d00aad49f61e899c7d1cf9f97d13c8378f6819e6db}" +GETH_API_KEY_TYPE="${GETH_API_KEY_TYPE:-x-goog-api-key}" +INDEXER_START_HEIGHT="${INDEXER_START_HEIGHT:-0}" +INDEXER_KEYRING_SECRET="${INDEXER_KEYRING_SECRET:-devnet-secret}" +INDEXER_P2P_LISTEN_ADDR="${INDEXER_P2P_LISTEN_ADDR:-/ip4/0.0.0.0/tcp/9174}" + +# ---- Indexer container / ports ---------------------------------------------- +INDEXER_IMAGE="${INDEXER_IMAGE:-ghcr.io/shinzonetwork/shinzo-indexer-client:standard}" +INDEXER_NAME="${INDEXER_NAME:-shinzo-indexer}" +INDEXER_API_PORT="${INDEXER_API_PORT:-9181}" # -> container 9181 (GraphQL API) +INDEXER_P2P_PUB_PORT="${INDEXER_P2P_PUB_PORT:-9171}" # -> container 9171 (published P2P) +INDEXER_HEALTH_PORT="${INDEXER_HEALTH_PORT:-8080}" # -> container 8080 (health) +INDEXER_HEALTH_URL="${INDEXER_HEALTH_URL:-http://localhost:${INDEXER_HEALTH_PORT}}" + +# ---- Host config / container / ports ---------------------------------------- +HOST_CONFIG="${HOST_CONFIG:-$HOME/host-config.yaml}" +KEYRING_SECRET="${KEYRING_SECRET:-host-devnet-secret}" # Host DefraDB keyring secret +HOST_IMAGE="${HOST_IMAGE:-ghcr.io/shinzonetwork/shinzo-host-client:standard}" +HOST_NAME="${HOST_NAME:-shinzo-host}" +HOST_API_PORT="${HOST_API_PORT:-9182}" # -> container 9181 (GraphQL API) +HOST_P2P_PORT="${HOST_P2P_PORT:-9172}" # -> container 9171 (P2P) +HOST_HEALTH_PORT="${HOST_HEALTH_PORT:-8081}" # -> container 8080 (health) + +# How the Host container reaches the Indexer's P2P endpoint. +# Leave both empty to auto-detect from the indexer's advertised /health addresses +# (recommended). Override INDEXER_P2P_IP with a remote VM's IP for separate-VM +# setups, or INDEXER_P2P_PORT to pin the libp2p port explicitly. +INDEXER_P2P_IP="${INDEXER_P2P_IP:-}" +INDEXER_P2P_PORT="${INDEXER_P2P_PORT:-}" + +log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +die() { printf '\033[1;31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +# ---- Preflight -------------------------------------------------------------- +command -v docker >/dev/null || die "docker not found" +command -v curl >/dev/null || die "curl not found" +command -v jq >/dev/null || die "jq not found (install: sudo apt-get install -y jq)" + +# ---- Step 1: (re)start a fresh Indexer -------------------------------------- +if docker ps -a --format '{{.Names}}' | grep -qx "$INDEXER_NAME"; then + log "Removing existing Indexer container '${INDEXER_NAME}'" + docker rm -f "$INDEXER_NAME" >/dev/null +fi + +log "Starting Indexer container '${INDEXER_NAME}'" +docker run -d \ + --name "$INDEXER_NAME" \ + -e GETH_RPC_URL="${GETH_RPC_URL}" \ + -e GETH_WS_URL="${GETH_WS_URL}" \ + -e GETH_API_KEY="${GETH_API_KEY}" \ + -e GETH_API_KEY_TYPE="${GETH_API_KEY_TYPE}" \ + -e INDEXER_START_HEIGHT="${INDEXER_START_HEIGHT}" \ + -e DEFRADB_KEYRING_SECRET="${INDEXER_KEYRING_SECRET}" \ + -e DEFRADB_PLAYGROUND=true \ + -e DEFRADB_P2P_ENABLED=true \ + -e DEFRADB_P2P_LISTEN_ADDR="${INDEXER_P2P_LISTEN_ADDR}" \ + -e LOGGER_DEBUG=true \ + -p "${INDEXER_API_PORT}:9181" \ + -p "${INDEXER_P2P_PUB_PORT}:9171" \ + -p "${INDEXER_HEALTH_PORT}:8080" \ + "$INDEXER_IMAGE" >/dev/null + +# ---- Step 2: wait for the Indexer's P2P self-info --------------------------- +# The indexer's /health endpoint reports the Peer ID AND the real libp2p +# listen addresses. The internal P2P port is NOT necessarily the published +# 9171 (DEFRADB_P2P_LISTEN_ADDR may bind 9174), so read it from here rather +# than assuming. We also read a non-loopback container IP so the Host +# container can dial the indexer directly over the shared docker bridge. +log "Waiting for Indexer P2P self-info from ${INDEXER_HEALTH_URL} ..." +SELF_JSON="" +for i in $(seq 1 60); do + SELF_JSON="$(curl -fsS -H 'Accept: application/json' "${INDEXER_HEALTH_URL}/health" 2>/dev/null \ + | jq -c '.p2p.self // empty' 2>/dev/null || true)" + [[ -z "$SELF_JSON" ]] && SELF_JSON="$(curl -fsS "${INDEXER_HEALTH_URL}/registration" 2>/dev/null \ + | jq -c '.p2p.self // empty' 2>/dev/null || true)" + if [[ -n "$SELF_JSON" && "$(jq -r '.id // empty' <<<"$SELF_JSON")" != "" ]]; then + break + fi + sleep 2 + if [[ "$i" -eq 60 ]]; then + docker logs --tail 30 "$INDEXER_NAME" 2>&1 || true + die "Indexer did not report P2P self-info in time. Check: docker logs ${INDEXER_NAME}" + fi +done + +PEER_ID="$(jq -r '.id // empty' <<<"$SELF_JSON")" +[[ -n "$PEER_ID" ]] || die "Indexer reported no Peer ID." +log "Indexer Peer ID: ${PEER_ID}" + +# Derive the P2P port from the indexer's advertised addresses (first non-loopback, +# else first address). Allow explicit override via INDEXER_P2P_PORT. +if [[ -z "${INDEXER_P2P_PORT:-}" ]]; then + INDEXER_P2P_PORT="$(jq -r ' + ([.addresses[]? | select(test("127\\.0\\.0\\.1")|not)] + .addresses)[0] + | capture("/tcp/(?

[0-9]+)").p // empty' <<<"$SELF_JSON")" + INDEXER_P2P_PORT="${INDEXER_P2P_PORT:-9171}" +fi + +# Determine how the Host container reaches the indexer: +# - If INDEXER_P2P_IP is preset (e.g. a remote VM IP), use it. +# - Else prefer the indexer's own non-loopback container IP from its +# advertised addresses (direct container-to-container over the bridge). +# - Else fall back to the docker0 gateway. +if [[ -z "$INDEXER_P2P_IP" ]]; then + INDEXER_P2P_IP="$(jq -r ' + [.addresses[]? | capture("/ip4/(?[0-9.]+)/").ip + | select(. != "127.0.0.1" and . != "0.0.0.0")][0] // empty' <<<"$SELF_JSON")" +fi +if [[ -z "$INDEXER_P2P_IP" ]]; then + INDEXER_P2P_IP="$(ip -4 addr show docker0 2>/dev/null | grep -oP 'inet \K[\d.]+' || true)" + INDEXER_P2P_IP="${INDEXER_P2P_IP:-172.17.0.1}" +fi +log "Indexer P2P reachable at ${INDEXER_P2P_IP}:${INDEXER_P2P_PORT} (from inside the Host container)" + +BOOTSTRAP_PEER="/ip4/${INDEXER_P2P_IP}/tcp/${INDEXER_P2P_PORT}/p2p/${PEER_ID}" +log "Bootstrap peer: ${BOOTSTRAP_PEER}" + +# ---- Step 3: write the Host config ------------------------------------------ +log "Writing Host config to ${HOST_CONFIG}" +cat > "$HOST_CONFIG" </dev/null +fi + +log "Starting Host container '${HOST_NAME}'" +docker run -d \ + --name "$HOST_NAME" \ + -e BOOTSTRAP_PEERS="${BOOTSTRAP_PEER}" \ + -v "${HOST_CONFIG}:/app/config.yaml:ro" \ + -p "${HOST_API_PORT}:9181" \ + -p "${HOST_P2P_PORT}:9171" \ + -p "${HOST_HEALTH_PORT}:8080" \ + "$HOST_IMAGE" >/dev/null + +# ---- Step 5: verify --------------------------------------------------------- +log "Waiting for Host to connect to the Indexer ..." +CONNECTED=false +for i in $(seq 1 45); do + PEERS="$(curl -fsS -H 'Accept: application/json' "http://localhost:${HOST_HEALTH_PORT}/health" 2>/dev/null \ + | jq -r '[.p2p.peers[]?.id] | join(",")' 2>/dev/null || true)" + if [[ "$PEERS" == *"$PEER_ID"* ]]; then + CONNECTED=true + break + fi + sleep 2 +done + +log "Host health / P2P peers:" +curl -fsS -H 'Accept: application/json' "http://localhost:${HOST_HEALTH_PORT}/health" \ + | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' || true + +if [[ "$CONNECTED" == true ]]; then + log "SUCCESS: Host is connected to the Indexer over P2P." +else + log "ERROR: Host has not listed the Indexer peer yet. Check: docker logs -f ${HOST_NAME}" +fi + +cat < Date: Fri, 12 Jun 2026 13:13:39 -0400 Subject: [PATCH 2/8] Completes draft. Need a run through now. --- content/introduction/quickstart/draft-b.mdx | 161 ------------------ .../quickstart/{draft-a.mdx => index.mdx} | 56 ++---- 2 files changed, 18 insertions(+), 199 deletions(-) delete mode 100644 content/introduction/quickstart/draft-b.mdx rename content/introduction/quickstart/{draft-a.mdx => index.mdx} (75%) diff --git a/content/introduction/quickstart/draft-b.mdx b/content/introduction/quickstart/draft-b.mdx deleted file mode 100644 index fc5aa8a..0000000 --- a/content/introduction/quickstart/draft-b.mdx +++ /dev/null @@ -1,161 +0,0 @@ ---- -title: Quickstart -description: Bring up an Indexer and a Host on one machine, peer them over libp2p, and query the result. ---- - -# Quickstart - -Run a Shinzo Indexer and Host on the same machine, peer them, and query indexed Ethereum data through the Host's GraphQL API. About ten minutes. - -For background on what these components are, see [How it works](/introduction/how-it-works) and the [Architecture overview](/reference/architecture-overview). - -## Prerequisites - -- Docker, `curl`, `jq`. -- An Ethereum execution node (Geth or compatible) reachable over JSON-RPC and WebSocket. Self-hosted, GCP Blockchain Node Engine, Alchemy, QuickNode, etc. all work. If your node uses an API key, the Indexer auto-detects the right header. See [Do you need an API key?](/indexers/install#do-you-need-an-api-key). - -No wallet, funds, or ShinzoHub registration required. - -## Set your Geth endpoint - -```shell -export GETH_RPC_URL="" -export GETH_WS_URL="" -export GETH_API_KEY="" # leave empty if unauthenticated -``` - -## Start the Indexer - -```shell -docker run -d \ - --name shinzo-indexer \ - -e GETH_RPC_URL="$GETH_RPC_URL" \ - -e GETH_WS_URL="$GETH_WS_URL" \ - -e GETH_API_KEY="$GETH_API_KEY" \ - -e INDEXER_START_HEIGHT=0 \ - -e DEFRADB_KEYRING_SECRET=devnet-secret \ - -e DEFRADB_PLAYGROUND=true \ - -e DEFRADB_P2P_ENABLED=true \ - -e DEFRADB_P2P_LISTEN_ADDR=/ip4/0.0.0.0/tcp/9171 \ - -e LOGGER_DEBUG=true \ - -p 9181:9181 -p 9171:9171 -p 8080:8080 \ - ghcr.io/shinzonetwork/shinzo-indexer-client:standard -``` - -Ports: `9181` GraphQL API, `9171` libp2p, `8080` health and metrics. See the [Indexer install reference](/indexers/install#exposed-ports) for details. - -## Capture the Indexer's bootstrap peer - -Wait for the Indexer to publish its libp2p self-info on `/health`, then build the multiaddr the Host will dial. - -```shell -until curl -fsS http://localhost:8080/health | jq -e '.p2p.self.id' >/dev/null 2>&1; do - sleep 2 -done - -PEER_ID=$(curl -s http://localhost:8080/health | jq -r '.p2p.self.id') -INDEXER_IP=$(curl -s http://localhost:8080/health \ - | jq -r '[.p2p.self.addresses[] | capture("/ip4/(?[0-9.]+)/").ip - | select(. != "127.0.0.1" and . != "0.0.0.0")][0]') -BOOTSTRAP_PEER="/ip4/${INDEXER_IP}/tcp/9171/p2p/${PEER_ID}" -echo "$BOOTSTRAP_PEER" -``` - -## Write the Host config - -```shell -cat > ~/host-config.yaml </dev/null 2>&1; do - sleep 2 -done - -curl -s http://localhost:8081/health \ - | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' -``` - -Check the Indexer's side too: - -```shell -curl -s http://localhost:8080/health | jq '[.p2p.peers[].id]' -``` - -If each lists the other's Peer ID, libp2p is connected and DefraDB is replicating. - -## Query the Host - -Once `current_block` is non-zero on the Host's `/health`, query through the Host's GraphQL API on port `9182`: - -```shell -curl -s -X POST http://localhost:9182/api/v0/graphql \ - -H 'Content-Type: application/json' \ - -d '{"query":"query { Ethereum__Mainnet__Log(order: { blockNumber: DESC }, limit: 5) { address topics blockNumber transactionHash logIndex } }"}' \ - | jq -``` - -More queries: [Host examples](/hosts/examples). - -## Teardown - -```shell -docker rm -f shinzo-indexer shinzo-host -rm ~/host-config.yaml -``` - -## Troubleshooting - -| Symptom | Likely cause | -| --- | --- | -| Indexer `/health` never returns a Peer ID | Indexer can't reach Geth. Check `docker logs shinzo-indexer`. | -| Host never lists the Indexer peer | Bootstrap multiaddr points at `127.0.0.1` instead of the bridge IP, or the Peer ID has drifted. Rebuild `BOOTSTRAP_PEER`. | -| `Permission denied` on `.defra/keys` | Volume not owned by UID `1001`. See [Indexer troubleshooting](/indexers/install#permission-denied-on-defrakeys). | -| Port already in use | Bump the published ports in the `docker run` flags. Container-side ports stay the same. | - -## Next steps - -- Production install on a VM: [Indexer](/indexers/install#using-docker), [Host](/hosts/install#vm-deployment). -- Join the network: [Indexer registration](/indexers/register), [Host registration](/hosts/register). -- Build against it: [Building apps with Shinzo](/guides/building-apps-with-shinzo). -- Component internals: [Architecture overview](/reference/architecture-overview). diff --git a/content/introduction/quickstart/draft-a.mdx b/content/introduction/quickstart/index.mdx similarity index 75% rename from content/introduction/quickstart/draft-a.mdx rename to content/introduction/quickstart/index.mdx index 62000bb..32d35d9 100644 --- a/content/introduction/quickstart/draft-a.mdx +++ b/content/introduction/quickstart/index.mdx @@ -31,7 +31,7 @@ Two containers through one shared Docker bridge. Both containers run on the same - Both `curl` and `jq`. - A reachable Ethereum execution node (Geth or compatible) exposing JSON-RPC and WebSocket. The Indexer reads from this node; it does not run one for you. Acceptable sources include a node you self-host, a node co-located with a validator, GCP Blockchain Node Engine, or a managed provider like Alchemy or QuickNode. If your node is behind authentication, see the [Indexer install guide's notes on API keys](/indexers/install#do-you-need-an-api-key). -You don't need a wallet, funds, or a ShinzoHub registration for the quickstart. Registration is what lets your operators participate in the network and earn rewards. It's covered on the [Indexer registration](/indexers/register) and [Host registration](/hosts/register) pages. +You don't need a wallet, funds, or a ShinzoHub registration for this quickstart. Registration is what lets your operators participate in the network and earn rewards. It's covered on the [Indexer registration](/indexers/register) and [Host registration](/hosts/register) pages. ## Set your Geth endpoint @@ -79,28 +79,24 @@ docker run -d \ ghcr.io/shinzonetwork/shinzo-indexer-client:standard ``` -`DEFRADB_KEYRING_SECRET` is the password that protects the Indexer's signing key. The Indexer uses this key to sign every document it produces, which gives downstream consumers a way to verify the data came from a real Indexer. `devnet-secret` is fine here. Use something real for anything beyond a throwaway. +`DEFRADB_KEYRING_SECRET` is the password that protects the Indexer's signing key. The Indexer uses this key to sign every document it produces, which gives downstream consumers a way to verify the data came from a real Indexer. The `devnet-secret` password is fine for this quickstart, but remember use something more secure for anything in a production environment. -`DEFRADB_P2P_LISTEN_ADDR` tells DefraDB which interface and port to bind libp2p to inside the container. Binding to `0.0.0.0:9171` means the Host container, running on the same docker bridge, can reach it. +`DEFRADB_P2P_LISTEN_ADDR` tells DefraDB which interface and port to bind libp2p to inside the container. Binding to `0.0.0.0:9171` means the Host container, running on the same Docker bridge, can reach it. -`INDEXER_START_HEIGHT=0` starts indexing from genesis. For Ethereum Mainnet that's a lot of history, but the quickstart only needs a few committed blocks to verify the pipeline. To start closer to head, set this to a recent block height. +`INDEXER_START_HEIGHT=0` starts indexing from genesis. For Ethereum Mainnet that's a lot of history, but this quickstart only needs a few committed blocks to verify the pipeline. To start closer to head, set this to a recent block height. -`DEFRADB_PLAYGROUND=true` enables a browser GraphQL playground on the API port. +`DEFRADB_PLAYGROUND=true` enables a browser-based GraphQL playground on the API port. ## Read the Indexer's P2P address The Host needs two things to connect: -1. The Indexer's libp2p Peer ID -1. A multiaddr it can dial. +1. The Indexer's libp2p Peer ID. +1. A multiaddr it can dial. Both come from the Indexer's `/health` endpoint once it finishes starting up. ```shell -until curl -fsS http://localhost:8080/health | jq -e '.p2p.self.id' >/dev/null 2>&1; do - sleep 2 -done - curl -s http://localhost:8080/health | jq '.p2p.self' ``` @@ -118,12 +114,13 @@ The response looks something like: `id` is the Indexer's libp2p Peer ID, derived from its keyring secret. It's stable across restarts as long as the secret stays the same. -`addresses` are the multiaddrs the Indexer is actually listening on. The first one (`127.0.0.1`) is loopback and useless to other containers. The second (`172.17.0.2` here) is the Indexer container's IP on the docker bridge. That's the one the Host will dial. +`addresses` are the multiaddrs the Indexer is actually listening on. The first one (`127.0.0.1`) is loopback and useless to other containers. The second (`172.17.0.2` here) is the Indexer container's IP on the Docker bridge. That's the one the Host will dial. Capture both into shell variables: ```shell PEER_ID=$(curl -s http://localhost:8080/health | jq -r '.p2p.self.id') + INDEXER_IP=$(curl -s http://localhost:8080/health \ | jq -r '[.p2p.self.addresses[] | capture("/ip4/(?[0-9.]+)/").ip | select(. != "127.0.0.1" and . != "0.0.0.0")][0]') @@ -132,14 +129,13 @@ BOOTSTRAP_PEER="/ip4/${INDEXER_IP}/tcp/9171/p2p/${PEER_ID}" echo "$BOOTSTRAP_PEER" ``` -`BOOTSTRAP_PEER` is a libp2p multiaddr: "speak IPv4 to this address on this TCP port, then handshake with this Peer ID." If the Peer ID doesn't match, the Host refuses the connection. That's what makes the connection authenticated end to end. +`BOOTSTRAP_PEER` is a libp2p multiaddr that essentially saus _"speak IPv4 to this address on this TCP port, then handshake with this Peer ID."_ If the Peer ID doesn't match, the Host refuses the connection. This is what makes the connection authenticated end to end. ## Write the Host config -The Host reads its configuration from a YAML file mounted into the container. The two values that matter here are the bootstrap peer (the multiaddr we just built) and the keyring secret. +The Host reads its configuration from a YAML file mounted into the container. The two values that matter here are the bootstrap peer (the multiaddr we just built) and the keyring secret. Create this file and save it as `~/host-config.yaml`: ```shell -cat > ~/host-config.yaml </dev/null 2>&1; do - sleep 2 -done - -curl -s http://localhost:8081/health \ - | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' +curl -s http://localhost:8081/health | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' ``` A connected response looks like: @@ -227,7 +216,7 @@ Check the Indexer's side too. Its peer list should now contain the Host: curl -s http://localhost:8080/health | jq '[.p2p.peers[].id]' ``` -If both list each other, libp2p is connected and DefraDB is replicating between them. Data the Indexer commits lands in the Host within a few seconds. +If both list each other, libp2p is connected and DefraDB is replicating between them! Data from the Indexer lands in the Host within a few seconds. ## Query the Host @@ -240,19 +229,17 @@ curl -s -X POST http://localhost:9182/api/v0/graphql \ | jq ``` -The rows that come back started life as logs on Ethereum Mainnet, were pulled in by the Indexer over the Geth WebSocket, signed, gossiped over libp2p to the Host, and are now being served back to you over GraphQL. More queries are on the [Host examples](/hosts/examples) page. +The rows that come back started life as logs on Ethereum, were pulled in by the Indexer over the Geth WebSocket, signed, gossiped over libp2p to the Host, and are now being served back to you over GraphQL. More queries are on the [Host examples](/hosts/examples) page. ## Undo everything -If you want burn everything down and start again, run: +If you want to burn everything down and start again, just run: ```shell docker rm -f shinzo-indexer shinzo-host rm ~/host-config.yaml ``` -The Indexer's DefraDB state lives inside the container and is destroyed with it. If you re-run the quickstart with the same `DEFRADB_KEYRING_SECRET`, the Indexer recovers the same Peer ID and the Host's saved bootstrap peer continues to work. - ## Troubleshooting ### The Indexer's `/health` never returns a Peer ID @@ -280,10 +267,3 @@ The container runs as UID `1001`. If you mounted a host directory for persistenc ### Port already in use The Indexer or the Host is colliding with something else on the machine. Change the published ports in the `docker run` commands. Container-side ports stay the same. - -## Next steps - -- [Indexer install guide](/indexers/install) and [Host install guide](/hosts/install) for production setups: Nginx termination, proper data volumes, resource sizing. -- [Indexer registration](/indexers/register) and [Host registration](/hosts/register) to participate in the network and earn rewards. -- [Building apps with Shinzo](/guides/building-apps-with-shinzo) to use the Host's GraphQL API from an application. -- [Architecture overview](/reference/architecture-overview) for how Indexers, Hosts, ShinzoHub, and Outposts fit together. From 677fce53ab5136bedcef8b55f70d62115db34299 Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:47:15 -0400 Subject: [PATCH 3/8] Updates outputs. Renames title. --- content/introduction/quickstart/index.mdx | 58 ++++++++++++++++------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/content/introduction/quickstart/index.mdx b/content/introduction/quickstart/index.mdx index 32d35d9..2a686b2 100644 --- a/content/introduction/quickstart/index.mdx +++ b/content/introduction/quickstart/index.mdx @@ -1,10 +1,8 @@ --- -title: Quickstart +title: Operator Quickstart description: Bring up an Indexer and a Host on one machine, peer them over libp2p, and query the result. --- -# Quickstart - Run a Shinzo Indexer and a Host on the same machine, peer them over libp2p, and query indexed Ethereum data through the Host's GraphQL API. If your Geth node is already reachable, the whole thing takes about ten minutes. When you're done you'll have: @@ -100,15 +98,14 @@ Both come from the Indexer's `/health` endpoint once it finishes starting up. curl -s http://localhost:8080/health | jq '.p2p.self' ``` -The response looks something like: - -```json +```output { - "id": "12D3KooWMYhYNBo4zAi9j7TpyGQJBSvbwSSNkgsMrLs6vHUnFUzY", + "id": "12D3KooWK8zmiDmX91PwDV1PsqtgA1UUDuuyipVBVPEjrvwgoFJH", "addresses": [ - "/ip4/127.0.0.1/tcp/9171/p2p/12D3KooW...", - "/ip4/172.17.0.2/tcp/9171/p2p/12D3KooW..." - ] + "/ip4/127.0.0.1/tcp/9171", + "/ip4/172.17.0.2/tcp/9171" + ], + "public_key": "8a7f061eeaaec8b8130ce4b9d6e519bbe76b9a4bc038b7e6743a773ad3915e02" } ``` @@ -129,6 +126,10 @@ BOOTSTRAP_PEER="/ip4/${INDEXER_IP}/tcp/9171/p2p/${PEER_ID}" echo "$BOOTSTRAP_PEER" ``` +```output +/ip4/172.17.0.2/tcp/9171/p2p/12D3KooWK8zmiDmX91PwDV1PsqtgA1UUDuuyipVBVPEjrvwgoFJH +``` + `BOOTSTRAP_PEER` is a libp2p multiaddr that essentially saus _"speak IPv4 to this address on this TCP port, then handshake with this Peer ID."_ If the Peer ID doesn't match, the Host refuses the connection. This is what makes the connection authenticated end to end. ## Write the Host config @@ -197,15 +198,15 @@ The Host's `/health` lists the peers it's connected to. Once the Indexer's Peer curl -s http://localhost:8081/health | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' ``` -A connected response looks like: - -```json +```output { - "status": "ok", - "current_block": 25071330, + "status": "healthy", + "current_block": 25303386, "p2p": { - "self": "12D3KooW...HostID", - "peers": ["12D3KooW...IndexerID"] + "self": "12D3KooWK76zTyFW73BSwoQRkuM45Aky7SNSCQGkzjGvxGy8Y76Z", + "peers": [ + "12D3KooWK8zmiDmX91PwDV1PsqtgA1UUDuuyipVBVPEjrvwgoFJH" + ] } } ``` @@ -216,6 +217,12 @@ Check the Indexer's side too. Its peer list should now contain the Host: curl -s http://localhost:8080/health | jq '[.p2p.peers[].id]' ``` +```output +[ + "12D3KooWK76zTyFW73BSwoQRkuM45Aky7SNSCQGkzjGvxGy8Y76Z" +] +``` + If both list each other, libp2p is connected and DefraDB is replicating between them! Data from the Indexer lands in the Host within a few seconds. ## Query the Host @@ -229,7 +236,22 @@ curl -s -X POST http://localhost:9182/api/v0/graphql \ | jq ``` -The rows that come back started life as logs on Ethereum, were pulled in by the Indexer over the Geth WebSocket, signed, gossiped over libp2p to the Host, and are now being served back to you over GraphQL. More queries are on the [Host examples](/hosts/examples) page. +```output +{ + "data": { + "Ethereum__Mainnet__Log": [ + { + "address": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + "blockNumber": 25303402, + "logIndex": 899, + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000710ed94f2a25859b7a45337f4245da34a6f15190", + +... +``` + +The rows that come back were originally just logs on Ethereum, then pulled in by the Indexer over the Geth WebSocket, signed, gossiped over libp2p to the Host, and are now being served back to you over GraphQL. More queries are on the [Host examples](/hosts/examples) page. ## Undo everything From 22408b323e488a969efe4a7a102150357babef14 Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:47:37 -0400 Subject: [PATCH 4/8] Moves quickstart. --- .../introduction/{quickstart => operator-quickstart}/index.mdx | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename content/introduction/{quickstart => operator-quickstart}/index.mdx (100%) diff --git a/content/introduction/quickstart/index.mdx b/content/introduction/operator-quickstart/index.mdx similarity index 100% rename from content/introduction/quickstart/index.mdx rename to content/introduction/operator-quickstart/index.mdx From 4b3268834697f2c44e879f02236524c59f375523 Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:05:41 +0100 Subject: [PATCH 5/8] Moves guide into it's own folder. --- .../index.mdx} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename content/guides/{building-apps-with-shinzo.md => building-apps-with-shinzo/index.mdx} (100%) diff --git a/content/guides/building-apps-with-shinzo.md b/content/guides/building-apps-with-shinzo/index.mdx similarity index 100% rename from content/guides/building-apps-with-shinzo.md rename to content/guides/building-apps-with-shinzo/index.mdx From 639c0e534faf2d64a4510537b77db489cb1612be Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:05:55 +0100 Subject: [PATCH 6/8] Moves operator quickstart into guides. --- content/{introduction => guides}/operator-quickstart/index.mdx | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename content/{introduction => guides}/operator-quickstart/index.mdx (100%) diff --git a/content/introduction/operator-quickstart/index.mdx b/content/guides/operator-quickstart/index.mdx similarity index 100% rename from content/introduction/operator-quickstart/index.mdx rename to content/guides/operator-quickstart/index.mdx From 6526a57d5fe8312f252e385565bd679f01704a96 Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:06:48 +0100 Subject: [PATCH 7/8] Includes file moves in sidebar. --- sidebars.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sidebars.ts b/sidebars.ts index 8c3ad74..44f1e00 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -52,7 +52,8 @@ const sidebars: SidebarsConfig = { type: "category", label: "Guides", items: [ - "guides/building-apps-with-shinzo" + "guides/building-apps-with-shinzo/index", + "guides/operator-quickstart/index" ], }, { From a6419835b3881fbe23ef9d052b88b1f4da5ed315 Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:08:10 +0100 Subject: [PATCH 8/8] Deletes script. --- start-stack.sh | 225 ------------------------------------------------- 1 file changed, 225 deletions(-) delete mode 100755 start-stack.sh diff --git a/start-stack.sh b/start-stack.sh deleted file mode 100755 index 57407b6..0000000 --- a/start-stack.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bash -# -# start-stack.sh — bring up a Shinzo Indexer and a Shinzo Host on the SAME VM, -# connect the Host to the Indexer over P2P, and verify the connection. -# -# Runs end-to-end with no human interaction: -# 1. (re)starts a fresh Indexer container -# 2. waits for the Indexer health endpoint + P2P self-info -# 3. writes the Host config pointing at the Indexer's real libp2p address -# 4. (re)starts the Host container on remapped ports -# 5. verifies both peers see each other -# -# Requirements: docker, curl, jq. -# -set -euo pipefail - -# ---- Indexer Geth connection (override via env) ----------------------------- -GETH_RPC_URL="${GETH_RPC_URL:-http://35.193.228.182:8080}" -GETH_WS_URL="${GETH_WS_URL:-ws://35.193.228.182:8080}" -GETH_API_KEY="${GETH_API_KEY:-df7fc1e2c686d61a26d390d00aad49f61e899c7d1cf9f97d13c8378f6819e6db}" -GETH_API_KEY_TYPE="${GETH_API_KEY_TYPE:-x-goog-api-key}" -INDEXER_START_HEIGHT="${INDEXER_START_HEIGHT:-0}" -INDEXER_KEYRING_SECRET="${INDEXER_KEYRING_SECRET:-devnet-secret}" -INDEXER_P2P_LISTEN_ADDR="${INDEXER_P2P_LISTEN_ADDR:-/ip4/0.0.0.0/tcp/9174}" - -# ---- Indexer container / ports ---------------------------------------------- -INDEXER_IMAGE="${INDEXER_IMAGE:-ghcr.io/shinzonetwork/shinzo-indexer-client:standard}" -INDEXER_NAME="${INDEXER_NAME:-shinzo-indexer}" -INDEXER_API_PORT="${INDEXER_API_PORT:-9181}" # -> container 9181 (GraphQL API) -INDEXER_P2P_PUB_PORT="${INDEXER_P2P_PUB_PORT:-9171}" # -> container 9171 (published P2P) -INDEXER_HEALTH_PORT="${INDEXER_HEALTH_PORT:-8080}" # -> container 8080 (health) -INDEXER_HEALTH_URL="${INDEXER_HEALTH_URL:-http://localhost:${INDEXER_HEALTH_PORT}}" - -# ---- Host config / container / ports ---------------------------------------- -HOST_CONFIG="${HOST_CONFIG:-$HOME/host-config.yaml}" -KEYRING_SECRET="${KEYRING_SECRET:-host-devnet-secret}" # Host DefraDB keyring secret -HOST_IMAGE="${HOST_IMAGE:-ghcr.io/shinzonetwork/shinzo-host-client:standard}" -HOST_NAME="${HOST_NAME:-shinzo-host}" -HOST_API_PORT="${HOST_API_PORT:-9182}" # -> container 9181 (GraphQL API) -HOST_P2P_PORT="${HOST_P2P_PORT:-9172}" # -> container 9171 (P2P) -HOST_HEALTH_PORT="${HOST_HEALTH_PORT:-8081}" # -> container 8080 (health) - -# How the Host container reaches the Indexer's P2P endpoint. -# Leave both empty to auto-detect from the indexer's advertised /health addresses -# (recommended). Override INDEXER_P2P_IP with a remote VM's IP for separate-VM -# setups, or INDEXER_P2P_PORT to pin the libp2p port explicitly. -INDEXER_P2P_IP="${INDEXER_P2P_IP:-}" -INDEXER_P2P_PORT="${INDEXER_P2P_PORT:-}" - -log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } -die() { printf '\033[1;31mERROR:\033[0m %s\n' "$*" >&2; exit 1; } - -# ---- Preflight -------------------------------------------------------------- -command -v docker >/dev/null || die "docker not found" -command -v curl >/dev/null || die "curl not found" -command -v jq >/dev/null || die "jq not found (install: sudo apt-get install -y jq)" - -# ---- Step 1: (re)start a fresh Indexer -------------------------------------- -if docker ps -a --format '{{.Names}}' | grep -qx "$INDEXER_NAME"; then - log "Removing existing Indexer container '${INDEXER_NAME}'" - docker rm -f "$INDEXER_NAME" >/dev/null -fi - -log "Starting Indexer container '${INDEXER_NAME}'" -docker run -d \ - --name "$INDEXER_NAME" \ - -e GETH_RPC_URL="${GETH_RPC_URL}" \ - -e GETH_WS_URL="${GETH_WS_URL}" \ - -e GETH_API_KEY="${GETH_API_KEY}" \ - -e GETH_API_KEY_TYPE="${GETH_API_KEY_TYPE}" \ - -e INDEXER_START_HEIGHT="${INDEXER_START_HEIGHT}" \ - -e DEFRADB_KEYRING_SECRET="${INDEXER_KEYRING_SECRET}" \ - -e DEFRADB_PLAYGROUND=true \ - -e DEFRADB_P2P_ENABLED=true \ - -e DEFRADB_P2P_LISTEN_ADDR="${INDEXER_P2P_LISTEN_ADDR}" \ - -e LOGGER_DEBUG=true \ - -p "${INDEXER_API_PORT}:9181" \ - -p "${INDEXER_P2P_PUB_PORT}:9171" \ - -p "${INDEXER_HEALTH_PORT}:8080" \ - "$INDEXER_IMAGE" >/dev/null - -# ---- Step 2: wait for the Indexer's P2P self-info --------------------------- -# The indexer's /health endpoint reports the Peer ID AND the real libp2p -# listen addresses. The internal P2P port is NOT necessarily the published -# 9171 (DEFRADB_P2P_LISTEN_ADDR may bind 9174), so read it from here rather -# than assuming. We also read a non-loopback container IP so the Host -# container can dial the indexer directly over the shared docker bridge. -log "Waiting for Indexer P2P self-info from ${INDEXER_HEALTH_URL} ..." -SELF_JSON="" -for i in $(seq 1 60); do - SELF_JSON="$(curl -fsS -H 'Accept: application/json' "${INDEXER_HEALTH_URL}/health" 2>/dev/null \ - | jq -c '.p2p.self // empty' 2>/dev/null || true)" - [[ -z "$SELF_JSON" ]] && SELF_JSON="$(curl -fsS "${INDEXER_HEALTH_URL}/registration" 2>/dev/null \ - | jq -c '.p2p.self // empty' 2>/dev/null || true)" - if [[ -n "$SELF_JSON" && "$(jq -r '.id // empty' <<<"$SELF_JSON")" != "" ]]; then - break - fi - sleep 2 - if [[ "$i" -eq 60 ]]; then - docker logs --tail 30 "$INDEXER_NAME" 2>&1 || true - die "Indexer did not report P2P self-info in time. Check: docker logs ${INDEXER_NAME}" - fi -done - -PEER_ID="$(jq -r '.id // empty' <<<"$SELF_JSON")" -[[ -n "$PEER_ID" ]] || die "Indexer reported no Peer ID." -log "Indexer Peer ID: ${PEER_ID}" - -# Derive the P2P port from the indexer's advertised addresses (first non-loopback, -# else first address). Allow explicit override via INDEXER_P2P_PORT. -if [[ -z "${INDEXER_P2P_PORT:-}" ]]; then - INDEXER_P2P_PORT="$(jq -r ' - ([.addresses[]? | select(test("127\\.0\\.0\\.1")|not)] + .addresses)[0] - | capture("/tcp/(?

[0-9]+)").p // empty' <<<"$SELF_JSON")" - INDEXER_P2P_PORT="${INDEXER_P2P_PORT:-9171}" -fi - -# Determine how the Host container reaches the indexer: -# - If INDEXER_P2P_IP is preset (e.g. a remote VM IP), use it. -# - Else prefer the indexer's own non-loopback container IP from its -# advertised addresses (direct container-to-container over the bridge). -# - Else fall back to the docker0 gateway. -if [[ -z "$INDEXER_P2P_IP" ]]; then - INDEXER_P2P_IP="$(jq -r ' - [.addresses[]? | capture("/ip4/(?[0-9.]+)/").ip - | select(. != "127.0.0.1" and . != "0.0.0.0")][0] // empty' <<<"$SELF_JSON")" -fi -if [[ -z "$INDEXER_P2P_IP" ]]; then - INDEXER_P2P_IP="$(ip -4 addr show docker0 2>/dev/null | grep -oP 'inet \K[\d.]+' || true)" - INDEXER_P2P_IP="${INDEXER_P2P_IP:-172.17.0.1}" -fi -log "Indexer P2P reachable at ${INDEXER_P2P_IP}:${INDEXER_P2P_PORT} (from inside the Host container)" - -BOOTSTRAP_PEER="/ip4/${INDEXER_P2P_IP}/tcp/${INDEXER_P2P_PORT}/p2p/${PEER_ID}" -log "Bootstrap peer: ${BOOTSTRAP_PEER}" - -# ---- Step 3: write the Host config ------------------------------------------ -log "Writing Host config to ${HOST_CONFIG}" -cat > "$HOST_CONFIG" </dev/null -fi - -log "Starting Host container '${HOST_NAME}'" -docker run -d \ - --name "$HOST_NAME" \ - -e BOOTSTRAP_PEERS="${BOOTSTRAP_PEER}" \ - -v "${HOST_CONFIG}:/app/config.yaml:ro" \ - -p "${HOST_API_PORT}:9181" \ - -p "${HOST_P2P_PORT}:9171" \ - -p "${HOST_HEALTH_PORT}:8080" \ - "$HOST_IMAGE" >/dev/null - -# ---- Step 5: verify --------------------------------------------------------- -log "Waiting for Host to connect to the Indexer ..." -CONNECTED=false -for i in $(seq 1 45); do - PEERS="$(curl -fsS -H 'Accept: application/json' "http://localhost:${HOST_HEALTH_PORT}/health" 2>/dev/null \ - | jq -r '[.p2p.peers[]?.id] | join(",")' 2>/dev/null || true)" - if [[ "$PEERS" == *"$PEER_ID"* ]]; then - CONNECTED=true - break - fi - sleep 2 -done - -log "Host health / P2P peers:" -curl -fsS -H 'Accept: application/json' "http://localhost:${HOST_HEALTH_PORT}/health" \ - | jq '{status, current_block, p2p: {self: .p2p.self.id, peers: [.p2p.peers[].id]}}' || true - -if [[ "$CONNECTED" == true ]]; then - log "SUCCESS: Host is connected to the Indexer over P2P." -else - log "ERROR: Host has not listed the Indexer peer yet. Check: docker logs -f ${HOST_NAME}" -fi - -cat <