diff --git a/bin/gravity_node/src/cli.rs b/bin/gravity_node/src/cli.rs index f8f0fd15..170fb904 100644 --- a/bin/gravity_node/src/cli.rs +++ b/bin/gravity_node/src/cli.rs @@ -155,10 +155,17 @@ impl, Ext: clap::Args + fmt::Debug> Cl self.logs.log_file_directory = self.logs.log_file_directory.join(self.chain.chain.to_string()); + // greth keeps the node-subcommand default (5 files) outside LogArgs, so `--log.file.*` + // are otherwise accepted while the file layer is never installed (see its app.rs). + if matches!(self.command, Commands::Node(_)) { + self.logs.apply_node_defaults(); + } + let _guard = self.init_tracing()?; debug!(target: "reth::cli", "Initialized tracing, log directory: {}, log level {:?}", self.logs.log_file_directory, self.logs.verbosity); let runner = CliRunner::try_default_runtime()?; + // reth v2.3.0: init/init-state execute() take a Runtime. let runtime = runner.runtime(); let components = |spec: Arc| { (EthEvmConfig::ethereum(spec.clone()), Arc::new(EthBeaconConsensus::new(spec))) diff --git a/cluster/deploy.sh b/cluster/deploy.sh index 5e258758..b224d8c7 100755 --- a/cluster/deploy.sh +++ b/cluster/deploy.sh @@ -449,14 +449,44 @@ START_SCRIPT SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKSPACE="$SCRIPT_DIR/.." +# Wait until the kernel has reaped the process. greth v2.3+ can keep the +# RocksDB LOCK held for several seconds after SIGTERM while flushing; if we +# delete the pid file and return early, e2e restart races the lock and dies +# with "Resource temporarily unavailable". +wait_pid_gone() { + local pid="$1" + local max_iters="$2" + local i + for i in $(seq 1 "$max_iters"); do + if ! kill -0 "$pid" 2>/dev/null; then + return 0 + fi + sleep 0.5 + done + return 1 +} + if [ -e "${WORKSPACE}/script/node.pid" ]; then pid=$(cat "${WORKSPACE}/script/node.pid") if kill -0 "$pid" 2>/dev/null; then - kill "$pid" - echo "Stopped node (PID: $pid)" + kill "$pid" 2>/dev/null || true + # ~30s graceful (SIGTERM) + if wait_pid_gone "$pid" 60; then + echo "Stopped node (PID: $pid)" + else + echo "Node (PID: $pid) still alive after SIGTERM; sending SIGKILL" + kill -9 "$pid" 2>/dev/null || true + # ~20s after SIGKILL for process to disappear + if wait_pid_gone "$pid" 40; then + echo "Stopped node (PID: $pid, forced)" + else + echo "ERROR: node PID $pid still alive after SIGKILL" >&2 + fi + fi else echo "Node not running (stale PID file)" fi + # Only drop the pid file after the process is gone (or we gave up). rm -f "${WORKSPACE}/script/node.pid" else echo "No PID file found" @@ -573,14 +603,44 @@ START_SCRIPT SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKSPACE="$SCRIPT_DIR/.." +# Wait until the kernel has reaped the process. greth v2.3+ can keep the +# RocksDB LOCK held for several seconds after SIGTERM while flushing; if we +# delete the pid file and return early, e2e restart races the lock and dies +# with "Resource temporarily unavailable". +wait_pid_gone() { + local pid="$1" + local max_iters="$2" + local i + for i in $(seq 1 "$max_iters"); do + if ! kill -0 "$pid" 2>/dev/null; then + return 0 + fi + sleep 0.5 + done + return 1 +} + if [ -e "${WORKSPACE}/script/node.pid" ]; then pid=$(cat "${WORKSPACE}/script/node.pid") if kill -0 "$pid" 2>/dev/null; then - kill "$pid" - echo "Stopped node (PID: $pid)" + kill "$pid" 2>/dev/null || true + # ~30s graceful (SIGTERM) + if wait_pid_gone "$pid" 60; then + echo "Stopped node (PID: $pid)" + else + echo "Node (PID: $pid) still alive after SIGTERM; sending SIGKILL" + kill -9 "$pid" 2>/dev/null || true + # ~20s after SIGKILL for process to disappear + if wait_pid_gone "$pid" 40; then + echo "Stopped node (PID: $pid, forced)" + else + echo "ERROR: node PID $pid still alive after SIGKILL" >&2 + fi + fi else echo "Node not running (stale PID file)" fi + # Only drop the pid file after the process is gone (or we gave up). rm -f "${WORKSPACE}/script/node.pid" else echo "No PID file found" @@ -706,14 +766,44 @@ START_SCRIPT SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKSPACE="$SCRIPT_DIR/.." +# Wait until the kernel has reaped the process. greth v2.3+ can keep the +# RocksDB LOCK held for several seconds after SIGTERM while flushing; if we +# delete the pid file and return early, e2e restart races the lock and dies +# with "Resource temporarily unavailable". +wait_pid_gone() { + local pid="$1" + local max_iters="$2" + local i + for i in $(seq 1 "$max_iters"); do + if ! kill -0 "$pid" 2>/dev/null; then + return 0 + fi + sleep 0.5 + done + return 1 +} + if [ -e "${WORKSPACE}/script/node.pid" ]; then pid=$(cat "${WORKSPACE}/script/node.pid") if kill -0 "$pid" 2>/dev/null; then - kill "$pid" - echo "Stopped node (PID: $pid)" + kill "$pid" 2>/dev/null || true + # ~30s graceful (SIGTERM) + if wait_pid_gone "$pid" 60; then + echo "Stopped node (PID: $pid)" + else + echo "Node (PID: $pid) still alive after SIGTERM; sending SIGKILL" + kill -9 "$pid" 2>/dev/null || true + # ~20s after SIGKILL for process to disappear + if wait_pid_gone "$pid" 40; then + echo "Stopped node (PID: $pid, forced)" + else + echo "ERROR: node PID $pid still alive after SIGKILL" >&2 + fi + fi else echo "Node not running (stale PID file)" fi + # Only drop the pid file after the process is gone (or we gave up). rm -f "${WORKSPACE}/script/node.pid" else echo "No PID file found" diff --git a/cluster/stop.sh b/cluster/stop.sh index 362d49bf..11817d0a 100755 --- a/cluster/stop.sh +++ b/cluster/stop.sh @@ -113,9 +113,13 @@ stop_node() { log_info "Stopping $node_id (PID: $pid)..." kill "$pid" 2>/dev/null || true - - # Wait for graceful shutdown - for i in {1..10}; do + + # Wait until the process is really gone before dropping the pid file. + # greth v2.3+ can hold the RocksDB LOCK for several seconds after SIGTERM + # while flushing; returning early races the next start + # ("Resource temporarily unavailable" on .../db/state/LOCK). + # ~30s graceful + for i in {1..60}; do if ! kill -0 "$pid" 2>/dev/null; then rm -f "$pid_file" log_info "$node_id stopped" @@ -123,12 +127,22 @@ stop_node() { fi sleep 0.5 done - - # Force kill if still running + + # Force kill if still running, then wait again for the kernel to reap it. log_warn "$node_id: Force killing..." kill -9 "$pid" 2>/dev/null || true + for i in {1..40}; do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$pid_file" + log_info "$node_id stopped (forced)" + return 0 + fi + sleep 0.5 + done + + # Last resort: drop pid bookkeeping so callers are not stuck, but warn. rm -f "$pid_file" - log_info "$node_id stopped (forced)" + log_error "$node_id: PID $pid still alive after SIGKILL" } # Main diff --git a/gravity_e2e/cluster_test_cases/prague/genesis.toml b/gravity_e2e/cluster_test_cases/prague/genesis.toml index 978ac7cf..65f20dc0 100644 --- a/gravity_e2e/cluster_test_cases/prague/genesis.toml +++ b/gravity_e2e/cluster_test_cases/prague/genesis.toml @@ -3,6 +3,11 @@ # !! pragueTime must be > genesis_timestamp_secs so the EIP-2935 deployment # hook (pipe-exec eip_2935.rs gate parent_ts < pragueTime <= current_ts) # fires on block 1. Change one, change the other. +# +# !! betaTime (gravity-reth #412): until Beta, filter_invalid_txs wholesale-rejects +# type-4 and from/to-delegated traffic (EIP-7702 emergency lockdown). Missing +# betaTime → lockdown forever (fail-closed). This suite exercises Prague 7702 +# behaviour, so release lockdown at the same timestamp as pragueTime. [dependencies.genesis_contracts] repo = "https://github.com/Galxe/gravity_chain_core_contracts.git" @@ -29,6 +34,8 @@ initial_locked_until_micros = 1798848000000000 [genesis.hardforks] pragueTime = 1775664001 +# Same wall-clock as pragueTime: Prague protocol + Beta lockdown release together. +betaTime = 1775664001 [genesis.faucet] address = "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" diff --git a/gravity_e2e/gravity_e2e/cluster/node.py b/gravity_e2e/gravity_e2e/cluster/node.py index d0e409f2..9371863b 100644 --- a/gravity_e2e/gravity_e2e/cluster/node.py +++ b/gravity_e2e/gravity_e2e/cluster/node.py @@ -274,6 +274,13 @@ async def stop(self) -> bool: """ Stop this individual node. Returns True if node is now STOPPED. + + Important: ``NodeState.STOPPED`` from get_state() only means the PID + file is gone or the process is not reachable via that file. The + generated stop.sh used to delete the pid file right after SIGTERM + while gravity_node was still flushing RocksDB. We capture the PID + *before* stop.sh runs and wait until the OS reports it gone so the + next start does not race the RocksDB LOCK. """ if not self.stop_script.exists(): LOG.warning( @@ -288,6 +295,14 @@ async def stop(self) -> bool: LOG.info(f"Node {self.id} is already stopped.") return True + # Capture PID before stop.sh deletes the pid file. + pid: Optional[int] = None + if self.pid_file.exists(): + try: + pid = int(self.pid_file.read_text().strip()) + except ValueError: + pid = None + LOG.info(f"Stopping node {self.id}...") try: @@ -304,9 +319,16 @@ async def stop(self) -> bool: LOG.error(f"Node {self.id} stop script failed: {stderr.decode()}") return False - # Verify stopped - await asyncio.sleep(1) - # Re-check live state + # Belt-and-suspenders: wait for the real process to exit even if + # stop.sh returned early (older generated scripts, or race). + if pid is not None: + if not await self._wait_for_pid_exit(pid, timeout=50.0): + LOG.error( + f"Node {self.id}: PID {pid} still alive after stop script; " + f"restart would race RocksDB LOCK" + ) + return False + final_state, _ = await self.get_state() if final_state == NodeState.STOPPED: @@ -322,11 +344,29 @@ async def stop(self) -> bool: LOG.error(f"Exception stopping node {self.id}: {e}") return False + async def _wait_for_pid_exit(self, pid: int, timeout: float = 50.0) -> bool: + """Wait until ``pid`` is no longer alive (ProcessLookupError on kill 0).""" + import os + import time + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + except PermissionError: + # Process exists but not owned by us — treat as still alive. + pass + await asyncio.sleep(0.25) + return False + async def restart(self) -> bool: """Bounce the node.""" if not await self.stop(): return False - await asyncio.sleep(2) # Grace period + # stop() already waits for process exit / RocksDB unlock; short settle only. + await asyncio.sleep(0.5) return await self.start() def is_running(self) -> bool: