diff --git a/scripts/devnet/soak-check.Tests.ps1 b/scripts/devnet/soak-check.Tests.ps1 new file mode 100644 index 0000000..bfd341c --- /dev/null +++ b/scripts/devnet/soak-check.Tests.ps1 @@ -0,0 +1,114 @@ +# Focused tests for the soak sampler's classification-spread confirmation +# logic (Resolve-ClassificationSpread) and its interaction with the shared +# Get-ConvergenceClassification. No external test framework is used; the +# function under test is extracted from soak-check.ps1 via the PowerShell +# parser so the script's normal entry point is never executed. +# +# Usage: powershell -NoProfile -ExecutionPolicy Bypass -File .\soak-check.Tests.ps1 + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Get-ConvergenceClassification (shared, unmodified) comes from config. +. (Join-Path $PSScriptRoot 'devnet-config.ps1') + +# Define Resolve-ClassificationSpread by extracting it from soak-check.ps1 +# without running the script. +$src = Get-Content (Join-Path $PSScriptRoot 'soak-check.ps1') -Raw +$ast = [System.Management.Automation.Language.Parser]::ParseInput($src, [ref]$null, [ref]$null) +$fn = $ast.FindAll({ + param($n) + ($n -is [System.Management.Automation.Language.FunctionDefinitionAst]) -and + ($n.Name -eq 'Resolve-ClassificationSpread') + }, $true) | Select-Object -First 1 +if ($null -eq $fn) { throw 'Resolve-ClassificationSpread not found in soak-check.ps1' } +Invoke-Expression $fn.Extent.Text + +$fail = 0 +function Assert-Eq($label, $expected, $actual) { + if ("$expected" -eq "$actual") { + Write-Host " PASS $label (= $actual)" + } else { + Write-Host " FAIL $label (expected $expected, got $actual)" + $script:fail++ + } +} + +# Height readers record invocations in a shared hashtable ($Probe is a +# reference, so increments made inside the closure remain visible). +function New-Reader([object[]]$Heights, [hashtable]$Probe) { + return { + $Probe.calls++ + return $Heights + }.GetNewClosure() +} +function New-ThrowingReader([hashtable]$Probe) { + return { + $Probe.calls++ + throw 'RPC read failed' + }.GetNewClosure() +} +$probe = @{ calls = 0 } + +# Full pipeline: resolve the classification spread, then classify, exactly +# as the sampler does. DelaySeconds 0 keeps the tests fast. +function Classify($rawSpread, $allReachable, $tipsConsistent, $reader, $producerDelta) { + $cs = Resolve-ClassificationSpread -RawSpread $rawSpread -AllReachable $allReachable ` + -TipsConsistent $tipsConsistent -ExpectedNodeCount 3 -HeightReader $reader -DelaySeconds 0 + $class = Get-ConvergenceClassification -AllReachable $allReachable -HeightSpread $cs ` + -TipsConsistent $tipsConsistent -ProducerDelta $producerDelta -SkewAllowance 1 + return @{ spread = $cs; class = $class } +} + +Write-Host 'Resolve-ClassificationSpread tests:' + +# a. raw spread 1, re-read confirms 0 -> classification spread 0 -> converged +$probe.calls = 0 +$r = Classify 1 $true $true (New-Reader @(100, 100, 100) $probe) 5 +Assert-Eq 'a. re-read 0 => classificationSpread 0' 0 $r.spread +Assert-Eq 'a. re-read 0 => converged' 'converged' $r.class +Assert-Eq 'a. reader was consulted once' 1 $probe.calls + +# b. raw spread 1, re-read still 1 -> keep 1 -> temporarily-skewed +$probe.calls = 0 +$r = Classify 1 $true $true (New-Reader @(100, 100, 101) $probe) 5 +Assert-Eq 'b. re-read 1 => classificationSpread 1' 1 $r.spread +Assert-Eq 'b. re-read 1 => temporarily-skewed' 'temporarily-skewed' $r.class + +# c. raw spread 1, confirmation read fails -> keep raw spread +$probe.calls = 0 +$r = Classify 1 $true $true (New-ThrowingReader $probe) 5 +Assert-Eq 'c. failed re-read => keeps raw spread 1' 1 $r.spread +Assert-Eq 'c. failed re-read => temporarily-skewed' 'temporarily-skewed' $r.class + +# c2. wrong number of heights returned -> keep raw spread +$probe.calls = 0 +$r = Classify 1 $true $true (New-Reader @(100, 100) $probe) 5 +Assert-Eq 'c2. wrong height count => keeps raw spread 1' 1 $r.spread + +# d. ancestry divergence (tips inconsistent) -> no re-read, stays divergent +$probe.calls = 0 +$r = Classify 1 $true $false (New-Reader @(100, 100, 100) $probe) 5 +Assert-Eq 'd. inconsistent tips => no re-read' 0 $probe.calls +Assert-Eq 'd. inconsistent tips => divergent' 'divergent' $r.class + +# e. raw spread > 1 -> no special re-read, stays divergent +$probe.calls = 0 +$r = Classify 2 $true $true (New-Reader @(100, 100, 100) $probe) 5 +Assert-Eq 'e. spread 2 => no re-read' 0 $probe.calls +Assert-Eq 'e. spread 2 => classificationSpread 2' 2 $r.spread +Assert-Eq 'e. spread 2 => divergent' 'divergent' $r.class + +# f. unreachable node -> no re-read, unreachable regardless of spread +$probe.calls = 0 +$r = Classify 1 $false $true (New-Reader @(100, 100, 100) $probe) 5 +Assert-Eq 'f. unreachable => no re-read' 0 $probe.calls +Assert-Eq 'f. unreachable => unreachable' 'unreachable' $r.class + +Write-Host '' +if ($fail -eq 0) { + Write-Host 'ALL RESOLVE-CLASSIFICATION-SPREAD TESTS PASSED' + exit 0 +} +Write-Host "$fail TEST(S) FAILED" +exit 1 diff --git a/scripts/devnet/soak-check.ps1 b/scripts/devnet/soak-check.ps1 index 9a45f94..663e454 100644 --- a/scripts/devnet/soak-check.ps1 +++ b/scripts/devnet/soak-check.ps1 @@ -92,6 +92,44 @@ function Get-BlockJson([int]$Port, [long]$Height) { } catch { return $null } } +# Resolves the spread used ONLY for convergence classification, leaving +# the raw observed spread (recorded in the CSV session row) untouched. +# +# The three nodes are read sequentially over RPC, so a block boundary +# crossed mid-read can make one node appear exactly one block ahead of +# another even though the cluster is fully converged. When everything is +# otherwise healthy (all nodes reachable, ancestry consistent) and the +# raw spread is exactly one block, this waits briefly and re-reads ONLY +# the three heights: a confirmation spread of 0 means the earlier gap was +# a non-atomic-read artifact and 0 is used for classification. If the +# gap persists, a re-read fails, or the wrong number of heights is +# returned, the raw spread is kept so genuine skew/divergence is never +# hidden. This never touches producerDelta, heightDelta, or any raw +# per-node metric. +function Resolve-ClassificationSpread { + param( + [long]$RawSpread, + [bool]$AllReachable, + [bool]$TipsConsistent, + [int]$ExpectedNodeCount, + [scriptblock]$HeightReader, + [int]$DelaySeconds = 1 + ) + $classificationSpread = $RawSpread + if ($AllReachable -and $TipsConsistent -and ($RawSpread -eq 1)) { + if ($DelaySeconds -gt 0) { Start-Sleep -Seconds $DelaySeconds } + $confirm = $null + try { $confirm = @(& $HeightReader) } catch { $confirm = $null } + if (($null -ne $confirm) -and (@($confirm).Count -eq $ExpectedNodeCount)) { + $vals = @($confirm | ForEach-Object { [long]$_ }) + $cSpread = [long]($vals | Measure-Object -Maximum).Maximum - + [long]($vals | Measure-Object -Minimum).Minimum + if ($cSpread -eq 0) { $classificationSpread = 0 } + } + } + return $classificationSpread +} + # --- One sample -------------------------------------------------------- function Invoke-SoakSample { $now = (Get-Date).ToUniversalTime() @@ -239,8 +277,22 @@ function Invoke-SoakSample { } } + # The raw spread above stays in the CSV as the observed measurement. + # For classification only, confirm a 1-block spread with a brief + # re-read of just the heights, to absorb non-atomic sequential RPC + # reads (a block boundary crossed while polling the three nodes). + $rawSpreadNum = $(if ("$spread" -eq '') { 0 } else { [long]$spread }) + $classificationSpread = Resolve-ClassificationSpread ` + -RawSpread $rawSpreadNum -AllReachable $allReachable -TipsConsistent $tipsConsistent ` + -ExpectedNodeCount $DevnetNodes.Count ` + -HeightReader { + $h = @() + foreach ($n in $DevnetNodes) { $h += [long](Get-DevnetHeight $n.Rpc) } + $h + } + $classification = Get-ConvergenceClassification -AllReachable $allReachable ` - -HeightSpread $(if ("$spread" -eq '') { 0 } else { [long]$spread }) ` + -HeightSpread $classificationSpread ` -TipsConsistent $tipsConsistent -ProducerDelta $producerDelta ` -SkewAllowance ([int]$session.thresholds.ConvergenceSkewBlocks)