Skip to content
12 changes: 11 additions & 1 deletion internal/connector/recovery_acp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,19 @@ func fakeACPAdapter(w *fakeWorker) int {
// A server that was starting when the connector died still says what
// became of its token: the parent checks that every worker either took
// one or said why it could not, and a process that exited with the bind
// still in flight would answer neither.
// still in flight would answer neither. Nor would one whose handshake
// never reached the read-back — a connector that died first, a session
// it ended on the way — so that exit says so too: no server started, no
// token was asked for. The spawn fakes bind before they speak and so
// always say a word; this one speaks last, and must not leave without.
said := false
defer func() {
if starting {
<-bound
return
}
if !said {
w.log(0, 0, "bind-failed: the session ended before its MCP servers started, so no token was asked for")
}
}()
awaitBind := func() error {
Expand Down Expand Up @@ -205,6 +214,7 @@ func fakeACPAdapter(w *fakeWorker) int {
reported = "bypassPermissions"
w.log(0, 0, "bad-mode")
w.log(0, 0, "bind-failed: the session ended in its handshake, before its MCP servers started")
said = true
}
notify("session/update", map[string]any{"sessionId": sessionID,
"update": map[string]any{"sessionUpdate": "current_mode_update", "currentModeId": reported}})
Expand Down
2 changes: 1 addition & 1 deletion internal/connector/recovery_connector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ func runHarnessConnector(dir string) error {
return err
}
outbox, err := NewOutbox(OutboxOptions{
Ledger: ledger, Poster: storePoster{dir: dir, kill: kill},
Ledger: ledger, Poster: storePoster{dir: dir, kill: kill, fault: os.Getenv(harnessFaultEnv)},
Paused: ledger.Held, Lines: lines, Logger: logger,
// A sending intent a previous process left is reconciled once it is
// this old, so a restart settles it rather than waiting out the
Expand Down
22 changes: 21 additions & 1 deletion internal/connector/recovery_dispatch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,8 +275,11 @@ func recordedAttempts(t *testing.T, l *Ledger) []recordedAttempt {
)
require.NoError(t, rows.Scan(&a.id, &a.state, &a.process.PID, &a.process.PGID, &started))
if started.Valid {
// The ledger writes the stamp only when it is the kernel's
// (AttemptProcess.startedStamp), so one read back is exact.
a.process.StartedAt, err = parseStamp(started.String)
require.NoError(t, err)
a.process.StartedExact = true
}
out = append(out, a)
}
Expand Down Expand Up @@ -614,12 +617,29 @@ func TestRecoveryTheGuardAcknowledgementIsPostedAtMostOnce(t *testing.T) {
t.Run(row.name, func(t *testing.T) {
// A worker that never calls get_dispatch is what the guard is
// for: the acknowledgement falls to the connector.
//
// The guard is due a delay after admission, and the rows
// kill the connector as it posts. That kill must land on an
// attempt already running — a worker recorded, so a restart
// can end it and settle — and the launch it races (a working
// directory, the ledger, the token socket, the agent's
// wrapper and, for the acp row, its whole handshake) takes
// what the machine gives it: on a loaded CI runner, more than
// the guard's 50ms, and the attempt was left launching with
// no worker to identify, held by the restart rather than
// settled, as it must be. So the fake Basecamp holds the
// acknowledgement until the connector has written its running
// line ("guard-after-running"): the ordering is made, not
// waited for, and the guard's period can stay short. The
// check after the kill is what that ordering promises.
h := newHarness(t, d, harnessScenario{
GuardDelay: 50 * time.Millisecond,
Plans: map[string][]string{"101#1": {"linger"}},
})
h.publish(feedEntry{Event: todoEvent(101, 5001)})
h.run(harnessRun{Kill: row.kill, Killed: true})
h.run(harnessRun{Kill: row.kill, Killed: true, Fault: "guard-after-running"})
require.NotEmpty(t, recordedWorkers(t, h.ledger()),
"the kill landed on an attempt still launching: the acknowledgement was posted before the running line")
h.run(harnessRun{})
h.run(harnessRun{})

Expand Down
34 changes: 34 additions & 0 deletions internal/connector/recovery_fakes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -508,9 +508,28 @@ func (h *harness) connectorPosts() []storedMessage {
type storePoster struct {
dir string
kill *killSpec
// fault is the run's standing misbehavior (harnessRun.Fault):
// "guard-after-running" holds a guard acknowledgement until the
// connector has written its running line for an attempt.
fault string
}

func (p storePoster) Post(ctx context.Context, dest Destination, body string) (int64, error) {
if p.fault == "guard-after-running" && body == GuardAckBody {
// The guard is due a delay after admission, whether or not the
// launch it races has finished; a row that kills the connector as
// the acknowledgement is posted needs the kill to land on an
// attempt already running, with a worker recorded, or the restart
// holds an attempt it cannot identify. Basecamp is the one party
// that can wait for that without a seam in the connector: the
// running line is this process's own, written to the harness's
// file once MarkRunning has committed, and the post is made
// outside any transaction of the ledger's, so nothing the launch
// needs is held while it waits.
if err := waitFor(ctx, func() (bool, error) { return runningLineWritten(p.dir) }); err != nil {
return 0, fmt.Errorf("guard-after-running: no attempt was recorded running: %w", err)
}
}
if p.kill.at("post-before") {
die()
}
Expand All @@ -524,6 +543,21 @@ func (p storePoster) Post(ctx context.Context, dest Destination, body string) (i
return id, nil
}

// runningLineWritten reports whether the connector in dir has written a
// running line for any attempt: the line follows MarkRunning's commit, so a
// worker is recorded for it by then.
func runningLineWritten(dir string) (bool, error) {
running := false
err := readJSONLines(filepath.Join(dir, linesFile), func(line []byte) error {
var l DispatchLine
if json.Unmarshal(line, &l) == nil && l.Type == "dispatch" && l.State == string(AttemptRunning) {
running = true
}
return nil
})
return running, err
}

func (p storePoster) List(_ context.Context, dest Destination, since time.Time) ([]PostedMessage, error) {
all, err := storedMessages(p.dir)
if err != nil {
Expand Down
Loading
Loading