diff --git a/.surface b/.surface index 2df0a4638..e496ffbf1 100644 --- a/.surface +++ b/.surface @@ -671,6 +671,9 @@ CMD basecamp config untrust CMD basecamp connect CMD basecamp connect setup CMD basecamp connect show +CMD basecamp connect worktrees +CMD basecamp connect worktrees list +CMD basecamp connect worktrees prune CMD basecamp docs CMD basecamp docs archive CMD basecamp docs doc @@ -5425,6 +5428,72 @@ FLAG basecamp connect show --stats type=bool FLAG basecamp connect show --styled type=bool FLAG basecamp connect show --todolist type=string FLAG basecamp connect show --verbose type=count +FLAG basecamp connect worktrees --account type=string +FLAG basecamp connect worktrees --agent type=bool +FLAG basecamp connect worktrees --cache-dir type=string +FLAG basecamp connect worktrees --count type=bool +FLAG basecamp connect worktrees --help type=bool +FLAG basecamp connect worktrees --hints type=bool +FLAG basecamp connect worktrees --ids-only type=bool +FLAG basecamp connect worktrees --in type=string +FLAG basecamp connect worktrees --jq type=string +FLAG basecamp connect worktrees --json type=bool +FLAG basecamp connect worktrees --markdown type=bool +FLAG basecamp connect worktrees --md type=bool +FLAG basecamp connect worktrees --no-hints type=bool +FLAG basecamp connect worktrees --no-stats type=bool +FLAG basecamp connect worktrees --profile type=string +FLAG basecamp connect worktrees --project type=string +FLAG basecamp connect worktrees --quiet type=bool +FLAG basecamp connect worktrees --stats type=bool +FLAG basecamp connect worktrees --styled type=bool +FLAG basecamp connect worktrees --todolist type=string +FLAG basecamp connect worktrees --verbose type=count +FLAG basecamp connect worktrees list --account type=string +FLAG basecamp connect worktrees list --agent type=bool +FLAG basecamp connect worktrees list --cache-dir type=string +FLAG basecamp connect worktrees list --count type=bool +FLAG basecamp connect worktrees list --help type=bool +FLAG basecamp connect worktrees list --hints type=bool +FLAG basecamp connect worktrees list --ids-only type=bool +FLAG basecamp connect worktrees list --in type=string +FLAG basecamp connect worktrees list --jq type=string +FLAG basecamp connect worktrees list --json type=bool +FLAG basecamp connect worktrees list --markdown type=bool +FLAG basecamp connect worktrees list --md type=bool +FLAG basecamp connect worktrees list --no-hints type=bool +FLAG basecamp connect worktrees list --no-stats type=bool +FLAG basecamp connect worktrees list --profile type=string +FLAG basecamp connect worktrees list --project type=string +FLAG basecamp connect worktrees list --quiet type=bool +FLAG basecamp connect worktrees list --shadow type=bool +FLAG basecamp connect worktrees list --stats type=bool +FLAG basecamp connect worktrees list --styled type=bool +FLAG basecamp connect worktrees list --todolist type=string +FLAG basecamp connect worktrees list --verbose type=count +FLAG basecamp connect worktrees prune --account type=string +FLAG basecamp connect worktrees prune --agent type=bool +FLAG basecamp connect worktrees prune --cache-dir type=string +FLAG basecamp connect worktrees prune --count type=bool +FLAG basecamp connect worktrees prune --force type=stringArray +FLAG basecamp connect worktrees prune --help type=bool +FLAG basecamp connect worktrees prune --hints type=bool +FLAG basecamp connect worktrees prune --ids-only type=bool +FLAG basecamp connect worktrees prune --in type=string +FLAG basecamp connect worktrees prune --jq type=string +FLAG basecamp connect worktrees prune --json type=bool +FLAG basecamp connect worktrees prune --markdown type=bool +FLAG basecamp connect worktrees prune --md type=bool +FLAG basecamp connect worktrees prune --no-hints type=bool +FLAG basecamp connect worktrees prune --no-stats type=bool +FLAG basecamp connect worktrees prune --profile type=string +FLAG basecamp connect worktrees prune --project type=string +FLAG basecamp connect worktrees prune --quiet type=bool +FLAG basecamp connect worktrees prune --shadow type=bool +FLAG basecamp connect worktrees prune --stats type=bool +FLAG basecamp connect worktrees prune --styled type=bool +FLAG basecamp connect worktrees prune --todolist type=string +FLAG basecamp connect worktrees prune --verbose type=count FLAG basecamp docs --account type=string FLAG basecamp docs --agent type=bool FLAG basecamp docs --cache-dir type=string @@ -18606,6 +18675,9 @@ SUB basecamp config untrust SUB basecamp connect SUB basecamp connect setup SUB basecamp connect show +SUB basecamp connect worktrees +SUB basecamp connect worktrees list +SUB basecamp connect worktrees prune SUB basecamp docs SUB basecamp docs archive SUB basecamp docs doc diff --git a/e2e/smoke/smoke_lifecycle.bats b/e2e/smoke/smoke_lifecycle.bats index df00a6567..ef0daed8c 100644 --- a/e2e/smoke/smoke_lifecycle.bats +++ b/e2e/smoke/smoke_lifecycle.bats @@ -24,6 +24,14 @@ load smoke_helper mark_out_of_scope "Reads the connector policy a connected profile's setup wrote — covered by Go tests in internal/commands" } +@test "connect worktrees list is out of scope" { + mark_out_of_scope "Reads a local connector's ledger — covered by Go tests in internal/commands and internal/connector" +} + +@test "connect worktrees prune is out of scope" { + mark_out_of_scope "Removes local git worktrees a connector kept — covered by Go tests in internal/commands and internal/connector" +} + @test "auth refresh is out of scope" { mark_out_of_scope "Requires OAuth credentials" } diff --git a/internal/commands/commands.go b/internal/commands/commands.go index 8c2c75b7a..285f291c8 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -146,7 +146,7 @@ func CommandCategories() []CommandCategory { {Name: "bonfire", Category: "additional", Description: "Multi-chat orchestration", Actions: []string{"split", "layout"}, Experimental: true, DevOnly: true}, {Name: "api", Category: "additional", Description: "Raw API access"}, {Name: "mcp", Category: "additional", Description: "Serve Basecamp to MCP clients over stdio"}, - {Name: "connect", Category: "additional", Description: "Set up a local agent connector for a Basecamp agent", Actions: []string{"setup", "show"}}, + {Name: "connect", Category: "additional", Description: "Set up a local agent connector for a Basecamp agent", Actions: []string{"setup", "show", "worktrees"}}, {Name: "help", Category: "additional", Description: "Show help"}, {Name: "version", Category: "additional", Description: "Show version"}, }, diff --git a/internal/commands/connect.go b/internal/commands/connect.go index 6d8afbd31..280d882f8 100644 --- a/internal/commands/connect.go +++ b/internal/commands/connect.go @@ -66,6 +66,7 @@ isolated state directory and dispatches nothing. macOS and Linux only.`, cmd.AddCommand(newConnectSetupCmd()) cmd.AddCommand(newConnectWorkerMCPCmd()) cmd.AddCommand(newConnectShowCmd()) + cmd.AddCommand(newConnectWorktreesCmd()) return cmd } @@ -180,7 +181,7 @@ func connectShowDisplay(path string, f setup.File, markdown bool) map[string]any "agent": agent, "operator": fmt.Sprintf("person %d", f.Trust.OperatorID), "trust": trust, - "workers": fmt.Sprintf("%s, concurrency %d, deadline %s, worktrees %s", f.Driver, f.Concurrency, time.Duration(f.Deadline), worktrees), + "workers": fmt.Sprintf("%s %s, concurrency %d, deadline %s, worktrees %s", f.Driver, f.WorkerName(), f.Concurrency, time.Duration(f.Deadline), worktrees), "projects": strconv.Itoa(len(f.Projects)) + " routed", } for id, r := range f.Projects { diff --git a/internal/commands/connect_run.go b/internal/commands/connect_run.go index 07b58fbf1..b0938e557 100644 --- a/internal/commands/connect_run.go +++ b/internal/commands/connect_run.go @@ -88,13 +88,29 @@ func connectStateDir(file setup.File, shadow bool) (string, error) { if err != nil { return "", err } - group := "connect" + group, dir := connectStateParts(file, shadow) + return ensurePrivateChain(stateHome, "basecamp", group, dir) +} + +// connectStateDirPath is the same directory, named and not created: what +// reads a connector's state resolves. +func connectStateDirPath(file setup.File, shadow bool) (string, error) { + stateHome, err := connectStateHome() + if err != nil { + return "", err + } + group, dir := connectStateParts(file, shadow) + return filepath.Join(stateHome, "basecamp", group, dir), nil +} + +func connectStateParts(file setup.File, shadow bool) (group, dir string) { + group = "connect" if shadow { // An isolated ledger, lock and checkpoint: a shadow never shares a // position or a record with the connector it watches beside. group = "connect-shadow" } - return ensurePrivateChain(stateHome, "basecamp", group, connector.StateDirName(file.AccountID, file.Agent.PersonID)) + return group, connector.StateDirName(file.AccountID, file.Agent.PersonID) } // connectSessionsDir is where a session's short-lived files go — the MCP @@ -156,11 +172,6 @@ func runConnect(cmd *cobra.Command, f *connectRunFlags) error { case err != nil: return output.ErrUsage("connect.json cannot be used: " + err.Error()) } - if file.Worktrees && !f.shadow { - // Refused rather than ignored: workers would share the route's - // checkout while connect.json says each task gets its own. - return output.ErrUsage("connect.json asks for worktrees, which this basecamp does not support yet; run setup with --worktrees=false") - } driverName := file.Driver if f.driver != "" { driverName = f.driver @@ -274,12 +285,27 @@ func runConnect(cmd *cobra.Command, f *connectRunFlags) error { if err != nil { return output.ErrUsage(err.Error()) } - dispatcher, err = connector.NewDispatcher(connectDispatcherOptions(connectDispatch{ + // Built with worktrees off too, so the ones made while they were on + // are still settled and recovered. + worktreesRoot, err := ensurePrivateChain(stateDir, connectWorktreesDir) + if err != nil { + return err + } + workspaces, err := connector.NewWorktrees(connector.WorktreesOptions{ + Ledger: ledger, Root: worktreesRoot, Logger: logger, Off: !file.Worktrees, + Redaction: driver.Redaction{Dirs: []string{stateDir}}, + }) + if err != nil { + return err + } + options := connectDispatcherOptions(connectDispatch{ File: file, Buckets: buckets, Ledger: ledger, Driver: worker, Routes: routes.Current, Profile: name, Executable: exe, StateDir: stateDir, SessionsDir: sessions, Replies: connector.SDKReplies{Client: accountClient, AgentID: agentID}, Lines: lines, Logger: logger, - })) + }) + options.Workspaces = workspaces + dispatcher, err = connector.NewDispatcher(options) if err != nil { return err } diff --git a/internal/commands/connect_worktrees.go b/internal/commands/connect_worktrees.go new file mode 100644 index 000000000..b2ae0ee69 --- /dev/null +++ b/internal/commands/connect_worktrees.go @@ -0,0 +1,332 @@ +package commands + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" + + "github.com/basecamp/basecamp-cli/internal/appctx" + "github.com/basecamp/basecamp-cli/internal/config" + "github.com/basecamp/basecamp-cli/internal/connector" + "github.com/basecamp/basecamp-cli/internal/connector/setup" + "github.com/basecamp/basecamp-cli/internal/output" +) + +// connectWorktreesDir is where a connector's task worktrees live, under its +// state directory. +const connectWorktreesDir = "worktrees" + +func newConnectWorktreesCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "worktrees", + Short: "List and prune the git worktrees the connector kept", + Long: `With worktrees on (connect setup --worktrees), each task works in a git +worktree of its own, on a basecamp-connect/ branch. The connector never +removes one: when the task ends its worktree is kept and listed here, with +the task it was for and what it takes up on disk. You remove them with prune, +which goes by what could be lost — nothing on the disk but the files git +tracks, unchanged, no merge or rebase in progress, not locked, and every +commit it reaches held elsewhere — and keeps what could. + +They add up: every task leaves one, so prune is part of running a connector +with worktrees on. A Codex worker cannot commit — a worktree's git data is +outside the directory its sandbox may write — so with Codex every task that +edits anything leaves a worktree with work in it.`, + } + cmd.AddCommand(newConnectWorktreesListCmd(), newConnectWorktreesPruneCmd()) + return cmd +} + +func newConnectWorktreesListCmd() *cobra.Command { + var shadow bool + cmd := &cobra.Command{ + Use: "list", + Short: "List the worktrees kept for you to deal with", + Long: `List the worktrees the connector kept, with the task each was for, its size +on disk, git's record of it, and why it is kept: finished (its task ended — +the connector removes no worktree of its own accord), dirty (uncommitted +work), unpushed (commits nothing else holds), locked, moved (no longer where +the connector left it), orphaned (its directory is gone, while git's record +of it and the task branch are still there), or unverified (their state could +not be read). A prune says which of these a worktree turns out to be.`, + Example: ` basecamp connect worktrees list -P agent`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + app := appctx.FromContext(cmd.Context()) + wt, closeLedger, err := openConnectWorktrees(app, shadow) + if err != nil { + return err + } + defer closeLedger() + retained, err := wt.Retained(cmd.Context()) + if err != nil { + return err + } + out := make([]worktreeView, 0, len(retained)) + for _, r := range retained { + out = append(out, viewWorktree(r)) + } + return app.OK(out, output.WithSummary(fmt.Sprintf("%d worktree(s) kept", len(out)))) + }, + } + cmd.Flags().BoolVar(&shadow, "shadow", false, "Read the shadow connector's state instead") + return cmd +} + +func newConnectWorktreesPruneCmd() *cobra.Command { + var ( + force []string + shadow bool + ) + cmd := &cobra.Command{ + Use: "prune", + Short: "Remove the kept worktrees you have dealt with", + Long: `Remove every kept worktree that holds no work: clean, with every commit it +reaches held elsewhere. This is the only thing that removes a worktree. One +that still holds work is kept and listed with why. + +--force removes that worktree even with work in it; name each one, and +it tells you what goes. +Every commit it reaches that nothing else holds is first kept under +refs/basecamp-connect/retained/ (retained_refs), so a force on a worktree that +is still on disk discards files, never commits. A worktree holding a submodule's own git data, or a lock, is +never forced; neither is one that is no longer where it was (reason "moved"): +move it back, or remove it yourself and prune again. A force that could not go +through is reported as kept with force_refused. Worktrees of tasks still +running are never touched. + +A worktree whose directory something else removed (reason "orphaned") is left +exactly as it is — git's record of it and the task branch, whatever they reach +— and a plain prune leaves it alone. A force on its path deletes the task +branch and nothing else, leaving git's record for ` + "`git worktree prune`" + `: +commits only that branch or that record reached go when you do that, and +nothing here works out which those are. Move the directory back, or keep the +branch, if you want them. A worktree whose state could not be read +(reason "unverified") is kept; forcing it keeps every commit that could be +found, which in a repository that keeps no reflogs may not be all of them.`, + Example: ` basecamp connect worktrees prune -P agent + basecamp connect worktrees prune -P agent --force ~/.local/state/basecamp/connect/2914079-52007412/worktrees/app-1a2b3c4d/17-a1b2c3`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + app := appctx.FromContext(cmd.Context()) + for i, p := range force { + if !filepath.IsAbs(p) { + return output.ErrUsage(fmt.Sprintf("--force %q: name the worktree by its absolute path, as worktrees list shows it", p)) + } + force[i] = filepath.Clean(p) + } + wt, closeLedger, err := openConnectWorktrees(app, shadow) + if err != nil { + return err + } + defer closeLedger() + results, err := wt.Prune(cmd.Context(), force) + if errors.Is(err, connector.ErrNotRetained) { + return output.ErrUsageHint("Nothing was pruned: "+err.Error(), "--force takes a path from `basecamp connect worktrees list`.") + } + if err != nil { + return err + } + out := make([]pruneView, 0, len(results)) + removed, kept := 0, 0 + for _, r := range results { + out = append(out, pruneView{ + worktreeView: viewWorktree(r.Worktree), Action: string(r.Action), ForceRefused: r.ForceRefused, + RetainedRefs: r.RetainedRefs, BranchDeletedAt: r.BranchDeletedAt, + }) + if r.Action == connector.PruneKept { + kept++ + } else { + removed++ + } + } + return app.OK(out, output.WithSummary(fmt.Sprintf("%d removed, %d kept", removed, kept))) + }, + } + cmd.Flags().StringArrayVar(&force, "force", nil, "Remove this kept worktree even with work in it (repeatable; an absolute path from worktrees list)") + cmd.Flags().BoolVar(&shadow, "shadow", false, "Read the shadow connector's state instead") + return cmd +} + +// worktreeView is a kept worktree as the commands show it. +type worktreeView struct { + Path string `json:"path"` + State string `json:"state"` + // SizeBytes is what the worktree takes up on disk, so an operator can + // see what reclaiming it is worth; -1 when it is there and could not be + // read, and nothing at all for one that is gone. + SizeBytes int64 `json:"size_bytes,omitempty"` + WorkDir string `json:"work_dir"` + Branch string `json:"branch"` + Route string `json:"route"` + Reason string `json:"reason,omitempty"` + // Record is git's record of the worktree (/.git/worktrees/), + // which outlives a directory something else removed: what an operator + // needs to find what is left, and what `git worktree prune` clears. + Record string `json:"record,omitempty"` + EventID int64 `json:"event_id"` + TaskID int64 `json:"task_id,omitempty"` + RetainedAt string `json:"retained_at,omitempty"` +} + +type pruneView struct { + worktreeView + Action string `json:"action"` + ForceRefused bool `json:"force_refused,omitempty"` + RetainedRefs []string `json:"retained_refs,omitempty"` + // BranchDeletedAt is where the task branch stood when a force on an + // orphaned worktree deleted it: nothing worked out what it reached, so + // this is what puts it back (git branch ). + BranchDeletedAt string `json:"branch_deleted_at,omitempty"` +} + +// sizeLimit bounds how long reading a worktree's size may take: a listing is +// not worth holding for a tree that cannot be walked. +const sizeLimit = 5 * time.Second + +// reasonOf is why a worktree is kept: nothing, for one that is not. +func reasonOf(w connector.Worktree) string { + if w.State == connector.WorktreeRemoved { + return "" + } + return string(w.RetainedReason) +} + +// recordOf is git's record of the worktree, when it is still there: the +// directory an orphaned worktree leaves behind. +func recordOf(w connector.Worktree) string { + if w.AdminDir == "" || w.State == connector.WorktreeRemoved { + return "" + } + if _, err := os.Lstat(w.AdminDir); err != nil { + return "" + } + return w.AdminDir +} + +// sizeOf is what a worktree takes up on disk. A worktree that is not there +// takes up nothing, and is not walked for an answer; one a removal has +// frozen is under its removing name. +func sizeOf(w connector.Worktree) int64 { + for _, path := range []string{w.Path, w.Path + connector.RemovingSuffix} { + switch _, err := os.Lstat(path); { + case err == nil: + return dirSize(path) + case !errors.Is(err, os.ErrNotExist): + return -1 + } + } + return 0 +} + +// dirSize is what a directory takes up, in bytes, following no symlink; -1 +// when it cannot be read in time or at all. The walk runs apart from the +// answer: a filesystem call that never returns — a mount a worker left — +// keeps only its own goroutine, and never the listing. +func dirSize(path string) int64 { + deadline := time.Now().Add(sizeLimit) + walked := make(chan int64, 1) + go func() { + var total int64 + err := filepath.WalkDir(path, func(_ string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if time.Now().After(deadline) { + return errors.New("the worktree could not be read in time") + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if info.Mode().IsRegular() { + total += info.Size() + } + return nil + }) + if err != nil { + total = -1 + } + walked <- total + }() + timer := time.NewTimer(time.Until(deadline)) + defer timer.Stop() + select { + case total := <-walked: + return total + case <-timer.C: + return -1 + } +} + +func viewWorktree(w connector.Worktree) worktreeView { + v := worktreeView{ + Path: w.Path, State: string(w.State), SizeBytes: sizeOf(w), WorkDir: w.WorkDir, + Branch: w.Branch, Route: w.Route, Reason: reasonOf(w), Record: recordOf(w), + EventID: w.OriginatingEventID, TaskID: w.TaskID, + } + if !w.RetainedAt.IsZero() { + v.RetainedAt = w.RetainedAt.UTC().Format(time.RFC3339) + } + return v +} + +// openConnectWorktrees opens the ledger of the connector the active profile +// is set up as: the one it has, never a new one, and never a schema this +// binary would migrate under a connector that is running. +func openConnectWorktrees(app *appctx.App, shadow bool) (*connector.Worktrees, func(), error) { + if app == nil { + return nil, nil, errors.New("app not initialized") + } + name := app.Config.ActiveProfile + if name == "" { + return nil, nil, output.ErrUsageHint("Worktrees belong to a connector's profile", "Pass -P/--profile , a profile set up with `basecamp connect setup`.") + } + path, err := setup.Path(config.GlobalConfigDir(), name) + if err != nil { + return nil, nil, output.ErrUsage(err.Error()) + } + file, err := setup.Load(path) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, nil, output.ErrUsageHint(fmt.Sprintf("Profile %q is not set up as a connector", name), "Run: basecamp connect setup -P "+shellQuote(name)) + case err != nil: + return nil, nil, output.ErrUsage("connect.json cannot be used: " + err.Error()) + } + // Named, not created: reading what a connector left must not make a + // state directory for a connector that never ran. + stateDir, err := connectStateDirPath(file, shadow) + if err != nil { + return nil, nil, output.ErrUsage("The connector's state directory cannot be used: " + err.Error()) + } + ledgerPath := filepath.Join(stateDir, connector.LedgerFile) + if _, err := os.Lstat(ledgerPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil, output.ErrUsageHint("This connector has not run yet: there is no ledger in "+stateDir, "Run: basecamp connect -P "+shellQuote(name)) + } + return nil, nil, err + } + ledger, err := connector.OpenExistingLedger(context.Background(), ledgerPath) + if err != nil { + return nil, nil, err + } + wt, err := connector.NewWorktrees(connector.WorktreesOptions{ + Ledger: ledger, Root: filepath.Join(stateDir, connectWorktreesDir), + // What a removal refuses is said, not swallowed. + Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + if err != nil { + _ = ledger.Close() + return nil, nil, err + } + return wt, func() { _ = ledger.Close() }, nil +} diff --git a/internal/commands/connect_worktrees_test.go b/internal/commands/connect_worktrees_test.go new file mode 100644 index 000000000..931f4f9bc --- /dev/null +++ b/internal/commands/connect_worktrees_test.go @@ -0,0 +1,155 @@ +package commands + +import ( + "bytes" + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/appctx" + "github.com/basecamp/basecamp-cli/internal/config" + "github.com/basecamp/basecamp-cli/internal/connector" + "github.com/basecamp/basecamp-cli/internal/connector/admission" + "github.com/basecamp/basecamp-cli/internal/connector/setup" + "github.com/basecamp/basecamp-cli/internal/output" +) + +// worktreesCmdEnv is a set-up connector profile with a ledger holding one +// retained worktree whose directory the operator already removed. +func worktreesCmdEnv(t *testing.T) (*appctx.App, *bytes.Buffer, connector.Worktree) { + t.Helper() + root := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", filepath.Join(root, "config")) + t.Setenv("XDG_STATE_HOME", filepath.Join(root, "state")) + t.Setenv("USERPROFILE", root) + + file := setup.New("agent") + file.AccountID = "2914079" + file.Agent = setup.Agent{PersonID: 52007412, Kind: setup.KindAgent} + file.Trust.OperatorID = 26909558 + repo := filepath.Join(root, "repo") + require.NoError(t, os.MkdirAll(repo, 0o700)) + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"commit", "-q", "--allow-empty", "-m", "init"}} { + cmd := exec.CommandContext(context.Background(), "git", append([]string{"-c", "user.name=T", "-c", "user.email=t@example.invalid"}, args...)...) + cmd.Dir = repo + cmd.Env = []string{"HOME=" + root, "PATH=" + os.Getenv("PATH")} + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + } + file.Projects[48699913] = admission.Route{Path: repo} + path, err := setup.Path(config.GlobalConfigDir(), "agent") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + data, err := json.Marshal(file) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, data, 0o600)) + + stateDir, err := connectStateDir(file, false) + require.NoError(t, err) + ledger, err := connector.OpenLedger(filepath.Join(stateDir, connector.LedgerFile)) + require.NoError(t, err) + defer func() { _ = ledger.Close() }() + w := connector.Worktree{ + Path: filepath.Join(stateDir, "worktrees", "app-00000000", "7-abcdef"), Route: repo, Repository: repo, + Branch: connector.BranchPrefix + "7-abcdef", BaseCommit: "0123456789abcdef0123456789abcdef01234567", OriginatingEventID: 7, + } + w.WorkDir = w.Path + id, err := ledger.BeginWorktree(context.Background(), w) + require.NoError(t, err) + // Git's record of it, as the connector stores it once the worktree is + // made: what is left of an orphan. + w.AdminDir = filepath.Join(repo, ".git", "worktrees", "7-abcdef") + require.NoError(t, os.MkdirAll(w.AdminDir, 0o700)) + require.NoError(t, ledger.WorktreeAdminDir(context.Background(), id, w.AdminDir)) + require.NoError(t, ledger.RetainWorktree(context.Background(), id, connector.RetainedDirty, connector.WorktreeCreating)) + + cfg := config.Default() + cfg.ActiveProfile = "agent" + var out bytes.Buffer + app := &appctx.App{Config: cfg, Output: output.New(output.Options{Format: output.FormatJSON, Writer: &out})} + return app, &out, w +} + +func runWorktreesCmd(t *testing.T, app *appctx.App, args ...string) error { + t.Helper() + cmd := NewConnectCmd() + cmd.SetArgs(append([]string{"worktrees"}, args...)) + cmd.SetContext(appctx.WithApp(context.Background(), app)) + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + return cmd.Execute() +} + +func TestConnectWorktreesListShowsTheKeptOnes(t *testing.T) { + app, out, w := worktreesCmdEnv(t) + require.NoError(t, runWorktreesCmd(t, app, "list")) + assert.Contains(t, out.String(), w.Path) + assert.Contains(t, out.String(), `"reason": "dirty"`) +} + +// A listing says what each kept worktree takes up, and says nothing about the +// size of one that is gone. +func TestConnectWorktreesSayWhatTheyTakeUp(t *testing.T) { + app, out, w := worktreesCmdEnv(t) + require.NoError(t, os.MkdirAll(w.Path, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(w.Path, "notes.txt"), bytes.Repeat([]byte("x"), 1234), 0o600)) + require.NoError(t, runWorktreesCmd(t, app, "list")) + assert.Contains(t, out.String(), `"size_bytes": 1234`) + + out.Reset() + require.NoError(t, os.RemoveAll(w.Path)) + require.NoError(t, runWorktreesCmd(t, app, "prune")) + assert.Contains(t, out.String(), `"reason": "orphaned"`) + assert.NotContains(t, out.String(), `"size_bytes"`, "a worktree that is gone has no size") +} + +// An orphaned worktree is listed with git's record of it, which is what is +// left to deal with. +func TestConnectWorktreesShowTheRecordOfAnOrphan(t *testing.T) { + app, out, w := worktreesCmdEnv(t) + require.NoError(t, runWorktreesCmd(t, app, "list")) + assert.Contains(t, out.String(), `"record": "`+w.AdminDir+`"`) +} + +func TestConnectWorktreesPruneRefusesWhatItCannotName(t *testing.T) { + app, _, _ := worktreesCmdEnv(t) + err := runWorktreesCmd(t, app, "prune", "--force", "relative/path") + require.Error(t, err) + assert.Contains(t, err.Error(), "absolute path") + + err = runWorktreesCmd(t, app, "prune", "--force", "/not/a/kept/worktree") + require.Error(t, err) + assert.Contains(t, err.Error(), "Nothing was pruned") +} + +// A worktree whose directory is gone is reported as orphaned, with git's +// record of it, and a plain prune deletes none of what it left; the operator +// naming its path is what clears it. +func TestConnectWorktreesPruneLeavesAnOrphanAloneUntilItIsNamed(t *testing.T) { + app, out, w := worktreesCmdEnv(t) + require.NoError(t, runWorktreesCmd(t, app, "prune")) + assert.Contains(t, out.String(), `"reason": "orphaned"`) + assert.Contains(t, out.String(), `"record"`, "what is left of it") + assert.NotContains(t, out.String(), `"force_refused"`, "nothing was forced") + out.Reset() + require.NoError(t, runWorktreesCmd(t, app, "list")) + assert.Contains(t, out.String(), w.Path, "still listed for the operator") + + out.Reset() + require.NoError(t, runWorktreesCmd(t, app, "prune", "--force", w.Path)) + assert.Contains(t, out.String(), `"action": "forced"`) + out.Reset() + require.NoError(t, runWorktreesCmd(t, app, "list")) + assert.NotContains(t, out.String(), w.Path) +} diff --git a/internal/connector/driver/codex/codex.go b/internal/connector/driver/codex/codex.go new file mode 100644 index 000000000..a50d428cd --- /dev/null +++ b/internal/connector/driver/codex/codex.go @@ -0,0 +1,1226 @@ +// Package codex is the spawn driver for Codex: `codex exec --json`, adapted +// onto the driver package's ACP-shaped session. +// +// One process is one turn. `codex exec` reads its prompt from stdin to the end +// and exits once the turn is over, so a session takes a single prompt and +// advertises no follow-up prompts; a follow-up waits for a new attempt, and +// LoadSession continues the conversation in a new process with +// `codex exec resume`. +// +// # Invariants +// +// The driver package's invariants hold here, each by a test in codex_test.go: +// +// 1. Nothing is inherited. The process environment is SessionConfig.Env and +// the few variables Codex itself needs; the host's config.toml, rules, +// hooks, plugins, connected apps and skills are not loaded; the only MCP +// servers are SessionConfig.MCPServers. The model's shell gets Codex's +// core environment only. +// 2. Nothing is written to disk to start a session, and no MCP server's +// environment reaches Codex's own. A server's declared environment goes +// to Codex as its mcp_servers env table, which Codex hands only to that +// server. It carries no secret: the task token reaches the worker's MCP +// server over the connector's one-use socket (connector/tokensocket.go), +// never through the driver. (Codex's own env_vars would copy a variable +// from Codex's environment, and from there to the model's shell.) +// 3. The permission mode is set by flags and verified. `codex exec` echoes +// no mode, and an override Codex does not recognize is silently ignored, +// so the driver reads the policy Codex actually applied from the turn's +// turn_context record in its rollout file, and ends the session as +// unsafe (ErrUnsafeMode) when it is not the one asked for or cannot be +// read. Both the sandbox mode and the filesystem policy the sandbox is +// built from are checked: nothing but the working directory writable. +// A turn is never reported finished before that check passed, and +// a turn that fails or loses its process after Codex reported its thread +// waits for the check too, so an unsafe session is reported as unsafe. +// The check runs beside the turn, not before it: Codex writes the record +// as the turn starts, so the window is the first model response. +// 4. Every MCP server is required: Codex refuses to start a turn when one +// fails to initialize, so a worker never runs without its Basecamp +// server. +// 5. Cancel ends the process group the driver started. A turn ends as +// TurnCanceled only when Cancel asked for it. +// 6. Updates carry kinds, ids and counts, never the agent's text, a +// command, or a tool's arguments, and everything that leaves the driver — +// errors, updates, refusals, the stderr tail — goes through the shared +// redactor. +// 7. Every refusal is recorded through SessionConfig.Refusals as it is read, +// once per call, whichever way the turn ends: a refusal Codex puts only on +// its stderr is read before a canceled, failed or lost turn is finished. +// +// Codex's reach differs from Claude Code's, and this driver claims nothing +// beyond it: Codex reads and searches through shell commands, so its shell +// is not removed but confined by Codex's own sandbox (workspace-write: +// writes only inside the working directory, no network, no /tmp) with +// approvals set to never, so whatever the sandbox would refuse is refused +// without asking anyone. That is still policy, not containment: the sandbox +// is Codex's, not the connector's. One consequence is worth knowing: a +// worktree's git data lives outside the working directory, so a Codex worker +// cannot commit, and a Codex task that edits anything ends with its worktree +// kept. Codex's sandbox reads the whole filesystem, but runs the model's +// shell in a PID namespace of its own, so the processes outside it — MCP +// servers among them — are not visible to it. +package codex + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/basecamp/basecamp-cli/internal/connector/driver" +) + +// Name is the driver's name. +const Name = "codex" + +// Env is what Codex may take from the connector's environment besides +// driver.BaseEnv: where its state and login live, and an API key for a login +// that uses one. +var Env = []string{"CODEX_HOME", "CODEX_API_KEY"} + +// DefaultVerifyTimeout is how long the driver waits for Codex's rollout to +// show the policy it applied. +const DefaultVerifyTimeout = 15 * time.Second + +// Options configures the driver. +type Options struct { + // Binary is the codex executable; "codex" on PATH when empty. + Binary string + // Model is passed as --model when set. + Model string + // Lookup reads the connector's environment for Env; os.LookupEnv when + // nil. + Lookup func(string) (string, bool) + // CloseGrace is how long a session's process group has between SIGTERM + // and SIGKILL. + CloseGrace time.Duration + // VerifyTimeout bounds the wait for the rollout's policy record. + VerifyTimeout time.Duration +} + +// Driver starts Codex sessions. +type Driver struct { + opts Options +} + +var _ driver.Driver = (*Driver)(nil) + +// New builds the driver. +func New(opts Options) *Driver { + if opts.Binary == "" { + opts.Binary = "codex" + } + if opts.Lookup == nil { + opts.Lookup = os.LookupEnv + } + if opts.CloseGrace <= 0 { + opts.CloseGrace = 5 * time.Second + } + if opts.VerifyTimeout <= 0 { + opts.VerifyTimeout = DefaultVerifyTimeout + } + return &Driver{opts: opts} +} + +// Name implements driver.Driver. +func (d *Driver) Name() string { return Name } + +// Capabilities implements driver.Driver. A Codex process takes one prompt. +func (d *Driver) Capabilities() driver.Capabilities { + // LoadSession works (codex exec resume), but it is not advertised: the + // thread id is known only once the prompt is written, after the + // dispatcher has recorded the session, so no ledger record could name + // one to resume. + return driver.Capabilities{} +} + +// NewSession implements driver.Driver. The session's id is Codex's thread id, +// which Codex reports only once the prompt is written: ID is empty until then. +func (d *Driver) NewSession(ctx context.Context, cfg driver.SessionConfig) (driver.Session, error) { + s, err := d.start(ctx, cfg, "") + return s, d.redactor(cfg).Err(err) +} + +// redactor is what every error and text of a session passes through: the +// dispatcher's Redaction, plus the environment this driver builds, its MCP +// servers' environments and its private directory. +func (d *Driver) redactor(cfg driver.SessionConfig) *driver.Redactor { + more := driver.Redaction{Env: d.env(cfg), Dirs: []string{cfg.PrivateDir}} + for _, server := range cfg.MCPServers { + more.Env = append(more.Env, driver.EnvOf(server.Env)...) + } + return driver.NewRedactor(cfg.Redaction.With(more)) +} + +// env is the worker's whole environment: the dispatcher's, plus what Codex +// itself needs. +func (d *Driver) env(cfg driver.SessionConfig) []string { + return mergeEnv(cfg.Env, driver.BuildEnv(Env, d.opts.Lookup, nil)) +} + +// LoadSession implements driver.Driver: `codex exec resume `. +func (d *Driver) LoadSession(ctx context.Context, cfg driver.SessionConfig, sessionID string) (driver.Session, error) { + if !validThreadID(sessionID) { + return nil, d.redactor(cfg).Err(fmt.Errorf("%w: %w: session id %q is not a Codex thread id", driver.ErrNotStarted, driver.ErrUnusable, sessionID)) + } + s, err := d.start(ctx, cfg, sessionID) + return s, d.redactor(cfg).Err(err) +} + +// Policy Codex runs every session under, as its turn_context spells it. +const ( + approvalNever = "never" + sandboxWorkdir = "workspace-write" +) + +// disabledFeatures are Codex features that reach past the session's MCP +// servers and working directory: the account's connected apps and plugins, +// the host's hooks, a browser and the desktop, image generation, memories +// shared across sessions, and installing what a skill asks for. +var disabledFeatures = []string{ + "apps", "plugins", "remote_plugin", "hooks", + "browser_use", "browser_use_external", "computer_use", "in_app_browser", + "image_generation", "memories", "skill_mcp_dependency_install", "tool_suggest", +} + +// allowedKinds are the tool kinds a policy may allow that Codex can honor: +// its reads, searches and planning run inside the sandbox that confines +// edits to the working directory. +var allowedKinds = []driver.ToolKind{driver.ToolRead, driver.ToolSearch, driver.ToolThink} + +var validServerName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +// Args is the command line for a session, without the binary. Exposed so the +// flags that hold the policy are tested as written. +func Args(cfg driver.SessionConfig, resumeID, model string) ([]string, error) { + if cfg.Policy == nil { + return nil, errors.New("codex: a session needs a policy") + } + rules := cfg.Policy.Rules() + if rules.Mode != driver.ModeEditsInWorkDir { + return nil, fmt.Errorf("%w: codex: no Codex sandbox for policy mode %q", driver.ErrUnusable, rules.Mode) + } + if filepath.Clean(rules.WorkDir) != filepath.Clean(cfg.Cwd) { + return nil, fmt.Errorf("%w: codex: the policy's working directory %q is not the session's %q", driver.ErrUnusable, rules.WorkDir, cfg.Cwd) + } + for _, kind := range rules.AllowKinds { + if !slices.Contains(allowedKinds, kind) { + return nil, fmt.Errorf("%w: codex: no Codex policy allows kind %q and nothing else", driver.ErrUnusable, kind) + } + } + + args := []string{"exec"} + if resumeID != "" { + args = append(args, "resume") + } + args = append(args, + "--json", + // A -c key Codex does not know is ignored in silence, and the flags + // below are what invariant 1 rests on. + "--strict-config", + // The host's config.toml (its MCP servers, profiles, hooks, trust) + // and its execpolicy rules are not this session's. + "--ignore-user-config", + "--ignore-rules", + // connect.json approved the directory; Codex's own trust prompt has + // nobody to answer it. + "--skip-git-repo-check", + "-c", "approval_policy="+tomlString(approvalNever), + "-c", "sandbox_mode="+tomlString(sandboxWorkdir), + "-c", "sandbox_workspace_write.network_access=false", + "-c", "sandbox_workspace_write.exclude_slash_tmp=true", + "-c", "sandbox_workspace_write.exclude_tmpdir_env_var=true", + "-c", "sandbox_workspace_write.writable_roots=[]", + // The model's shell commands get Codex's core variables, not the + // worker's whole environment. + "-c", "shell_environment_policy.inherit="+tomlString("core"), + "-c", "web_search="+tomlString("disabled"), + // Skills on the host (the connector's own front-thread skill among + // them) are not instructions this worker follows. + "-c", "skills.bundled.enabled=false", + "-c", "skills.include_instructions=false", + ) + for _, f := range disabledFeatures { + args = append(args, "--disable", f) + } + for _, s := range cfg.MCPServers { + if !validServerName.MatchString(s.Name) { + return nil, fmt.Errorf("%w: codex: MCP server name %q is not one Codex's config can key", driver.ErrUnusable, s.Name) + } + if s.Command == "" { + return nil, fmt.Errorf("%w: codex: MCP server %q has no command", driver.ErrUnusable, s.Name) + } + approval := "prompt" + if slices.Contains(rules.AllowMCPServers, s.Name) { + approval = "approve" + } + key := "mcp_servers." + s.Name + "." + env, err := tomlTable(s.Env) + if err != nil { + return nil, fmt.Errorf("%w: codex: MCP server %q: %w", driver.ErrUnusable, s.Name, err) + } + args = append(args, + "-c", key+"command="+tomlString(s.Command), + "-c", key+"args="+tomlArray(s.Args), + "-c", key+"env="+env, + "-c", key+"required=true", + "-c", key+"default_tools_approval_mode="+tomlString(approval), + ) + } + if model != "" { + args = append(args, "--model", model) + } + if resumeID != "" { + args = append(args, resumeID) + } + // The prompt is read from stdin, never argv. + return append(args, "-"), nil +} + +func (d *Driver) start(ctx context.Context, cfg driver.SessionConfig, resumeID string) (driver.Session, error) { + if cfg.Policy == nil || cfg.PrivateDir == "" || cfg.Cwd == "" { + return nil, fmt.Errorf("%w: a session needs a policy, a working directory and a private directory", driver.ErrNotStarted) + } + env := d.env(cfg) + sessions, err := sessionsDir(env) + if err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrNotStarted, err) + } + var offset int64 + if resumeID != "" { + path, err := findRollout(sessions, resumeID) + if err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrNotStarted, err) + } + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrNotStarted, err) + } + offset = info.Size() + } + args, err := Args(cfg, resumeID, d.opts.Model) + if err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrNotStarted, err) + } + worker, err := driver.StartWorker(ctx, cfg.Launcher, cfg.Scope, driver.Command{Path: d.opts.Binary, Args: args, Env: env, Dir: cfg.Cwd}) + if err != nil { + return nil, err + } + s := &session{ + red: d.redactor(cfg), + recorder: cfg.Refusals, + recorded: map[string]bool{}, + id: resumeID, + worker: worker, + cwd: cfg.Cwd, + sessions: sessions, + offset: offset, + grace: d.opts.CloseGrace, + verifyAfter: d.opts.VerifyTimeout, + writing: make(chan struct{}, 1), + updates: make(chan driver.Update, 256), + readerEnd: make(chan struct{}), + } + go s.read() //nolint:contextcheck // the reader outlives the start's context: it runs as long as the worker does + return s, nil +} + +// sessionsDir is where Codex writes rollouts for this environment. +func sessionsDir(env []string) (string, error) { + vars := driver.EnvMap(env) + home := vars["CODEX_HOME"] + if home == "" { + if vars["HOME"] == "" { + return "", errors.New("codex: the worker's environment names no HOME or CODEX_HOME") + } + home = filepath.Join(vars["HOME"], ".codex") + } + if !filepath.IsAbs(home) { + return "", fmt.Errorf("codex: CODEX_HOME %q is not absolute", home) + } + return filepath.Join(home, "sessions"), nil +} + +// mergeEnv adds the driver's own variables to the dispatcher's allowlisted +// environment. A variable the dispatcher set wins. +func mergeEnv(base, extra []string) []string { + have := map[string]bool{} + for _, kv := range base { + k, _, _ := strings.Cut(kv, "=") + have[k] = true + } + out := slices.Clone(base) + if out == nil { + out = []string{} + } + for _, kv := range extra { + k, _, _ := strings.Cut(kv, "=") + if !have[k] { + out = append(out, kv) + } + } + slices.Sort(out) + return out +} + +var validEnvName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// tomlTable is an inline TOML table of strings, keys sorted. +func tomlTable(values map[string]string) (string, error) { + keys := make([]string, 0, len(values)) + for k := range values { + if !validEnvName.MatchString(k) { + return "", fmt.Errorf("environment variable name %q", k) + } + keys = append(keys, k) + } + slices.Sort(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, tomlString(k)+"="+tomlString(values[k])) + } + return "{" + strings.Join(parts, ",") + "}", nil +} + +// tomlString is a TOML basic string. Only \\, \" and \uXXXX escapes are +// used, which TOML and JSON read alike. +func tomlString(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range s { + switch { + case r == '"' || r == '\\': + b.WriteByte('\\') + b.WriteRune(r) + case r < 0x20 || r == 0x7f: + fmt.Fprintf(&b, `\u%04x`, r) + default: + b.WriteRune(r) + } + } + b.WriteByte('"') + return b.String() +} + +func tomlArray(items []string) string { + quoted := make([]string, len(items)) + for i, s := range items { + quoted[i] = tomlString(s) + } + return "[" + strings.Join(quoted, ",") + "]" +} + +// session is one Codex process. +type session struct { + worker *driver.Worker + cwd string + sessions string + offset int64 + grace time.Duration + verifyAfter time.Duration + + // red is what every error, update text and stderr tail of this session + // passes through. + red *driver.Redactor + // recorder records each refusal once, as it is made or read (driver's + // "Refusals"); recorded is what has been recorded already, by tool call + // id for the refusals on the stream and by line for the ones Codex only + // logs. + recorder driver.RefusalRecorder + recorded map[string]bool + + updates chan driver.Update + readerEnd chan struct{} + + mu sync.Mutex + id string + prompted bool + // ended is the reader's record that the worker's output is over. + ended bool + // cancelEarly is a Cancel before any prompt: the prompt, when it comes, + // is not sent. + cancelEarly bool + turn *turn + verifyDone chan struct{} + verifyErr error + closed bool + // writing is a one-slot semaphore around the worker's stdin. A lock + // would be worse: a worker that stops reading its input blocks the + // write, and everything waiting on the lock — Close among them — waits + // with it. Whoever cannot take it in time goes on without it and ends + // the process instead. + writing chan struct{} +} + +// turn is the prompt in flight. +type turn struct { + done chan struct{} + result driver.PromptResult + err error + canceled bool + refusals []driver.Refusal +} + +var _ driver.Session = (*session)(nil) + +func (s *session) ID() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.id +} +func (s *session) Process() driver.Process { return s.worker.Process() } + +// StderrTail is the last line of the worker's stderr, redacted, for a caller +// diagnosing an end. +func (s *session) StderrTail() string { return s.worker.StderrTail(s.red) } +func (s *session) Updates() <-chan driver.Update { return s.updates } +func (s *session) Done() <-chan struct{} { return s.worker.Done() } +func (s *session) Exit() driver.Exit { return s.worker.Exit() } + +// errOnePrompt is a second prompt to a Codex process. +var errOnePrompt = fmt.Errorf("%w: a Codex session takes one prompt", driver.ErrSessionEnded) + +// Prompt implements driver.Session: the prompt is written to stdin, which is +// then closed, and the turn runs to its end. +func (s *session) Prompt(ctx context.Context, prompt string) (driver.PromptResult, error) { + s.mu.Lock() + switch { + case s.closed: + s.mu.Unlock() + return driver.PromptResult{}, driver.ErrSessionEnded + case s.prompted: + s.mu.Unlock() + return driver.PromptResult{}, errOnePrompt + case s.cancelEarly: + // Cancel came before the prompt and already ended the worker: + // nothing is written, and the turn is canceled, even if the worker's + // output is over by now. + s.prompted = true + s.mu.Unlock() + return driver.PromptResult{Stop: driver.TurnCanceled}, nil + case s.ended: + // The worker's output ended while this prompt was on its way in: a + // turn installed now would wait for a result nobody is left to write. + s.mu.Unlock() + return driver.PromptResult{}, driver.ErrSessionEnded + } + s.prompted = true + t := &turn{done: make(chan struct{})} + s.turn = t + s.mu.Unlock() + + // The write runs apart: a worker that stops reading blocks it, and a ctx + // that ends must still end the wait (driver.Session's contract), while the + // turn itself is ended by Cancel or Close. What it may end up recording — + // a refusal read from the worker's last word — outlives this prompt's + // context, as every refusal does. + //nolint:contextcheck // the recorder's write is not this prompt's to cancel + go func() { + s.writing <- struct{}{} + _, err := io.WriteString(s.worker.Stdin(), prompt) + if closeErr := s.worker.Stdin().Close(); err == nil { + err = closeErr + } + <-s.writing + if err == nil { + return + } + // A cancel that closed the worker's stdin is what made the write + // fail: the turn is canceled, not a session that ended on its own. + s.mu.Lock() + canceled := t.canceled + s.mu.Unlock() + if canceled { + s.finishCanceled(t) + } else { + s.finish(t, driver.PromptResult{}, fmt.Errorf("%w: %w", driver.ErrSessionEnded, err)) + } + }() + select { + case <-t.done: + return t.result, s.red.Err(t.err) + case <-ctx.Done(): + return driver.PromptResult{}, ctx.Err() + } +} + +// Cancel implements driver.Session: the process group is ended, and the turn +// in flight ends canceled. A Cancel before the session's prompt cancels that +// prompt, which the dispatcher may send from another goroutine an instant +// later. +func (s *session) Cancel(context.Context) error { + s.mu.Lock() + t := s.turn + if t != nil { + t.canceled = true + } else if !s.prompted { + // A cancel that races the prompt it is meant for: the worker is ended + // now, whether that prompt ever comes or not. + s.cancelEarly = true + t = &turn{} + } + s.mu.Unlock() + if t == nil { + return nil + } + go s.worker.Terminate(s.grace) + return nil +} + +// Close implements driver.Session. +func (s *session) Close() error { + s.mu.Lock() + s.closed = true + s.mu.Unlock() + // Stdin is closed under the semaphore when it is free; a prompt still + // blocked writing it keeps it, and Terminate below ends that. + select { + case s.writing <- struct{}{}: + _ = s.worker.Stdin().Close() + <-s.writing + case <-time.After(s.grace): + } + select { + case <-s.worker.Done(): + case <-time.After(s.grace): + } + s.worker.Terminate(s.grace) + select { + case <-s.readerEnd: + case <-time.After(s.grace): + // The worker is gone and a descendant outside its group still holds + // the output: stop reading it, rather than hold the attempt, its + // working directory and the connector's shutdown open forever. + s.worker.CloseStdout() + <-s.readerEnd + } + return nil +} + +func (s *session) finish(t *turn, result driver.PromptResult, err error) { + s.mu.Lock() + if s.turn != t { + s.mu.Unlock() + return + } + s.turn = nil + s.mu.Unlock() + t.result, t.err = result, err + close(t.done) +} + +func (s *session) emit(u driver.Update) { + u.At = time.Now() + u.Tool = s.red.Sanitize(u.Tool) + u.ToolCallID = s.red.Sanitize(u.ToolCallID) + select { + case s.updates <- u: + default: + } +} + +// read maps the process's JSON lines onto updates and the turn's result until +// the process closes its stdout. +func (s *session) read() { + defer func() { + // The updates channel closes last: finishing the turn still emits + // (a refusal read from stderr), and a send on a closed channel is a + // panic, not a dropped update. + defer close(s.updates) + s.mu.Lock() + s.ended = true + t := s.turn + s.mu.Unlock() + // Whatever ended the turn, and whether or not one is still in flight, + // a refusal Codex only logged is read before the session is done: a + // cancel, which finishes its turn early, is where they would + // otherwise be lost. The stderr is whole only once the process is + // gone, which closing its stdout does not say. + if s.worker != nil { + select { + case <-s.worker.Done(): + case <-time.After(s.grace): + } + } + s.stderrRefusals() + if t != nil { + s.mu.Lock() + canceled := t.canceled + s.mu.Unlock() + refusals := s.refusalsOf(t) + switch { + case canceled: + s.finishCanceled(t) + default: + err := s.failedVerification() + if err == nil { + err = driver.ErrSessionEnded + } + s.finish(t, driver.PromptResult{Refusals: refusals}, err) + } + } + close(s.readerEnd) + }() + scanner := bufio.NewScanner(s.worker.Stdout()) + scanner.Buffer(make([]byte, 64<<10), 64<<20) + for scanner.Scan() { + s.handle(scanner.Bytes()) + } + // Drain what a scanner error left, so the process never blocks writing. + _, _ = io.Copy(io.Discard, s.worker.Stdout()) +} + +// event is the part of a `codex exec --json` line the driver reads. Text, +// commands, arguments and results are never decoded into anything kept. +type event struct { + Type string `json:"type"` + ThreadID string `json:"thread_id"` + Item *struct { + ID string `json:"id"` + Type string `json:"type"` + Status string `json:"status"` + Server string `json:"server"` + Tool string `json:"tool"` + Text string `json:"text"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } `json:"item"` + Usage *struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + } `json:"usage"` +} + +func (s *session) handle(line []byte) { + var e event + if err := json.Unmarshal(line, &e); err != nil { + return + } + switch e.Type { + case "thread.started": + s.threadStarted(e.ThreadID) + case "item.started", "item.updated", "item.completed": + if e.Item != nil { + s.item(e.Type, e) + } + case "turn.completed": + s.turnCompleted(e) + case "turn.failed": + s.turnFailed() + } +} + +// threadStarted records the thread id and starts reading the rollout for the +// policy Codex applied (invariant 3). An unsafe session is ended as soon as the +// check fails, while the model may still be thinking; a turn that ends first +// waits for the check. +func (s *session) threadStarted(id string) { + s.mu.Lock() + defer s.mu.Unlock() + if s.verifyDone != nil { + return + } + done := make(chan struct{}) + s.verifyDone = done + if !validThreadID(id) || (s.id != "" && s.id != id) { + s.verifyErr = fmt.Errorf("%w: Codex reported thread %q", driver.ErrUnsafeMode, sanitize(id)) + close(done) + go s.unsafe(s.verifyErr) + return + } + s.id = id + go func() { + err := verifyRollout(s.sessions, id, s.offset, s.cwd, s.verifyAfter) + s.mu.Lock() + s.verifyErr = err + s.mu.Unlock() + close(done) + if err != nil { + s.unsafe(err) + } + }() +} + +// unsafe ends the turn in flight with err and the process group. +func (s *session) unsafe(err error) { + s.mu.Lock() + t := s.turn + s.mu.Unlock() + if t == nil { + s.worker.Terminate(0) + return + } + s.finishUnsafe(t, err) +} + +// finishUnsafe ends a turn whose session did not run under the policy it was +// asked to: the worker goes first, then its last word is read, so the result +// carries the refusals it made and logged before it was stopped, as every +// other ending does. +func (s *session) finishUnsafe(t *turn, err error) { + s.worker.Terminate(0) + s.lastWord() + s.finish(t, driver.PromptResult{Refusals: s.refusalsOf(t)}, err) +} + +// failedVerification is a turn that ended some other way than completed: once +// Codex reported its thread, the check's verdict is waited for, so an unsafe +// session is reported as unsafe rather than as a plain failure. Before a +// thread there was no turn to verify. +func (s *session) failedVerification() error { + s.mu.Lock() + started := s.verifyDone != nil + s.mu.Unlock() + if !started { + return nil + } + return s.verified() +} + +// lastWord waits for the worker to go, bounded by the grace, and reads the +// refusals it only logged. Whatever ends a turn ends it after this, so a +// refusal Codex wrote on its way out is in the turn's result and not only in +// the ledger. +func (s *session) lastWord() { + if s.worker != nil { + select { + case <-s.worker.Done(): + case <-time.After(s.grace): + } + } + s.stderrRefusals() +} + +// finishCanceled ends a turn the connector canceled, after the worker's last +// word. A policy check that has already failed is reported over the cancel; +// one still running is not waited for, because the process it would judge is +// being ended by the cancel anyway. +func (s *session) finishCanceled(t *turn) { + s.lastWord() + // The turn's refusals are read after the worker's last word, so the + // result carries what the ledger carries. + refusals := s.refusalsOf(t) + s.mu.Lock() + done := s.verifyDone + s.mu.Unlock() + if done != nil { + select { + case <-done: + s.mu.Lock() + verdict := s.verifyErr + s.mu.Unlock() + if verdict != nil { + s.finish(t, driver.PromptResult{Refusals: refusals}, verdict) + return + } + default: + } + } + s.finish(t, driver.PromptResult{Stop: driver.TurnCanceled, Refusals: refusals}, nil) +} + +// verified waits for the policy check's verdict. +func (s *session) verified() error { + s.mu.Lock() + done := s.verifyDone + s.mu.Unlock() + if done == nil { + return fmt.Errorf("%w: the turn ended before Codex reported its thread", driver.ErrUnsafeMode) + } + select { + case <-done: + case <-time.After(s.verifyAfter + 5*time.Second): + return fmt.Errorf("%w: the policy check did not finish", driver.ErrUnsafeMode) + } + s.mu.Lock() + defer s.mu.Unlock() + return s.verifyErr +} + +func (s *session) item(kind string, e event) { + it := e.Item + u := driver.Update{ToolCallID: it.ID, Status: toolStatus(kind, it.Status)} + switch it.Type { + case "agent_message": + if kind == "item.completed" { + s.emit(driver.Update{Kind: driver.UpdateAgentMessageChunk, Chars: len(it.Text)}) + } + return + case "reasoning", "error", "user_message": + return + case "command_execution": + u.Tool, u.ToolKind = "exec", driver.ToolExecute + case "file_change": + u.Tool, u.ToolKind = "apply_patch", driver.ToolEdit + case "mcp_tool_call": + u.Tool, u.ToolKind = "mcp__"+sanitize(it.Server)+"__"+sanitize(it.Tool), driver.ToolOther + case "web_search": + u.Tool, u.ToolKind = "web_search", driver.ToolFetch + case "todo_list": + if kind == "item.completed" || kind == "item.started" { + s.emit(driver.Update{Kind: driver.UpdatePlan}) + } + return + default: + u.Tool, u.ToolKind = sanitize(it.Type), driver.ToolOther + } + if kind == "item.started" { + u.Kind = driver.UpdateToolCall + } else { + u.Kind = driver.UpdateToolCallUpdate + } + s.emit(u) + if kind == "item.completed" && it.Type == "mcp_tool_call" && it.Error != nil && refusedByApproval(it.Error.Message) { + s.refused("item:"+it.ID, it.ID, u.Tool, u.ToolKind) + } +} + +func toolStatus(kind, status string) driver.ToolStatus { + switch status { + case "completed": + return driver.ToolCompleted + case "failed", "declined": + return driver.ToolFailed + case "in_progress": + return driver.ToolInProgress + } + if kind == "item.started" { + return driver.ToolInProgress + } + return driver.ToolCompleted +} + +// refusedByApproval is Codex's message for a call its approval policy +// refused: under approvals set to never, a call that needs one is refused. +func refusedByApproval(message string) bool { + return strings.Contains(message, "approval policy is never") || strings.Contains(message, "rejected by user approval settings") +} + +// refused is the moment a refusal is read: it is recorded through the +// session's recorder before anything else is done with it, and only the first +// time its tool call id is seen (driver's "Refusals"). A refusal Codex logs +// and gives no id gets the key the caller passes. +func (s *session) refused(key, id, tool string, kind driver.ToolKind) { + refusal := driver.Refusal{ToolCallID: s.red.Sanitize(id), Tool: s.red.Sanitize(tool)} + s.mu.Lock() + first := !s.recorded[key] + if first { + s.recorded[key] = true + if s.turn != nil { + s.turn.refusals = append(s.turn.refusals, refusal) + } + } + s.mu.Unlock() + if !first { + return + } + if s.recorder != nil { + // The recorder owns what happens when the ledger refuses the write; + // the refusal happened either way. + _ = s.recorder.RecordRefusal(context.Background(), refusal) + } + s.emit(driver.Update{Kind: driver.UpdatePermission, ToolCallID: id, Tool: tool, ToolKind: kind, Allowed: false}) +} + +func (s *session) turnCompleted(e event) { + s.mu.Lock() + t := s.turn + s.mu.Unlock() + if t == nil { + return + } + s.mu.Lock() + canceled := t.canceled + s.mu.Unlock() + if canceled { + // A cancel that won does not wait out the policy check either. + s.finishCanceled(t) + return + } + if err := s.verified(); err != nil { + s.finishUnsafe(t, err) + return + } + // Codex exits right after the turn it completed, and its stderr is whole + // only once it has: a refusal it logged and did not put on the stream is + // in the tail by then. + select { + case <-s.worker.Done(): + case <-time.After(s.grace): + } + s.stderrRefusals() + s.mu.Lock() + result := driver.PromptResult{Stop: driver.TurnEndTurn, Refusals: slices.Clone(t.refusals)} + if t.canceled { + // Only a cancel the connector asked for reads as canceled. + result.Stop = driver.TurnCanceled + } + s.mu.Unlock() + if e.Usage != nil { + result.Usage = driver.Usage{InputTokens: e.Usage.InputTokens, OutputTokens: e.Usage.OutputTokens} + s.emit(driver.Update{Kind: driver.UpdateUsage, Usage: &result.Usage}) + } + s.finish(t, result, nil) +} + +func (s *session) turnFailed() { + s.mu.Lock() + t := s.turn + s.mu.Unlock() + if t == nil { + return + } + s.mu.Lock() + canceled := t.canceled + s.mu.Unlock() + if canceled { + s.finishCanceled(t) + return + } + // As after a completed turn: the stderr tail is whole once Codex exits. + select { + case <-s.worker.Done(): + case <-time.After(s.grace): + } + s.stderrRefusals() + refusals := s.refusalsOf(t) + if err := s.failedVerification(); err != nil { + s.finishUnsafe(t, err) + return + } + s.finish(t, driver.PromptResult{Refusals: refusals}, errors.New("codex: the turn failed")) +} + +// refusalsOf is a turn's refusals so far. +func (s *session) refusalsOf(t *turn) []driver.Refusal { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(t.refusals) +} + +// stderrRefusals counts the refusals Codex logs but does not put on its JSON +// stream: an edit outside the working directory. Best effort: the stderr +// kept is a tail. +func (s *session) stderrRefusals() { + if s.worker == nil { + return + } + // Every line the worker's stderr kept, sanitized: a refusal Codex logs + // and does not put on the stream is one of them, wherever it is in the + // output. + seen := map[string]int{} + for _, line := range s.worker.StderrLines(s.red) { + if !refusedByApproval(line) { + continue + } + tool, kind := "exec", driver.ToolExecute + if strings.Contains(line, "patch rejected") { + tool, kind = "apply_patch", driver.ToolEdit + } + // Codex gives these no id, so the key is the line and how many times + // it has been seen in this output: two refusals Codex logged the same + // way are two, and reading the same output again — every way a turn + // can end reads it — records each of them once. + seen[line]++ + s.refused("stderr:"+strconv.Itoa(seen[line])+":"+line, "", tool, kind) + } +} + +// turnContext is the part of a rollout's turn_context record the driver +// checks. +type turnContext struct { + Cwd string `json:"cwd"` + ApprovalPolicy string `json:"approval_policy"` + SandboxPolicy struct { + Type string `json:"type"` + NetworkAccess bool `json:"network_access"` + ExcludeTmpdirEnvVar bool `json:"exclude_tmpdir_env_var"` + ExcludeSlashTmp bool `json:"exclude_slash_tmp"` + WritableRoots []string `json:"writable_roots"` + } `json:"sandbox_policy"` + // FileSystem is the filesystem policy Codex's sandbox is actually built + // from; PermissionProfile carries the same in older records. + FileSystem *fileSystemPolicy `json:"file_system_sandbox_policy"` + PermissionProfile *struct { + FileSystem *fileSystemPolicy `json:"file_system"` + } `json:"permission_profile"` +} + +type fileSystemPolicy struct { + Kind string `json:"kind"` + Type string `json:"type"` + Entries []struct { + Path struct { + Type string `json:"type"` + Path string `json:"path"` + } `json:"path"` + Access string `json:"access"` + } `json:"entries"` +} + +// writesOnlyIn reports whether a filesystem policy is restricted and lets +// nothing but cwd be written. +func (p *fileSystemPolicy) writesOnlyIn(cwd string) bool { + if p == nil || (p.Kind != "restricted" && p.Type != "restricted") { + return false + } + writable := false + for _, e := range p.Entries { + if e.Access == "read" || e.Access == "none" { + continue + } + if e.Path.Type != "path" || !samePath(e.Path.Path, cwd) { + return false + } + writable = true + } + return writable +} + +// verifyRollout waits for the first turn_context record after offset in the +// thread's rollout and checks it is the policy the flags asked for. +func verifyRollout(sessions, threadID string, offset int64, cwd string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var path string + for { + if path == "" { + if p, err := findRollout(sessions, threadID); err == nil { + path = p + } + } + if path != "" { + tc, found, next, err := readTurnContext(path, offset) + if err != nil { + return fmt.Errorf("%w: reading Codex's rollout: %w", driver.ErrUnsafeMode, err) + } + offset = next + if found { + return checkTurnContext(tc, cwd) + } + } + if time.Now().After(deadline) { + return fmt.Errorf("%w: Codex's rollout showed no policy within %s", driver.ErrUnsafeMode, timeout) + } + time.Sleep(50 * time.Millisecond) + } +} + +func checkTurnContext(tc turnContext, cwd string) error { + p := tc.SandboxPolicy + switch { + case tc.ApprovalPolicy != approvalNever: + return fmt.Errorf("%w: asked for approvals %q, Codex applied %q", driver.ErrUnsafeMode, approvalNever, sanitize(tc.ApprovalPolicy)) + case p.Type != sandboxWorkdir: + return fmt.Errorf("%w: asked for sandbox %q, Codex applied %q", driver.ErrUnsafeMode, sandboxWorkdir, sanitize(p.Type)) + case p.NetworkAccess || !p.ExcludeSlashTmp || !p.ExcludeTmpdirEnvVar || len(p.WritableRoots) > 0: + return fmt.Errorf("%w: Codex's sandbox reaches past the working directory", driver.ErrUnsafeMode) + case !samePath(tc.Cwd, cwd): + return fmt.Errorf("%w: Codex runs in another directory than the session's", driver.ErrUnsafeMode) + } + fs := tc.FileSystem + if fs == nil && tc.PermissionProfile != nil { + fs = tc.PermissionProfile.FileSystem + } + if !fs.writesOnlyIn(cwd) { + return fmt.Errorf("%w: Codex's filesystem sandbox writes past the working directory, or was not reported", driver.ErrUnsafeMode) + } + return nil +} + +func samePath(a, b string) bool { + if a == "" || b == "" { + return false + } + if filepath.Clean(a) == filepath.Clean(b) { + return true + } + ra, errA := filepath.EvalSymlinks(a) + rb, errB := filepath.EvalSymlinks(b) + return errA == nil && errB == nil && ra == rb +} + +// readTurnContext scans complete lines from offset for a turn_context record. +// It returns the offset after the last complete line it read. +func readTurnContext(path string, offset int64) (turnContext, bool, int64, error) { + f, err := os.Open(path) + if err != nil { + return turnContext{}, false, offset, err + } + defer func() { _ = f.Close() }() + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return turnContext{}, false, offset, err + } + r := bufio.NewReaderSize(f, 64<<10) + for { + line, err := r.ReadBytes('\n') + if err != nil { + // A line without its newline is still being written. + if errors.Is(err, io.EOF) { + return turnContext{}, false, offset, nil + } + return turnContext{}, false, offset, err + } + offset += int64(len(line)) + var rec struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + } + if json.Unmarshal(line, &rec) != nil || rec.Type != "turn_context" { + continue + } + var tc turnContext + if err := json.Unmarshal(rec.Payload, &tc); err != nil { + return turnContext{}, false, offset, errors.New("an unreadable turn_context record") + } + return tc, true, offset, nil + } +} + +// findRollout finds a thread's rollout file: sessions/YYYY/MM/DD/rollout-*-.jsonl. +func findRollout(sessions, threadID string) (string, error) { + if !validThreadID(threadID) { + return "", fmt.Errorf("codex: %q is not a thread id", sanitize(threadID)) + } + matches, err := filepath.Glob(filepath.Join(sessions, "*", "*", "*", "rollout-*-"+threadID+".jsonl")) + if err != nil { + return "", err + } + switch len(matches) { + case 0: + return "", fmt.Errorf("codex: no rollout for thread %s", threadID) + case 1: + return matches[0], nil + } + return "", fmt.Errorf("codex: %d rollouts for thread %s", len(matches), threadID) +} + +var threadIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +func validThreadID(s string) bool { return threadIDPattern.MatchString(s) } + +// sanitize keeps a vendor token (a server or tool name, a policy value) to a +// short run of plain characters. +func sanitize(s string) string { + out := make([]rune, 0, len(s)) + for _, r := range s { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' { + out = append(out, r) + } + if len(out) >= 64 { + break + } + } + return string(out) +} diff --git a/internal/connector/driver/codex/codex_test.go b/internal/connector/driver/codex/codex_test.go new file mode 100644 index 000000000..494a506f8 --- /dev/null +++ b/internal/connector/driver/codex/codex_test.go @@ -0,0 +1,1138 @@ +//go:build unix + +package codex + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/connector" + "github.com/basecamp/basecamp-cli/internal/connector/driver" + "github.com/basecamp/basecamp-cli/internal/connector/driver/drivertest" +) + +const ( + testThread = "01a0adfe-499c-7f63-9553-b9975a3c4b55" + testToken = "test-token-not-real" + hostCanary = "host-canary-not-real" + serverOnly = "declared-for-the-server-only" +) + +// safeTurnContext is the policy the driver's flags ask for. +func safeTurnContext() map[string]any { + return map[string]any{ + "approval_policy": "never", + "sandbox_policy": map[string]any{ + "type": "workspace-write", "network_access": false, + "exclude_tmpdir_env_var": true, "exclude_slash_tmp": true, + }, + // "$CWD" is the fake's own working directory. + "file_system_sandbox_policy": map[string]any{ + "kind": "restricted", + "entries": []any{ + map[string]any{"path": map[string]any{"type": "special", "value": map[string]any{"kind": "root"}}, "access": "read"}, + map[string]any{"path": map[string]any{"type": "path", "path": "$CWD"}, "access": "write"}, + map[string]any{"path": map[string]any{"type": "path", "path": "$CWD/.git"}, "access": "read"}, + }, + }, + } +} + +func fsEntries(tc map[string]any) []any { + return tc["file_system_sandbox_policy"].(map[string]any)["entries"].([]any) +} + +type harness struct { + t *testing.T + home string // CODEX_HOME + workDir string + private string + mcpOut string + drv *Driver +} + +func newHarness(t *testing.T, sc scenario) *harness { + t.Helper() + root := t.TempDir() + h := &harness{ + t: t, + home: filepath.Join(root, "codex-home"), + workDir: filepath.Join(root, "work"), + private: filepath.Join(root, "private"), + mcpOut: filepath.Join(root, "mcp-env.txt"), + } + require.NoError(t, os.Mkdir(h.home, 0o700)) + require.NoError(t, os.Mkdir(h.workDir, 0o700)) + require.NoError(t, os.Mkdir(h.private, 0o700)) + if sc.Thread == "" { + sc.Thread = testThread + } + h.scenario(sc) + self, err := os.Executable() + require.NoError(t, err) + host := map[string]string{ + "CODEX_HOME": h.home, + "HOME": root, + "PATH": os.Getenv("PATH"), + "HOST_SECRET_NOT_REAL": hostCanary, + "OPENAI_API_KEY": hostCanary, + } + h.drv = New(Options{ + Binary: self, + Lookup: func(k string) (string, bool) { v, ok := host[k]; return v, ok }, + CloseGrace: 2 * time.Second, + VerifyTimeout: time.Second, + }) + return h +} + +func (h *harness) scenario(sc scenario) { + if sc.Thread == "" { + sc.Thread = testThread + } + data, err := json.Marshal(sc) + require.NoError(h.t, err) + require.NoError(h.t, os.WriteFile(filepath.Join(h.home, "scenario.json"), data, 0o600)) +} + +type testPolicy struct { + workDir string + kinds []driver.ToolKind + servers []string + mode driver.PermissionMode +} + +func (p testPolicy) Decide(context.Context, driver.PermissionRequest) driver.PermissionDecision { + return driver.PermissionDecision{} +} + +func (p testPolicy) Rules() driver.PermissionRules { + mode := p.mode + if mode == "" { + mode = driver.ModeEditsInWorkDir + } + return driver.PermissionRules{Mode: mode, WorkDir: p.workDir, AllowKinds: p.kinds, AllowMCPServers: p.servers} +} + +func (h *harness) config() driver.SessionConfig { + return driver.SessionConfig{ + Cwd: h.workDir, + Env: []string{"HOME=" + filepath.Dir(h.home), "PATH=" + os.Getenv("PATH")}, + MCPServers: []driver.MCPServer{{ + Name: "basecamp", + Command: "/bin/sh", + Args: []string{"-c", `env > "$MCP_ENV_OUT"`}, + Env: map[string]string{"MCP_ENV_OUT": h.mcpOut, "SERVER_ONLY_NOT_SECRET": serverOnly, "PATH": os.Getenv("PATH")}, + }}, + Policy: connector.DefaultPolicy(h.workDir), + Scope: driver.Scope{WorkDir: h.workDir}, + PrivateDir: h.private, + } +} + +func (h *harness) observed() observed { + h.t.Helper() + data, err := os.ReadFile(filepath.Join(h.home, "observed.json")) + require.NoError(h.t, err) + var obs observed + require.NoError(h.t, json.Unmarshal(data, &obs)) + return obs +} + +func (h *harness) run(ctx context.Context, cfg driver.SessionConfig) (driver.Session, driver.PromptResult, error) { + h.t.Helper() + s, err := h.drv.NewSession(ctx, cfg) + require.NoError(h.t, err) + h.t.Cleanup(func() { _ = s.Close() }) + result, err := s.Prompt(ctx, "Task 1. Event 2.") + return s, result, err +} + +func turnCompleted() string { + return `{"type":"turn.completed","usage":{"input_tokens":120,"output_tokens":7}}` +} + +// The flags hold the v1 policy as written: the host's configuration, rules, +// features and skills off; approvals never; the sandbox confined to the +// working directory; the MCP server required, its tools approved only when +// the policy allows its server; the prompt on stdin. +func TestArgsHoldThePolicy(t *testing.T) { + cfg := driver.SessionConfig{ + Cwd: "/work/app", + Policy: connector.DefaultPolicy("/work/app"), + MCPServers: []driver.MCPServer{ + {Name: "basecamp", Command: "/bin/basecamp", Args: []string{"connect", "worker-mcp", "--socket", "/run/token.sock"}, Env: map[string]string{"HOME": "/home/op", "BASECAMP_NO_KEYRING": `a"quoted\value`}}, + {Name: "other", Command: "/bin/other"}, + }, + } + args, err := Args(cfg, "", "") + require.NoError(t, err) + + joined := strings.Join(args, "\x00") + for _, want := range [][]string{ + {"--json"}, {"--ignore-user-config"}, {"--ignore-rules"}, + {"-c", `approval_policy="never"`}, + {"-c", `sandbox_mode="workspace-write"`}, + {"-c", "sandbox_workspace_write.network_access=false"}, + {"-c", "sandbox_workspace_write.exclude_slash_tmp=true"}, + {"-c", "sandbox_workspace_write.exclude_tmpdir_env_var=true"}, + {"-c", "sandbox_workspace_write.writable_roots=[]"}, + {"-c", `shell_environment_policy.inherit="core"`}, + {"-c", "skills.include_instructions=false"}, + {"-c", "skills.bundled.enabled=false"}, + {"--disable", "apps"}, {"--disable", "plugins"}, {"--disable", "hooks"}, + {"--strict-config"}, + {"-c", `mcp_servers.basecamp.command="/bin/basecamp"`}, + {"-c", `mcp_servers.basecamp.args=["connect","worker-mcp","--socket","/run/token.sock"]`}, + {"-c", `mcp_servers.basecamp.env={"BASECAMP_NO_KEYRING"="a\"quoted\\value","HOME"="/home/op"}`}, + {"-c", "mcp_servers.basecamp.required=true"}, + {"-c", `mcp_servers.basecamp.default_tools_approval_mode="approve"`}, + {"-c", "mcp_servers.other.required=true"}, + {"-c", `mcp_servers.other.default_tools_approval_mode="prompt"`}, + } { + assert.Contains(t, joined, strings.Join(want, "\x00")) + } + assert.Equal(t, "exec", args[0]) + assert.Equal(t, "-", args[len(args)-1], "the prompt is read from stdin") + + resumed, err := Args(cfg, testThread, "gpt-test") + require.NoError(t, err) + assert.Equal(t, []string{"exec", "resume"}, resumed[:2]) + assert.Equal(t, []string{"--model", "gpt-test", testThread, "-"}, resumed[len(resumed)-4:]) +} + +// A policy Codex's flags cannot hold is refused before anything starts. +func TestArgsRefuseAPolicyCodexCannotHold(t *testing.T) { + server := []driver.MCPServer{{Name: "basecamp", Command: "/bin/basecamp"}} + for name, cfg := range map[string]driver.SessionConfig{ + "another mode": {Cwd: "/w", Policy: testPolicy{workDir: "/w", mode: "anything"}, MCPServers: server}, + "another workdir": {Cwd: "/w", Policy: testPolicy{workDir: "/elsewhere"}, MCPServers: server}, + "execute allowed": {Cwd: "/w", Policy: testPolicy{workDir: "/w", kinds: []driver.ToolKind{driver.ToolExecute}}, MCPServers: server}, + "fetch allowed": {Cwd: "/w", Policy: testPolicy{workDir: "/w", kinds: []driver.ToolKind{driver.ToolFetch}}, MCPServers: server}, + "unkeyable server": {Cwd: "/w", Policy: testPolicy{workDir: "/w"}, MCPServers: []driver.MCPServer{{Name: "a.b", Command: "/bin/x"}}}, + "no command": {Cwd: "/w", Policy: testPolicy{workDir: "/w"}, MCPServers: []driver.MCPServer{{Name: "other"}}}, + "unkeyable env name": {Cwd: "/w", Policy: testPolicy{workDir: "/w"}, MCPServers: []driver.MCPServer{{Name: "other", Command: "/bin/x", Env: map[string]string{"A=B": "x"}}}}, + } { + _, err := Args(cfg, "", "") + assert.ErrorIs(t, err, driver.ErrUnusable, name) + } +} + +// Invariants 1 and 2 under the connector's own token carriage: the MCP +// server gets exactly its declared environment, Codex's own environment gets +// none of it and nothing of the host's, and with a task token served on its +// one-use socket (as the dispatcher serves it) the token is in no +// environment, no argv, no log and no file, at any moment of the session. +func TestTheTaskTokenIsNowhereTheDriverTouches(t *testing.T) { + h := newHarness(t, scenario{RunMCP: true, TurnContext: safeTurnContext(), Events: []string{`{"type":"turn.started"}`, turnCompleted()}}) + tokens, err := connector.ServeTaskToken(h.private, testToken, time.Minute) + require.NoError(t, err) + t.Cleanup(tokens.Close) + cfg := h.config() + cfg.MCPServers[0].Args = append(cfg.MCPServers[0].Args, "--socket", tokens.Path()) + + var s driver.Session + stop := drivertest.WatchForSecretFiles(testToken, h.private, h.workDir, h.home) + s, result, err := h.run(context.Background(), cfg) + require.NoError(t, err) + require.NoError(t, s.Close()) + assert.Empty(t, stop(), "no file ever held the token") + assert.Equal(t, driver.TurnEndTurn, result.Stop) + assert.Equal(t, testThread, s.ID()) + + obs := h.observed() + for _, kv := range obs.Env { + assert.NotContains(t, kv, hostCanary, "nothing outside the allowlist is inherited") + assert.NotContains(t, kv, serverOnly, "an MCP server's environment is not Codex's") + } + assert.Contains(t, obs.Env, "CODEX_HOME="+h.home) + assert.Equal(t, "Task 1. Event 2.", obs.Prompt) + serverEnv, err := os.ReadFile(h.mcpOut) + require.NoError(t, err) + assert.Contains(t, string(serverEnv), "SERVER_ONLY_NOT_SECRET="+serverOnly) + + drivertest.RequireNoSecret(t, testToken, drivertest.Places{ + Env: obs.Env, + Args: obs.Args, + Texts: []string{s.(*session).StderrTail(), string(serverEnv)}, + Dirs: []string{h.workDir, h.private, h.home}, + }) +} + +// The credential rule, while the session runs, in drivertest's own form: no +// file under the session's directories ever carries the token. +func TestNoTokenFileEverExists(t *testing.T) { + h := newHarness(t, scenario{RunMCP: true, TurnContext: safeTurnContext(), Events: []string{turnCompleted()}}) + tokens, err := connector.ServeTaskToken(h.private, testToken, time.Minute) + require.NoError(t, err) + t.Cleanup(tokens.Close) + cfg := h.config() + cfg.MCPServers[0].Args = append(cfg.MCPServers[0].Args, "--socket", tokens.Path()) + drivertest.RequireNoSecretFilesDuring(t, testToken, []string{h.private, h.workDir, h.home}, func() { + s, _, err := h.run(context.Background(), cfg) + require.NoError(t, err) + require.NoError(t, s.Close()) + }) +} + +// Invariant 3: a turn is finished only once the rollout shows the policy the +// flags asked for; any other policy, or none, ends the session as unsafe. +func TestTheAppliedPolicyIsVerified(t *testing.T) { + events := []string{`{"type":"turn.started"}`, turnCompleted()} + unsafe := map[string]func(tc map[string]any){ + "approvals on request": func(tc map[string]any) { tc["approval_policy"] = "on-request" }, + "full access": func(tc map[string]any) { tc["sandbox_policy"].(map[string]any)["type"] = "danger-full-access" }, + "network": func(tc map[string]any) { tc["sandbox_policy"].(map[string]any)["network_access"] = true }, + "slash tmp": func(tc map[string]any) { tc["sandbox_policy"].(map[string]any)["exclude_slash_tmp"] = false }, + "writable roots": func(tc map[string]any) { tc["sandbox_policy"].(map[string]any)["writable_roots"] = []string{"/"} }, + "another directory": func(tc map[string]any) { tc["cwd"] = "/" }, + "no filesystem policy": func(tc map[string]any) { delete(tc, "file_system_sandbox_policy") }, + "root writable": func(tc map[string]any) { + fsEntries(tc)[0].(map[string]any)["access"] = "write" + }, + "another path writable": func(tc map[string]any) { + tc["file_system_sandbox_policy"].(map[string]any)["entries"] = append(fsEntries(tc), + map[string]any{"path": map[string]any{"type": "path", "path": "/tmp"}, "access": "write"}) + }, + "unrestricted": func(tc map[string]any) { tc["file_system_sandbox_policy"].(map[string]any)["kind"] = "unrestricted" }, + } + for name, mutate := range unsafe { + t.Run(name, func(t *testing.T) { + tc := safeTurnContext() + mutate(tc) + h := newHarness(t, scenario{TurnContext: tc, Events: events}) + s, _, err := h.run(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + waitDone(t, s) + }) + } + t.Run("no policy record", func(t *testing.T) { + h := newHarness(t, scenario{Events: events}) + s, _, err := h.run(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + waitDone(t, s) + }) + t.Run("no thread", func(t *testing.T) { + h := newHarness(t, scenario{NoThread: true, TurnContext: safeTurnContext(), Events: events}) + s, _, err := h.run(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + waitDone(t, s) + }) + t.Run("the policy asked for", func(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Events: events}) + _, result, err := h.run(context.Background(), h.config()) + require.NoError(t, err) + assert.Equal(t, driver.TurnEndTurn, result.Stop) + assert.Equal(t, driver.Usage{InputTokens: 120, OutputTokens: 7}, result.Usage) + }) +} + +// An unsafe session is ended while its turn is still running, not when the +// turn ends. +func TestAnUnsafeSessionIsEndedMidTurn(t *testing.T) { + tc := safeTurnContext() + tc["approval_policy"] = "untrusted" + h := newHarness(t, scenario{TurnContext: tc, Hang: true, Child: true, Events: []string{`{"type":"turn.started"}`}}) + start := time.Now() + s, _, err := h.run(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + waitDone(t, s) + assert.Less(t, time.Since(start), 30*time.Second) + assertGone(t, h.observed().ChildPID) +} + +// A resumed thread is judged by the turn it runs now, not by an earlier turn +// already in its rollout. +func TestAResumedThreadIsJudgedByItsNewTurn(t *testing.T) { + bad := safeTurnContext() + bad["approval_policy"] = "on-request" + h := newHarness(t, scenario{OldTurnContext: nil}) + // An earlier, safe turn is on disk before the resume. + rollout := filepath.Join(h.home, "sessions", "2026", "09", "16", "rollout-2026-09-16T08-00-00-"+testThread+".jsonl") + require.NoError(t, os.MkdirAll(filepath.Dir(rollout), 0o700)) + old := safeTurnContext() + old["cwd"] = h.workDir + line, err := json.Marshal(map[string]any{"type": "turn_context", "payload": old}) + require.NoError(t, err) + require.NoError(t, os.WriteFile(rollout, append(line, '\n'), 0o600)) + h.scenario(scenario{TurnContext: bad, Events: []string{turnCompleted()}}) + // The fake appends to the rollout it finds under today's name; point it at + // the same file. + require.NoError(t, os.MkdirAll(filepath.Join(h.home, "sessions", "2026", "09", "17"), 0o700)) + require.NoError(t, os.Rename(rollout, filepath.Join(h.home, "sessions", "2026", "09", "17", "rollout-2026-09-17T08-00-00-"+testThread+".jsonl"))) + + s, err := h.drv.LoadSession(context.Background(), h.config(), testThread) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + _, err = s.Prompt(context.Background(), "Event 3.") + require.ErrorIs(t, err, driver.ErrUnsafeMode) + assert.Equal(t, []string{"exec", "resume"}, h.observed().Args[:2]) +} + +func TestLoadSessionRefusesAThreadItCannotFind(t *testing.T) { + h := newHarness(t, scenario{}) + _, err := h.drv.LoadSession(context.Background(), h.config(), testThread) + require.ErrorIs(t, err, driver.ErrNotStarted) + _, err = h.drv.LoadSession(context.Background(), h.config(), "not-a-thread") + require.ErrorIs(t, err, driver.ErrNotStarted) + entries, err := os.ReadDir(h.private) + require.NoError(t, err) + assert.Empty(t, entries, "nothing is written for a session that never starts") +} + +// Invariant 4: an MCP server that fails leaves no turn: Codex refuses to start +// one, and the driver reports the session ended, never not-started, because +// a process existed. +func TestAFailedMCPServerEndsTheSession(t *testing.T) { + h := newHarness(t, scenario{RunMCP: true, TurnContext: safeTurnContext(), Events: []string{turnCompleted()}}) + cfg := h.config() + cfg.MCPServers[0].Args = []string{"-c", "exit 1"} + _, _, err := h.run(context.Background(), cfg) + require.Error(t, err) + assert.ErrorIs(t, err, driver.ErrSessionEnded) + assert.NotErrorIs(t, err, driver.ErrNotStarted) +} + +// Invariant 5: Cancel ends the whole process group, and only a cancel the +// connector asked for reads as canceled. +func TestCancelEndsTheProcessGroup(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Hang: true, Child: true, Events: []string{`{"type":"turn.started"}`}}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + + type answer struct { + result driver.PromptResult + err error + } + answers := make(chan answer, 1) + go func() { + r, err := s.Prompt(context.Background(), "Event 1.") + answers <- answer{r, err} + }() + pid := waitChild(t, h) + require.NoError(t, s.Cancel(context.Background())) + + select { + case a := <-answers: + require.NoError(t, a.err) + assert.Equal(t, driver.TurnCanceled, a.result.Stop) + case <-time.After(20 * time.Second): + t.Fatal("the canceled turn did not end") + } + waitDone(t, s) + assertGone(t, pid) +} + +func TestAWorkerThatExitsMidTurnIsNotCanceled(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Events: []string{`{"type":"turn.started"}`}, Exit: 0}) + _, result, err := h.run(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrSessionEnded) + assert.NotEqual(t, driver.TurnCanceled, result.Stop) +} + +func TestAFailedTurnIsAnError(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Events: []string{`{"type":"turn.failed","error":{"message":"someone@example.com"}}`}, Exit: 1}) + _, _, err := h.run(context.Background(), h.config()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "example.com") +} + +func TestASessionTakesOnePrompt(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Events: []string{turnCompleted()}}) + s, _, err := h.run(context.Background(), h.config()) + require.NoError(t, err) + _, err = s.Prompt(context.Background(), "Event 4.") + require.ErrorIs(t, err, driver.ErrSessionEnded) + assert.ErrorIs(t, err, errOnePrompt, "refused as a second prompt, not as a write to a closed pipe") + assert.False(t, h.drv.Capabilities().FollowUpPrompts) + assert.False(t, h.drv.Capabilities().LoadSession, "no ledger record can name a Codex thread to resume yet") +} + +// Invariant 6: updates carry kinds, ids and counts. A refusal Codex's +// approval policy made is the driver's own record, and does not read as a +// cancel. +func TestUpdatesCarryNoContentAndRefusalsAreRecorded(t *testing.T) { + secret := "SECRET-CONTENT-not-real" + events := []string{ + `{"type":"turn.started"}`, + `{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"` + secret + `"}}`, + `{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"cat ` + secret + `","status":"in_progress"}}`, + `{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"cat ` + secret + `","aggregated_output":"` + secret + `","exit_code":0,"status":"completed"}}`, + `{"type":"item.started","item":{"id":"item_2","type":"file_change","changes":[{"path":"/` + secret + `","kind":"add"}],"status":"in_progress"}}`, + `{"type":"item.completed","item":{"id":"item_3","type":"mcp_tool_call","server":"other","tool":"write","arguments":{"x":"` + secret + `"},"error":{"message":"MCP tool call requires approval, but approval policy is never"},"status":"failed"}}`, + turnCompleted(), + } + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Events: events}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + var updates []driver.Update + collected := make(chan struct{}) + go func() { + for u := range s.Updates() { + updates = append(updates, u) + } + close(collected) + }() + result, err := s.Prompt(context.Background(), "Event 1.") + require.NoError(t, err) + require.NoError(t, s.Close()) + <-collected + + assert.Equal(t, driver.TurnEndTurn, result.Stop) + require.Len(t, result.Refusals, 1) + assert.Equal(t, driver.Refusal{ToolCallID: "item_3", Tool: "mcp__other__write"}, result.Refusals[0]) + + data, err := json.Marshal(updates) + require.NoError(t, err) + assert.NotContains(t, string(data), secret) + kinds := make([]driver.UpdateKind, 0, len(updates)) + for _, u := range updates { + kinds = append(kinds, u.Kind) + } + for _, want := range []driver.UpdateKind{driver.UpdateAgentMessageChunk, driver.UpdateToolCall, driver.UpdateToolCallUpdate, driver.UpdatePermission, driver.UpdateUsage} { + assert.Contains(t, kinds, want) + } + i := slices.IndexFunc(updates, func(u driver.Update) bool { return u.Kind == driver.UpdateAgentMessageChunk }) + assert.Equal(t, len(secret), updates[i].Chars) +} + +// ErrNotStarted means no process: a missing binary is one, and leaves the +// session's private directory as it found it. +func TestAMissingBinaryIsNotStarted(t *testing.T) { + h := newHarness(t, scenario{}) + h.drv.opts.Binary = filepath.Join(t.TempDir(), "no-codex") + _, err := h.drv.NewSession(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrNotStarted) + entries, err := os.ReadDir(h.private) + require.NoError(t, err) + assert.Empty(t, entries) +} + +func waitDone(t *testing.T, s driver.Session) { + t.Helper() + select { + case <-s.Done(): + case <-time.After(20 * time.Second): + t.Fatal("the worker did not exit") + } +} + +func waitChild(t *testing.T, h *harness) int { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if data, err := os.ReadFile(filepath.Join(h.home, "observed.json")); err == nil { + var obs observed + if json.Unmarshal(data, &obs) == nil && obs.ChildPID > 0 { + return obs.ChildPID + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the fake never started its child") + return 0 +} + +func assertGone(t *testing.T, pid int) { + t.Helper() + require.Positive(t, pid) + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return + } + // A zombie still answers kill(0); its state is Z. + if stat, err := os.ReadFile(filepath.Join("/proc", itoa(pid), "stat")); err == nil && zombie(string(stat)) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("process %d outlived its group's end", pid) +} + +func itoa(n int) string { return strconv.Itoa(n) } + +// zombie reports whether a /proc//stat line is a zombie's. +func zombie(stat string) bool { + _, rest, ok := strings.Cut(stat, ") ") + return ok && strings.HasPrefix(rest, "Z") +} + +// Invariant 3: a turn that fails, or loses its process, before the policy +// check has spoken waits for it, so an unsafe session reads as unsafe. +func TestAFailedTurnWaitsForThePolicyCheck(t *testing.T) { + for name, sc := range map[string]scenario{ + "turn failed": {Events: []string{`{"type":"turn.started"}`, `{"type":"turn.failed","error":{"message":"x"}}`}, Exit: 1}, + "process gone": {Events: []string{`{"type":"turn.started"}`}, Exit: 1}, + } { + t.Run(name, func(t *testing.T) { + // No policy record: the check only fails when its timeout passes, + // well after the turn ended. + h := newHarness(t, sc) + _, _, err := h.run(context.Background(), h.config()) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + }) + } +} + +// Invariant 5: a Cancel that comes before the prompt it races cancels that +// prompt; nothing is sent and the worker is ended. +func TestACancelBeforeThePromptCancelsIt(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Hang: true, Events: []string{`{"type":"turn.started"}`}}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + require.NoError(t, s.Cancel(context.Background())) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result, err := s.Prompt(ctx, "Event 1.") + require.NoError(t, err) + assert.Equal(t, driver.TurnCanceled, result.Stop) + waitDone(t, s) +} + +// A canceled turn whose policy check has already failed is reported unsafe, +// not canceled; one whose check is still running is canceled at once. +func TestACanceledTurnReportsAFailedPolicyCheck(t *testing.T) { + for name, tc := range map[string]struct { + done bool + err error + want error + }{ + "check failed": {done: true, err: driver.ErrUnsafeMode, want: driver.ErrUnsafeMode}, + "check passed": {done: true}, + "check running": {}, + } { + t.Run(name, func(t *testing.T) { + s := &session{verifyDone: make(chan struct{}), verifyErr: tc.err} + if tc.done { + close(s.verifyDone) + } + turn := &turn{done: make(chan struct{})} + s.turn = turn + s.finishCanceled(turn) + <-turn.done + if tc.want != nil { + require.ErrorIs(t, turn.err, tc.want) + return + } + require.NoError(t, turn.err) + assert.Equal(t, driver.TurnCanceled, turn.result.Stop) + }) + } +} + +// Invariant 5: Close does not wait forever on a descendant that left the +// worker's process group and still holds its output. +func TestCloseDoesNotWaitForAnEscapedChild(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Escape: true, Events: []string{turnCompleted()}}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + _, err = s.Prompt(context.Background(), "Event 1.") + require.NoError(t, err) + t.Cleanup(func() { + if pid := h.observed().EscapedPID; pid > 0 { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + }) + done := make(chan struct{}) + go func() { _ = s.Close(); close(done) }() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Fatal("Close waited on an escaped child") + } +} + +// Invariant 6 and 3 together: a refusal Codex logs on stderr after the turn's +// last stdout line is still counted, and emitting it as the session ends does +// not send on a closed channel. +func TestARefusalLoggedAtTheVeryEndIsCounted(t *testing.T) { + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`, turnCompleted()}, + Stderr: "patch rejected: writing outside of the project; rejected by user approval settings", + }) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + drained := make(chan int, 1) + go func() { + n := 0 + for u := range s.Updates() { + if u.Kind == driver.UpdatePermission { + n++ + } + } + drained <- n + }() + result, err := s.Prompt(context.Background(), "Event 1.") + require.NoError(t, err) + require.NoError(t, s.Close()) + assert.Len(t, result.Refusals, 1) + assert.Positive(t, <-drained) +} + +// The same, when the process dies without completing its turn: the refusal is +// still emitted, and emitting it as the reader ends is not a send on a closed +// channel. +func TestARefusalLoggedAsTheWorkerDiesIsCounted(t *testing.T) { + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`}, + Stderr: "patch rejected: writing outside of the project; rejected by user approval settings", + Exit: 1, + }) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + drained := make(chan int, 1) + go func() { + n := 0 + for u := range s.Updates() { + if u.Kind == driver.UpdatePermission { + n++ + } + } + drained <- n + }() + result, err := s.Prompt(context.Background(), "Event 1.") + require.ErrorIs(t, err, driver.ErrSessionEnded) + assert.Len(t, result.Refusals, 1) + assert.Positive(t, <-drained) + require.NoError(t, s.Close()) +} + +// A worker that stops reading its input cannot hold Close or Cancel: the +// prompt's write waits on the worker, and nothing else waits on the write. +func TestAWorkerThatStopsReadingHoldsNothing(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Deaf: true, Hang: true, Events: []string{`{"type":"turn.started"}`}}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + go func() { _, _ = s.Prompt(context.Background(), strings.Repeat("Event 1. ", 200_000)) }() + waitDeaf(t, h) + + canceled := make(chan struct{}) + go func() { + require.NoError(t, s.Cancel(context.Background())) + close(canceled) + }() + select { + case <-canceled: + case <-time.After(20 * time.Second): + t.Fatal("a worker that stopped reading held Cancel") + } + + // And Close on its own, with no cancel to end the process first. + closed := make(chan struct{}) + go func() { + _ = s.Close() + close(closed) + }() + select { + case <-closed: + case <-time.After(30 * time.Second): + t.Fatal("a worker that stopped reading held Close") + } + waitDone(t, s) +} + +func waitDeaf(t *testing.T, h *harness) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if data, err := os.ReadFile(filepath.Join(h.home, "observed.json")); err == nil { + var obs observed + if json.Unmarshal(data, &obs) == nil && obs.Deaf { + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the fake never stopped reading") +} + +// The same for Close on its own: a prompt still blocked writing to a worker +// that stopped reading does not hold it. +func TestCloseSurvivesAWorkerThatStoppedReading(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Deaf: true, Hang: true, Events: []string{`{"type":"turn.started"}`}}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + go func() { _, _ = s.Prompt(context.Background(), strings.Repeat("Event 1. ", 200_000)) }() + waitDeaf(t, h) + + closed := make(chan struct{}) + go func() { + _ = s.Close() + close(closed) + }() + select { + case <-closed: + case <-time.After(30 * time.Second): + t.Fatal("a worker that stopped reading held Close") + } + waitDone(t, s) +} + +// A turn that completes while a cancel is pending ends canceled at once, not +// after the policy check's whole timeout. +func TestACompletedTurnThatWasCanceledDoesNotWaitForTheCheck(t *testing.T) { + s := &session{verifyDone: make(chan struct{}), verifyAfter: time.Hour} + turn := &turn{done: make(chan struct{}), canceled: true} + s.turn = turn + done := make(chan struct{}) + go func() { s.turnCompleted(event{}); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("a canceled turn waited for the policy check") + } + require.NoError(t, turn.err) + assert.Equal(t, driver.TurnCanceled, turn.result.Stop) +} + +// A cancel with no prompt yet ends the worker at once, whether the prompt ever +// comes or not. +func TestACancelWithNoPromptEndsTheWorker(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Hang: true}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + require.NoError(t, s.Cancel(context.Background())) + waitDone(t, s) + result, err := s.Prompt(context.Background(), "Event 1.") + require.NoError(t, err) + assert.Equal(t, driver.TurnCanceled, result.Stop, "canceled, not ended, though the worker is gone") +} + +// A refusal Codex logs after a failed turn's event is still counted. +func TestARefusalLoggedAfterAFailedTurnIsCounted(t *testing.T) { + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`, `{"type":"turn.failed","error":{"message":"x"}}`}, + Stderr: "patch rejected: writing outside of the project; rejected by user approval settings", + Exit: 1, + }) + _, result, err := h.run(context.Background(), h.config()) + require.Error(t, err) + assert.Len(t, result.Refusals, 1) +} + +// Prompt honors its context even while its write is blocked on a worker that +// stopped reading. +func TestAPromptBlockedWritingHonorsItsContext(t *testing.T) { + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Deaf: true, Hang: true}) + s, err := h.drv.NewSession(context.Background(), h.config()) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + returned := make(chan error, 1) + go func() { + _, err := s.Prompt(ctx, strings.Repeat("Event 1. ", 200_000)) + returned <- err + }() + select { + case err := <-returned: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(20 * time.Second): + t.Fatal("a blocked write held Prompt past its context") + } +} + +// redactionSecret is the value fed through every error path. It is obviously +// fake, and is planted everywhere a real secret would be: in the worker's +// environment, in its MCP server's environment, in the name of its private +// directory, and in what the agent writes back. +const redactionSecret = "test-token-not-real-c9f2b1" + +func redactionHarness(t *testing.T, sc scenario) (*harness, driver.SessionConfig) { + t.Helper() + h := newHarness(t, sc) + private := filepath.Join(t.TempDir(), redactionSecret) + require.NoError(t, os.Mkdir(private, 0o700)) + cfg := h.config() + cfg.PrivateDir = private + cfg.Env = append(cfg.Env, "FAKE_CODEX_SECRET="+redactionSecret) + cfg.MCPServers[0].Env["BASECAMP_CONNECT_TASK_TOKEN"] = redactionSecret + cfg.Redaction = driver.Redaction{Secrets: []string{redactionSecret}} + return h, cfg +} + +func drain(s driver.Session) []driver.Update { + var updates []driver.Update + for u := range s.Updates() { + updates = append(updates, u) + } + return updates +} + +// The redaction rule (driver's redact.go): nothing this driver hands back +// carries the secret, whichever way the session fails. +func TestNoErrorPathCarriesTheSecretOut(t *testing.T) { + secretEvents := []string{ + `{"type":"turn.started"}`, + `{"type":"item.completed","item":{"id":"` + redactionSecret + `","type":"mcp_tool_call","server":"` + redactionSecret + `","tool":"` + redactionSecret + `","status":"failed","error":{"message":"MCP tool call requires approval, but approval policy is never"}}}`, + } + stderr := "fatal: writing " + redactionSecret + ": patch rejected: writing outside of the project; rejected by user approval settings" + drivertest.RequireRedacted(t, redactionSecret, []drivertest.RedactionPath{ + {Name: "start", Run: func(t *testing.T) drivertest.Crossing { + h, cfg := redactionHarness(t, scenario{}) + h.drv.opts.Binary = filepath.Join(cfg.PrivateDir, "no-codex") + _, err := h.drv.NewSession(context.Background(), cfg) + require.ErrorIs(t, err, driver.ErrNotStarted) + return drivertest.Crossing{Errors: []error{err}} + }}, + {Name: "handshake", Run: func(t *testing.T) drivertest.Crossing { + tc := safeTurnContext() + tc["approval_policy"] = "on-request" + h, cfg := redactionHarness(t, scenario{TurnContext: tc, Events: append(secretEvents, turnCompleted()), Stderr: stderr}) + s, result, err := h.run(context.Background(), cfg) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + updates := make(chan []driver.Update, 1) + go func() { updates <- drain(s) }() + require.NoError(t, s.Close()) + return drivertest.Crossing{Errors: []error{err}, Results: []driver.PromptResult{result}, + Updates: <-updates, Texts: []string{s.(*session).StderrTail()}} + }}, + {Name: "prompt", Run: func(t *testing.T) drivertest.Crossing { + h, cfg := redactionHarness(t, scenario{TurnContext: safeTurnContext(), + Events: append(secretEvents, `{"type":"turn.failed","error":{"message":"`+redactionSecret+`"}}`), Stderr: stderr, Exit: 1}) + s, result, err := h.run(context.Background(), cfg) + require.Error(t, err) + updates := make(chan []driver.Update, 1) + go func() { updates <- drain(s) }() + require.NoError(t, s.Close()) + return drivertest.Crossing{Errors: []error{err}, Results: []driver.PromptResult{result}, + Updates: <-updates, Texts: []string{s.(*session).StderrTail()}} + }}, + {Name: "cancel", Run: func(t *testing.T) drivertest.Crossing { + h, cfg := redactionHarness(t, scenario{TurnContext: safeTurnContext(), Deaf: true, Hang: true, Stderr: stderr}) + s, err := h.drv.NewSession(context.Background(), cfg) + require.NoError(t, err) + go func() { _, _ = s.Prompt(context.Background(), strings.Repeat("x", 1<<20)) }() + waitDeaf(t, h) + cancelErr := s.Cancel(context.Background()) + closeErr := s.Close() + return drivertest.Crossing{Errors: []error{cancelErr, closeErr}, Texts: []string{s.(*session).StderrTail()}} + }}, + {Name: "close", Run: func(t *testing.T) drivertest.Crossing { + h, cfg := redactionHarness(t, scenario{TurnContext: safeTurnContext(), Events: secretEvents, Stderr: stderr, Exit: 1}) + s, result, err := h.run(context.Background(), cfg) + require.Error(t, err, "the worker died in the turn") + updates := make(chan []driver.Update, 1) + go func() { updates <- drain(s) }() + closeErr := s.Close() + after, afterErr := s.Prompt(context.Background(), "again") + return drivertest.Crossing{Errors: []error{err, closeErr, afterErr}, + Results: []driver.PromptResult{result, after}, Updates: <-updates, Texts: []string{s.(*session).StderrTail()}} + }}, + }) +} + +// The refusal rule (driver's "Refusals"): every refusal is recorded as it is +// read, once per call, whichever way the turn ends — a canceled turn included, +// where a refusal Codex only logged would otherwise go with the session. +func TestEveryRefusalIsRecordedOnce(t *testing.T) { + denial := `{"type":"item.completed","item":{"id":"item_7","type":"mcp_tool_call","server":"other","tool":"write","error":{"message":"MCP tool call requires approval, but approval policy is never"},"status":"failed"}}` + stderr := "patch rejected: writing outside of the project; rejected by user approval settings" + for name, tc := range map[string]struct { + events []string + exit int + cancel bool + wantErr bool + wantSeen int + }{ + "a completed turn": {events: []string{`{"type":"turn.started"}`, denial, turnCompleted()}, wantSeen: 2}, + "a failed turn": {events: []string{`{"type":"turn.started"}`, denial, `{"type":"turn.failed","error":{"message":"x"}}`}, exit: 1, wantErr: true, wantSeen: 2}, + "a lost worker": {events: []string{`{"type":"turn.started"}`, denial}, exit: 1, wantErr: true, wantSeen: 2}, + } { + t.Run(name, func(t *testing.T) { + recorder := &drivertest.Refusals{} + h := newHarness(t, scenario{TurnContext: safeTurnContext(), Events: tc.events, Stderr: stderr, Exit: tc.exit}) + cfg := h.config() + cfg.Refusals = recorder + s, result, err := h.run(context.Background(), cfg) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.NoError(t, s.Close()) + assert.Len(t, recorder.Recorded(), tc.wantSeen, "each refusal recorded once") + assert.Len(t, result.Refusals, tc.wantSeen) + }) + } +} + +// A worker whose output ends before it does: the refusal it logs on its way +// out is still read, because the reader waits for the process, not for its +// stdout. +func TestARefusalLoggedAfterTheOutputEndsIsStillRecorded(t *testing.T) { + recorder := &drivertest.Refusals{} + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`}, + CloseStdout: true, + Stderr: "patch rejected: writing outside of the project; rejected by user approval settings", + Exit: 1, + }) + cfg := h.config() + cfg.Refusals = recorder + s, result, err := h.run(context.Background(), cfg) + require.Error(t, err) + require.NoError(t, s.Close()) + assert.Len(t, recorder.Recorded(), 1, "the refusal Codex logged after closing its output") + assert.Len(t, result.Refusals, 1) +} + +// A session stopped for running under a policy it was not asked to run under +// still reports the refusals it made: they are the ledger's and the result's. +func TestAnUnsafeSessionStillReportsItsRefusals(t *testing.T) { + recorder := &drivertest.Refusals{} + denial := `{"type":"item.completed","item":{"id":"item_9","type":"mcp_tool_call","server":"other","tool":"write","error":{"message":"MCP tool call requires approval, but approval policy is never"},"status":"failed"}}` + unsafe := safeTurnContext() + unsafe["approval_policy"] = "on-request" + h := newHarness(t, scenario{ + TurnContext: unsafe, + Events: []string{`{"type":"turn.started"}`, denial, turnCompleted()}, + }) + cfg := h.config() + cfg.Refusals = recorder + s, result, err := h.run(context.Background(), cfg) + require.ErrorIs(t, err, driver.ErrUnsafeMode) + waitDone(t, s) + assert.Len(t, recorder.Recorded(), 1) + assert.Len(t, result.Refusals, 1, "the result carries what the ledger carries") +} + +// Codex logs its sandbox refusals and keeps writing: each one is recorded, +// not only whatever it said last. +func TestEveryRefusalCodexOnlyLogsIsRecorded(t *testing.T) { + recorder := &drivertest.Refusals{} + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`, turnCompleted()}, + Stderr: strings.Join([]string{ + "patch rejected: writing outside of the project; rejected by user approval settings", + "ERROR: command failed because the approval policy is never", + "thinking about the next step", + // The same diagnostic twice is two refusals, not one. + "ERROR: command failed because the approval policy is never", + }, "\n"), + }) + cfg := h.config() + cfg.Refusals = recorder + s, result, err := h.run(context.Background(), cfg) + require.NoError(t, err) + require.NoError(t, s.Close()) + assert.Len(t, recorder.Recorded(), 3, "every refusal, wherever it is and however it reads") + assert.Len(t, result.Refusals, 3) +} + +// A refusal Codex logged is recorded even when the turn it belonged to has +// already ended: the reader reads the stderr of a worker that is gone, with +// no turn left to hang it on. +func TestARefusalIsRecordedEvenWithNoTurnLeft(t *testing.T) { + recorder := &drivertest.Refusals{} + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Deaf: true, + Hang: true, + Events: []string{`{"type":"turn.started"}`}, + Stderr: "patch rejected: writing outside of the project; rejected by user approval settings", + }) + cfg := h.config() + cfg.Refusals = recorder + s, err := h.drv.NewSession(context.Background(), cfg) + require.NoError(t, err) + // A worker that never reads its input: the prompt's write blocks, and the + // cancel that closes its stdin ends the turn from the write's side, not + // the reader's. + go func() { _, _ = s.Prompt(context.Background(), strings.Repeat("Event 1. ", 200_000)) }() + waitDeaf(t, h) + require.Eventually(t, func() bool { return strings.Contains(s.(*session).StderrTail(), "rejected") }, 10*time.Second, 20*time.Millisecond) + require.NoError(t, s.Cancel(context.Background())) + require.NoError(t, s.Close()) + waitDone(t, s) + + assert.Len(t, recorder.Recorded(), 1, "the refusal is recorded, turn or no turn") +} + +// A refusal Codex logs on its way out of a canceled turn is in the turn's +// result, not only in the ledger: the cancel waits for the worker's last word. +func TestACanceledTurnCarriesALateRefusalInItsResult(t *testing.T) { + recorder := &drivertest.Refusals{} + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`}, + Hang: true, + // Codex logs the refusal as it is being ended, not before. + StderrOnTerm: "patch rejected: writing outside of the project; rejected by user approval settings", + }) + cfg := h.config() + cfg.Refusals = recorder + s, err := h.drv.NewSession(context.Background(), cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + answers := make(chan driver.PromptResult, 1) + go func() { + result, _ := s.Prompt(context.Background(), "Event 1.") + answers <- result + }() + require.Eventually(t, func() bool { + data, err := os.ReadFile(filepath.Join(h.home, "observed.json")) + return err == nil && strings.Contains(string(data), "Event 1.") + }, 10*time.Second, 20*time.Millisecond) + require.NoError(t, s.Cancel(context.Background())) + select { + case result := <-answers: + assert.Len(t, result.Refusals, 1, "the result carries what the ledger carries") + case <-time.After(20 * time.Second): + t.Fatal("the canceled turn did not end") + } + assert.Len(t, recorder.Recorded(), 1) +} + +// A canceled turn records what Codex logged before it went. +func TestACanceledTurnRecordsItsRefusals(t *testing.T) { + recorder := &drivertest.Refusals{} + h := newHarness(t, scenario{ + TurnContext: safeTurnContext(), + Events: []string{`{"type":"turn.started"}`}, + Stderr: "patch rejected: writing outside of the project; rejected by user approval settings", + Hang: true, + }) + cfg := h.config() + cfg.Refusals = recorder + s, err := h.drv.NewSession(context.Background(), cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + answers := make(chan driver.PromptResult, 1) + go func() { + result, _ := s.Prompt(context.Background(), "Event 1.") + answers <- result + }() + require.Eventually(t, func() bool { return strings.Contains(s.(*session).StderrTail(), "rejected") }, 10*time.Second, 20*time.Millisecond) + require.NoError(t, s.Cancel(context.Background())) + select { + case result := <-answers: + assert.Equal(t, driver.TurnCanceled, result.Stop) + assert.Len(t, result.Refusals, 1) + case <-time.After(20 * time.Second): + t.Fatal("the canceled turn did not end") + } + assert.Len(t, recorder.Recorded(), 1, "the refusal Codex logged is recorded, not lost with the cancel") +} diff --git a/internal/connector/driver/codex/fake_test.go b/internal/connector/driver/codex/fake_test.go new file mode 100644 index 000000000..c5a92d742 --- /dev/null +++ b/internal/connector/driver/codex/fake_test.go @@ -0,0 +1,248 @@ +//go:build unix + +package codex + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "strings" + "syscall" + "testing" + "time" +) + +// The test binary doubles as a fake `codex`: run with "exec" as its first +// argument, it plays the scenario in $CODEX_HOME/scenario.json instead of +// running tests. Everything it saw (argv, environment, prompt, the MCP +// server's environment) is written beside the scenario. +func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == "exec" { + os.Exit(fakeCodex()) + } + os.Exit(m.Run()) +} + +type scenario struct { + Thread string `json:"thread"` + // TurnContext is written to the rollout as the turn_context payload; + // nil writes none. + TurnContext map[string]any `json:"turn_context"` + // OldTurnContext is written before the prompt is read, as an earlier + // turn of a resumed thread would be. + OldTurnContext map[string]any `json:"old_turn_context"` + // Events are written to stdout after thread.started. + Events []string `json:"events"` + // NoThread skips thread.started. + NoThread bool `json:"no_thread"` + // RunMCP starts each MCP server as Codex would and waits for it. + RunMCP bool `json:"run_mcp"` + // Child starts a child process in the fake's group and records its pid. + Child bool `json:"child"` + // Escape leaves a process of its own, outside the fake's process group, + // holding the fake's stdout. + Escape bool `json:"escape"` + // Stderr is written, slowly, after the events. + Stderr string `json:"stderr"` + // CloseStdout closes stdout before the stderr is written: the reader is + // done with the process well before the process is done. + CloseStdout bool `json:"close_stdout"` + // StderrOnTerm is written to stderr when the process is asked to end, as + // a refusal Codex logs on its way out of a cancel is. + StderrOnTerm string `json:"stderr_on_term"` + // Deaf never reads its stdin: the prompt's write blocks once the pipe + // fills. + Deaf bool `json:"deaf"` + // Hang waits to be killed after the events. + Hang bool `json:"hang"` + // Exit is the exit status. + Exit int `json:"exit"` +} + +type observed struct { + Args []string `json:"args"` + Env []string `json:"env"` + Cwd string `json:"cwd"` + Prompt string `json:"prompt"` + MCPExit int `json:"mcp_exit"` + ChildPID int `json:"child_pid"` + EscapedPID int `json:"escaped_pid"` + Deaf bool `json:"deaf"` +} + +func fakeCodex() int { + home := os.Getenv("CODEX_HOME") + data, err := os.ReadFile(filepath.Join(home, "scenario.json")) + if err != nil { + fmt.Fprintln(os.Stderr, "fake codex: no scenario:", err) + return 2 + } + var sc scenario + if err := json.Unmarshal(data, &sc); err != nil { + fmt.Fprintln(os.Stderr, "fake codex: bad scenario:", err) + return 2 + } + if sc.StderrOnTerm != "" { + ending := make(chan os.Signal, 1) + signal.Notify(ending, syscall.SIGTERM) + go func() { + <-ending + fmt.Fprintln(os.Stderr, sc.StderrOnTerm) + os.Exit(0) + }() + } + obs := observed{Args: os.Args[1:], Env: os.Environ()} + obs.Cwd, _ = os.Getwd() + save := func() { + out, _ := json.Marshal(obs) + _ = os.WriteFile(filepath.Join(home, "observed.json"), out, 0o600) + } + defer save() + + rollout := filepath.Join(home, "sessions", "2026", "09", "17", "rollout-2026-09-17T08-00-00-"+sc.Thread+".jsonl") + _ = os.MkdirAll(filepath.Dir(rollout), 0o700) + if sc.OldTurnContext != nil { + appendRecord(rollout, "turn_context", sc.OldTurnContext) + } + + if sc.Deaf { + obs.Deaf = true + save() + } else { + prompt, _ := io.ReadAll(os.Stdin) + obs.Prompt = string(prompt) + save() + } + + if sc.RunMCP { + for _, server := range mcpServers(os.Args) { + cmd := exec.CommandContext(context.Background(), server.command, server.args...) //nolint:gosec // the fake runs what the driver configured + // As Codex does: a near-empty environment plus the declared env. + cmd.Env = []string{"HOME=" + os.Getenv("HOME"), "PATH=" + os.Getenv("PATH")} + for k, v := range server.env { + cmd.Env = append(cmd.Env, k+"="+v) + } + if err := cmd.Run(); err != nil { + obs.MCPExit = 1 + fmt.Fprintln(os.Stderr, "required MCP servers failed to initialize") + return 1 + } + } + save() + } + + if sc.Escape { + // setsid puts it in a group of its own, and it inherits stdout. + escaped := exec.CommandContext(context.Background(), "setsid", "sleep", "120") + escaped.Stdout = os.Stdout + if err := escaped.Start(); err == nil { + obs.EscapedPID = escaped.Process.Pid + save() + } + } + if sc.Child { + child := exec.CommandContext(context.Background(), "sleep", "300") + if err := child.Start(); err == nil { + obs.ChildPID = child.Process.Pid + save() + } + } + + appendRecord(rollout, "session_meta", map[string]any{"id": sc.Thread}) + if sc.TurnContext != nil { + tc := map[string]any{} + for k, v := range sc.TurnContext { + tc[k] = v + } + if _, ok := tc["cwd"]; !ok { + tc["cwd"] = obs.Cwd + } + raw, _ := json.Marshal(tc) + _ = json.Unmarshal([]byte(strings.ReplaceAll(string(raw), "$CWD", obs.Cwd)), &tc) + appendRecord(rollout, "turn_context", tc) + } + if !sc.NoThread { + fmt.Printf(`{"type":"thread.started","thread_id":%q}`+"\n", sc.Thread) + } + for _, e := range sc.Events { + fmt.Println(e) + } + if sc.CloseStdout { + _ = os.Stdout.Close() + } + if sc.Stderr != "" { + // After the last stdout line, as a sandbox refusal Codex logs is. + time.Sleep(50 * time.Millisecond) + fmt.Fprintln(os.Stderr, sc.Stderr) + } + if sc.Hang { + time.Sleep(5 * time.Minute) + } + return sc.Exit +} + +func appendRecord(path, kind string, payload map[string]any) { + line, _ := json.Marshal(map[string]any{"type": kind, "payload": payload}) + f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o600) + if err != nil { + return + } + _, _ = f.Write(append(line, '\n')) + _ = f.Close() +} + +type fakeServer struct { + command string + args []string + env map[string]string +} + +var tomlPair = regexp.MustCompile(`("(?:[^"\\]|\\.)*")=("(?:[^"\\]|\\.)*")`) + +// mcpServers reads the mcp_servers overrides back from argv. The values are +// the JSON-compatible subset of TOML the driver writes; an env table is +// {"K"="v",...}. +func mcpServers(argv []string) []fakeServer { + servers := map[string]*fakeServer{} + get := func(name string) *fakeServer { + if servers[name] == nil { + servers[name] = &fakeServer{env: map[string]string{}} + } + return servers[name] + } + for i := 0; i+1 < len(argv); i++ { + if argv[i] != "-c" { + continue + } + key, value, _ := strings.Cut(argv[i+1], "=") + rest, ok := strings.CutPrefix(key, "mcp_servers.") + if !ok { + continue + } + name, field, _ := strings.Cut(rest, ".") + switch field { + case "command": + _ = json.Unmarshal([]byte(value), &get(name).command) + case "args": + _ = json.Unmarshal([]byte(value), &get(name).args) + case "env": + for _, m := range tomlPair.FindAllStringSubmatch(value, -1) { + var k, v string + _ = json.Unmarshal([]byte(m[1]), &k) + _ = json.Unmarshal([]byte(m[2]), &v) + get(name).env[k] = v + } + } + } + out := make([]fakeServer, 0, len(servers)) + for _, s := range servers { + out = append(out, *s) + } + return out +} diff --git a/internal/connector/driver/spawn/spawn.go b/internal/connector/driver/spawn/spawn.go index fcfa37802..f1e69f490 100644 --- a/internal/connector/driver/spawn/spawn.go +++ b/internal/connector/driver/spawn/spawn.go @@ -8,6 +8,7 @@ import ( "github.com/basecamp/basecamp-cli/internal/connector/driver" "github.com/basecamp/basecamp-cli/internal/connector/driver/claude" + "github.com/basecamp/basecamp-cli/internal/connector/driver/codex" "github.com/basecamp/basecamp-cli/internal/connector/setup" ) @@ -22,6 +23,7 @@ type Options struct { // adds its row here. var constructors = map[string]func(Options) driver.Driver{ setup.WorkerClaude: func(o Options) driver.Driver { return claude.New(claude.Options{Lookup: o.Lookup}) }, + setup.WorkerCodex: func(o Options) driver.Driver { return codex.New(codex.Options{Lookup: o.Lookup}) }, } // New is the spawn driver for worker. diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index 4f17364c6..2d101e9af 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -823,6 +823,16 @@ END; // acknowledgement trigger before this branch landed, and a shipped // migration is never renumbered under a ledger that has applied it. migrationTasksAndAttempts, + // Migration 8. The git worktrees a task runs in: the row written before + // `git worktree add`, the states it moves through, and the ones kept when + // a task ends holding work. See ledger_worktrees.go. + // + // This was also 8 on the base this branch was written against, but behind + // a different 7 — card 20's outbox, which has not landed. It follows the + // dispatcher's tables at 7 now; the number stayed and the reason for it + // moved. Whatever lands next takes 9: a shipped migration is never + // renumbered under a ledger that has already applied it. + migrationWorktrees, } func (l *Ledger) migrate(ctx context.Context) error { diff --git a/internal/connector/ledger_worktrees.go b/internal/connector/ledger_worktrees.go new file mode 100644 index 000000000..2acc59bf0 --- /dev/null +++ b/internal/connector/ledger_worktrees.go @@ -0,0 +1,373 @@ +package connector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) + +// Worktrees in the ledger: every git worktree the connector made for a task, +// from the moment it decided to make one until it is gone. +// +// A row is written creating before `git worktree add` runs, so a crash at any +// point leaves a row that says a directory may exist; live once the worktree +// is there; retained, with a reason, when the task ended and the worktree +// held work that was not safe to remove; removing while a removal holds the +// worktrees lock; removed at the end, with who removed it (straight from any +// open state when the directory is found gone). The states move along those +// edges only, held by a trigger. +const migrationWorktrees = ` +CREATE TABLE worktrees ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + work_dir TEXT NOT NULL, + route TEXT NOT NULL, + repository TEXT NOT NULL, + branch TEXT NOT NULL, + base_commit TEXT NOT NULL, + originating_event_id INTEGER NOT NULL, + branch_created INTEGER NOT NULL DEFAULT 0, + admin_dir TEXT NOT NULL DEFAULT '', + task_id INTEGER REFERENCES tasks (id), + state TEXT NOT NULL + CHECK (state IN ('creating', 'live', 'retained', 'removing', 'removed')), + retained_reason TEXT NOT NULL DEFAULT '' + CHECK (retained_reason IN ('', 'dirty', 'unpushed', 'locked', 'moved', 'unverified', 'finished', 'orphaned')), + created_at TEXT NOT NULL, + finished_at TEXT, + retained_at TEXT, + removed_at TEXT, + removed_by TEXT NOT NULL DEFAULT '' + CHECK (removed_by IN ('', 'prune', 'prune_forced', 'missing', 'never_created')), + CHECK (state <> 'retained' OR retained_reason <> ''), + CHECK ((state = 'removed') = (removed_by <> '')) +); +CREATE UNIQUE INDEX worktrees_open_path ON worktrees (path) WHERE state <> 'removed'; +CREATE UNIQUE INDEX worktrees_open_work_dir ON worktrees (work_dir) WHERE state <> 'removed'; +CREATE INDEX worktrees_state ON worktrees (state); + +CREATE TRIGGER worktrees_state_edges +BEFORE UPDATE OF state ON worktrees +WHEN NEW.state <> OLD.state AND NOT ( + (OLD.state = 'creating' AND NEW.state IN ('live', 'retained', 'removing', 'removed')) + OR (OLD.state = 'live' AND NEW.state IN ('retained', 'removing', 'removed')) + OR (OLD.state = 'retained' AND NEW.state IN ('removing', 'removed')) + OR (OLD.state = 'removing' AND NEW.state IN ('retained', 'removed'))) +BEGIN + SELECT RAISE(ABORT, 'a worktree state moves along its edges only'); +END; +` + +// WorktreeState is where a task's worktree is. +type WorktreeState string + +const ( + WorktreeCreating WorktreeState = "creating" + WorktreeLive WorktreeState = "live" + WorktreeRetained WorktreeState = "retained" + WorktreeRemoving WorktreeState = "removing" + WorktreeRemoved WorktreeState = "removed" +) + +// RetainedReason is why a worktree was kept. +type RetainedReason string + +const ( + // RetainedDirty is uncommitted work: modified or untracked files, or a + // merge, rebase, cherry-pick, revert or bisect in progress. + RetainedDirty RetainedReason = "dirty" + // RetainedUnpushed is a commit no remote branch and no other local branch + // holds. + RetainedUnpushed RetainedReason = "unpushed" + // RetainedLocked is a worktree someone locked with `git worktree lock`. + RetainedLocked RetainedReason = "locked" + // RetainedUnverified is a worktree whose state could not be read. It is + // kept, because a check that failed proves nothing is safe to delete. + RetainedUnverified RetainedReason = "unverified" + // RetainedMoved is a worktree that is no longer where the ledger says: + // someone moved it, and its files are theirs to deal with. + RetainedMoved RetainedReason = "moved" + // RetainedOrphaned is a worktree whose directory something outside the + // connector removed. Git's record of it and the task branch are still + // there, reaching whatever they reach; the connector neither judges that + // nor deletes any of it. An operator's explicit discard does, and is + // told what goes. + RetainedOrphaned RetainedReason = "orphaned" + // RetainedFinished is a worktree whose task ended. Nothing the connector + // does removes a worktree, so this is why most kept worktrees are kept: + // the work is done with, and an operator says when it goes. + RetainedFinished RetainedReason = "finished" +) + +// RemovedBy is who removed a worktree. +type RemovedBy string + +const ( + // There is no connector: nothing the connector does of its own accord + // removes a worktree. + RemovedByPrune RemovedBy = "prune" + RemovedByPruneForced RemovedBy = "prune_forced" + RemovedMissing RemovedBy = "missing" + RemovedNeverCreated RemovedBy = "never_created" +) + +// Worktree is a worktree's ledger row. +type Worktree struct { + ID int64 + // Path is the worktree's root; WorkDir is where the task worked in it, + // the route's place inside the repository. + Path string + WorkDir string + Route string + Repository string + Branch string + BaseCommit string + OriginatingEventID int64 + // BranchCreated is this row's proof that the connector made the task + // branch, so deleting it can never delete someone else's. + BranchCreated bool + // AdminDir is the repository's own record of this worktree + // (/.git/worktrees/), which says where it is even after + // someone moves it or changes what it has checked out. + AdminDir string + // TaskID is the task that last worked in it; zero before one launched. + TaskID int64 + State WorktreeState + RetainedReason RetainedReason + CreatedAt time.Time + FinishedAt time.Time + RetainedAt time.Time + RemovedAt time.Time + RemovedBy RemovedBy +} + +const worktreeColumns = `id, path, work_dir, route, repository, branch, base_commit, originating_event_id, branch_created, admin_dir, COALESCE(task_id, 0), +state, retained_reason, created_at, finished_at, retained_at, removed_at, removed_by` + +func scanWorktree(row interface{ Scan(...any) error }) (Worktree, error) { + var ( + w Worktree + state, reason, removedBy, created string + finished, retained, removed sql.NullString + ) + if err := row.Scan(&w.ID, &w.Path, &w.WorkDir, &w.Route, &w.Repository, &w.Branch, &w.BaseCommit, &w.OriginatingEventID, &w.BranchCreated, &w.AdminDir, &w.TaskID, + &state, &reason, &created, &finished, &retained, &removed, &removedBy); err != nil { + return Worktree{}, err + } + w.State, w.RetainedReason, w.RemovedBy = WorktreeState(state), RetainedReason(reason), RemovedBy(removedBy) + var err error + if w.CreatedAt, err = parseStamp(created); err != nil { + return Worktree{}, err + } + for _, f := range []struct { + src sql.NullString + dst *time.Time + }{{finished, &w.FinishedAt}, {retained, &w.RetainedAt}, {removed, &w.RemovedAt}} { + if f.src.Valid { + if *f.dst, err = parseStamp(f.src.String); err != nil { + return Worktree{}, err + } + } + } + return w, nil +} + +// ErrWorktreeState is a worktree transition from a state it cannot leave that +// way, or for a row that is not there. +var ErrWorktreeState = errors.New("the worktree is not in a state that allows this") + +// BeginWorktree records a worktree about to be created. Nothing is on disk +// yet. +func (l *Ledger) BeginWorktree(ctx context.Context, w Worktree) (int64, error) { + if w.Path == "" || w.WorkDir == "" || w.Route == "" || w.Repository == "" || w.Branch == "" || w.BaseCommit == "" { + return 0, errors.New("connector: a worktree needs its path, working directory, route, repository, branch and base commit") + } + var id int64 + err := retryBusy(func() error { + res, err := l.db.ExecContext(ctx, ` +INSERT INTO worktrees (path, work_dir, route, repository, branch, base_commit, originating_event_id, state, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, 'creating', ?)`, + w.Path, w.WorkDir, w.Route, w.Repository, w.Branch, w.BaseCommit, w.OriginatingEventID, l.timestamp()) + if err != nil { + return fmt.Errorf("connector: record worktree %s: %w", w.Path, err) + } + id, err = res.LastInsertId() + return err + }) + return id, err +} + +// WorktreeBranchCreated records that the connector created the task branch +// for a worktree, which is what lets it be deleted again. +func (l *Ledger) WorktreeBranchCreated(ctx context.Context, id int64) error { + return retryBusy(func() error { + _, err := l.db.ExecContext(ctx, `UPDATE worktrees SET branch_created = 1 WHERE id = ?`, id) + if err != nil { + return fmt.Errorf("connector: worktree %d: %w", id, err) + } + return nil + }) +} + +// WorktreeAdminDir records the repository's directory for a worktree. +func (l *Ledger) WorktreeAdminDir(ctx context.Context, id int64, dir string) error { + return retryBusy(func() error { + _, err := l.db.ExecContext(ctx, `UPDATE worktrees SET admin_dir = ? WHERE id = ?`, dir, id) + if err != nil { + return fmt.Errorf("connector: worktree %d: %w", id, err) + } + return nil + }) +} + +// MoveWorktree moves a worktree from one of from to state. It reports +// ErrWorktreeState when the row is in none of them. +func (l *Ledger) MoveWorktree(ctx context.Context, id int64, state WorktreeState, from ...WorktreeState) error { + return l.moveWorktree(ctx, id, state, "", "", from) +} + +// RetainWorktree keeps a worktree, with the reason, from one of from. +func (l *Ledger) RetainWorktree(ctx context.Context, id int64, reason RetainedReason, from ...WorktreeState) error { + if reason == "" { + return errors.New("connector: a retained worktree needs a reason") + } + return l.moveWorktree(ctx, id, WorktreeRetained, reason, "", from) +} + +// RemovedWorktree records a worktree gone, and by whom, from one of from. +func (l *Ledger) RemovedWorktree(ctx context.Context, id int64, by RemovedBy, from ...WorktreeState) error { + if by == "" { + return errors.New("connector: a removed worktree needs who removed it") + } + return l.moveWorktree(ctx, id, WorktreeRemoved, "", by, from) +} + +func (l *Ledger) moveWorktree(ctx context.Context, id int64, state WorktreeState, reason RetainedReason, by RemovedBy, from []WorktreeState) error { + if len(from) == 0 { + return errors.New("connector: a worktree transition names the states it leaves") + } + return retryBusy(func() error { + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("connector: begin worktree update: %w", err) + } + defer func() { _ = tx.Rollback() }() + var ( + current, workDir string + finished sql.NullString + ) + switch err := tx.QueryRowContext(ctx, `SELECT state, work_dir, finished_at FROM worktrees WHERE id = ?`, id).Scan(¤t, &workDir, &finished); { + case errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("connector: worktree %d: %w", id, ErrWorktreeState) + case err != nil: + return fmt.Errorf("connector: worktree %d: %w", id, err) + } + allowed := false + for _, f := range from { + allowed = allowed || WorktreeState(current) == f + } + if !allowed { + return fmt.Errorf("connector: worktree %d is %s: %w", id, current, ErrWorktreeState) + } + now := l.timestamp() + // The task that last worked in the directory, for status. + var taskID sql.NullInt64 + if err := tx.QueryRowContext(ctx, `SELECT MAX(id) FROM tasks WHERE work_dir = ?`, workDir).Scan(&taskID); err != nil { + return fmt.Errorf("connector: worktree %d: %w", id, err) + } + switch state { + case WorktreeRetained: + _, err = tx.ExecContext(ctx, ` +UPDATE worktrees SET state = 'retained', retained_reason = ?, retained_at = ?, finished_at = COALESCE(finished_at, ?), + task_id = COALESCE(?, task_id) WHERE id = ?`, string(reason), now, now, taskID, id) + case WorktreeRemoved: + _, err = tx.ExecContext(ctx, ` +UPDATE worktrees SET state = 'removed', removed_by = ?, removed_at = ?, finished_at = COALESCE(finished_at, ?), + task_id = COALESCE(?, task_id) WHERE id = ?`, string(by), now, now, taskID, id) + case WorktreeRemoving: + _, err = tx.ExecContext(ctx, ` +UPDATE worktrees SET state = 'removing', finished_at = COALESCE(finished_at, ?), task_id = COALESCE(?, task_id) WHERE id = ?`, now, taskID, id) + default: + _, err = tx.ExecContext(ctx, `UPDATE worktrees SET state = ? WHERE id = ?`, string(state), id) + } + if err != nil { + return fmt.Errorf("connector: worktree %d to %s: %w", id, state, err) + } + return tx.Commit() + }) +} + +// WorktreeByWorkDir is the open (not removed) worktree a task works in. +func (l *Ledger) WorktreeByWorkDir(ctx context.Context, workDir string) (Worktree, bool, error) { + row := l.db.QueryRowContext(ctx, `SELECT `+worktreeColumns+` FROM worktrees WHERE work_dir = ? AND state <> 'removed'`, workDir) + w, err := scanWorktree(row) + switch { + case errors.Is(err, sql.ErrNoRows): + return Worktree{}, false, nil + case err != nil: + return Worktree{}, false, fmt.Errorf("connector: worktree for %s: %w", workDir, err) + } + return w, true, nil +} + +// Worktrees lists worktrees in the given states, oldest first; every state +// when none is given. +func (l *Ledger) Worktrees(ctx context.Context, states ...WorktreeState) ([]Worktree, error) { + query := `SELECT ` + worktreeColumns + ` FROM worktrees` + var args []any + if len(states) > 0 { + query += ` WHERE state IN (` + for i, s := range states { + if i > 0 { + query += `, ` + } + query += `?` + args = append(args, string(s)) + } + query += `)` + } + rows, err := l.db.QueryContext(ctx, query+` ORDER BY id`, args...) + if err != nil { + return nil, fmt.Errorf("connector: list worktrees: %w", err) + } + defer func() { _ = rows.Close() }() + var out []Worktree + for rows.Next() { + w, err := scanWorktree(rows) + if err != nil { + return nil, fmt.Errorf("connector: list worktrees: %w", err) + } + out = append(out, w) + } + return out, rows.Err() +} + +// RetainedWorktrees are the worktrees kept for a person to deal with. +func (l *Ledger) RetainedWorktrees(ctx context.Context) ([]Worktree, error) { + return l.Worktrees(ctx, WorktreeRetained) +} + +// UnfinishedWorktrees are worktrees a crash left between their creation and +// their task's end: creating, live or removing, with no live task working in +// them. +func (l *Ledger) UnfinishedWorktrees(ctx context.Context) ([]Worktree, error) { + rows, err := l.db.QueryContext(ctx, `SELECT `+worktreeColumns+` FROM worktrees w +WHERE state IN ('creating', 'live', 'removing') + AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.ended_at IS NULL AND t.work_dir = w.work_dir) +ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("connector: unfinished worktrees: %w", err) + } + defer func() { _ = rows.Close() }() + var out []Worktree + for rows.Next() { + w, err := scanWorktree(rows) + if err != nil { + return nil, fmt.Errorf("connector: unfinished worktrees: %w", err) + } + out = append(out, w) + } + return out, rows.Err() +} diff --git a/internal/connector/setup/file.go b/internal/connector/setup/file.go index 74a3b7a76..270269af9 100644 --- a/internal/connector/setup/file.go +++ b/internal/connector/setup/file.go @@ -59,11 +59,12 @@ const ( // Workers: the coding agent a driver runs. const ( WorkerClaude = "claude" + WorkerCodex = "codex" ) // Workers is every worker connect.json may name. A worker is a row here plus // its spawn constructor (internal/connector/driver/spawn). -var Workers = []string{WorkerClaude} +var Workers = []string{WorkerClaude, WorkerCodex} // Defaults, from the connector spec. const ( diff --git a/internal/connector/worktrees.go b/internal/connector/worktrees.go new file mode 100644 index 000000000..82fc906a1 --- /dev/null +++ b/internal/connector/worktrees.go @@ -0,0 +1,1904 @@ +package connector + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/basecamp/basecamp-cli/internal/connector/driver" + "github.com/basecamp/basecamp-cli/internal/connector/setup" +) + +// Worktrees is --worktrees: each task works in a git worktree of its own, +// branched from the route's HEAD, so tasks on one repository run side by +// side. When the task ends its worktree is kept, recorded in the ledger and +// listed by `basecamp connect status` and `basecamp connect worktrees list`, +// until an operator discards it with `basecamp connect worktrees prune`. +// +// # One worktree, one removal +// +// WHEN. Only an operator's `worktrees prune` removes a worktree. The +// connector never removes one of its own accord: a task's end (Finish, which +// the dispatcher calls at its release point, after the task has ended and +// ConfirmGroupGone has confirmed the worker's process group gone) and a +// start's recovery (Recover, before anything is dispatched) only ever keep +// it, whatever is in it. Prune touches only worktrees no live task holds, +// holds the worktrees lock, and goes through removeWorktree. Nothing else in +// the connector deletes a worktree's directory or git's record of it +// (/.git/worktrees/), and nothing runs `git worktree remove`. +// +// A worktree whose directory something outside the connector removed is a +// case of its own: what is left — git's record and the task branch — reaches +// whatever it reaches, and the connector neither judges that nor deletes any +// of it. The row is kept, said to be orphaned, and listed with the record, so +// an operator sees it; naming its path in a force is what deletes the branch, +// and git's own `worktree prune` is what clears the record. Nothing about +// reachability is decided on that path at all. +// +// WHAT is work. Anything on the disk that is not a tracked file, unchanged: +// a modified, staged, untracked or ignored file, a directory git has no file +// in, anything inside a submodule's directory, an index entry that hides an +// edit. An operation in progress (merge, rebase, cherry-pick, revert, +// bisect). A lock someone set. A submodule's git data. And every commit the +// worktree reaches — HEAD, the task branch, their reflogs, per-worktree refs +// (refs/worktree, refs/bisect, refs/rewritten) — that no ref the connector +// keeps holds, a kept ref being a remote branch, a local branch that is not a +// task's, or a ref a forced removal of this worktree kept it under. The commit +// the worktree was made from is one of those commits: the route's branch +// usually holds it, and a route reset since is not evidence that it does. A +// stash +// is in refs/stash, which belongs to the repository and is never touched. +// +// WHAT happens to work. The connector never discards it. A prune without an +// operator's force keeps the worktree, with its reason, and lists it. With +// the force, every commit the worktree reaches that nothing holds is first +// kept under refs/basecamp-connect/retained//; +// a worktree whose work cannot be kept that way (submodule git data, a HEAD +// that cannot be read) is not removed. +// +// HOW the check holds until the removal. removeWorktree freezes the worktree +// before it judges anything: it locks git's record of it (as `git worktree +// lock` does, so git's own prune leaves the frozen record alone), renames the +// record and then the directory aside, each an atomic rename. From then on no git command can +// move its HEAD or commit in it (its .git file names a record that is not +// there), and nothing that reaches it by path can write to it. The evidence +// is judged on the frozen copy, and the frozen copy is what is deleted — or +// both names are restored and the worktree retained. What the judgment leans +// on outside the frozen copy — the refs that hold the commits it reaches — is +// verified again where it was found, in the one ref transaction that ends the +// task branch, immediately before the frozen copy is deleted: a fetch, a +// reset or a deleted branch since makes git refuse the transaction, and the +// worktree is kept instead. A crash while frozen leaves a removing row, and +// the next start restores the names and judges again. The one writer outside the rule is a process that escaped the +// task's process group and holds a descriptor inside the directory. +// +// WHO forces. Only an operator, naming the worktree's path in `basecamp +// connect worktrees prune --force `. A force is a decision about work, +// not a judgment of it: it is the one thing that goes ahead where the rule +// above would keep a worktree, and what it can find is kept under refs first. +// +// # Invariants +// +// Each is held by a test in worktrees_test.go. +// +// 1. The rule above, the first half of it being that a task's end and a +// start's recovery keep every worktree they find. +// 2. The ledger first. A worktree is recorded creating before `git worktree +// add` runs, and removing before it is frozen, so a crash at any point +// leaves a row that says where a directory may be. +// 3. Nothing the repository, its configuration or a worker's files name +// runs: git never looks inside a submodule's directory (the disk is judged +// before git is asked anything that could recurse, and status ignores +// submodules), and every git call runs with hooks, the fsmonitor, +// signature verification and every content filter its configuration +// defines disabled, with a fixed environment. +// 4. A task branch is deleted only if this connector created it, and only in +// one ref transaction that deletes it at the commit judged held and +// verifies that every ref holding a commit the worktree reaches is still +// where the judgment found it. +// +// Placement goes through Options.Path, one function, because under the +// sandbox launcher (step 26) the working directory comes from broker-owned +// scopes instead. +type Worktrees struct { + ledger *Ledger + root string + git string + env []string + path func(root, repository, name string) string + log *slog.Logger + // red takes the connector's own paths and environment out of anything git + // says (the shared rule in driver/redact.go). + red *driver.Redactor + // walkLimit is WalkLimit; a test seam. + walkLimit time.Duration + // whileFrozen runs once a removal has frozen a worktree, before it is + // judged; a test seam. An error leaves it frozen, as a crash would. + whileFrozen func(dir string) error + now func() time.Time + + // Off leaves new tasks in their route; see WorktreesOptions.Off. + off bool + + mu sync.Mutex + failures map[string]prepareFailure +} + +var _ WaitingWorkspaces = (*Worktrees)(nil) + +// prepareFailure is a route that could not take a worktree, and when to try +// it again. +type prepareFailure struct { + count int + until time.Time +} + +// Prepare's backoff after a failure: doubling from the first, capped. +const ( + PrepareBackoff = time.Minute + PrepareBackoffMax = 30 * time.Minute +) + +// ErrPrepareBackoff is a Prepare on a route whose last worktree failed too +// recently to try again. +var ErrPrepareBackoff = errors.New("the last worktree on this route failed; waiting before trying again") + +// WorktreesOptions configures Worktrees. +type WorktreesOptions struct { + Ledger *Ledger + // Root is the owner-only directory worktrees are placed under: the + // connector state directory's worktrees/. + Root string + // Git is the git binary; "git" on PATH when empty. + Git string + // Lookup reads the connector's environment for git's; os.LookupEnv when + // nil. + Lookup func(string) (string, bool) + // Redaction is what is taken out of anything git says: the driver + // package's shared rule. + Redaction driver.Redaction + // Path places a task's worktree; DefaultWorktreePath when nil. + Path func(root, repository, name string) string + Logger *slog.Logger + // Off gives new tasks no worktree: they work in the route itself. The + // worktrees made while it was on are still settled and recovered, so + // switching worktrees off never strands one. + Off bool +} + +var ( + _ PerTaskWorkspaces = (*Worktrees)(nil) + _ RecoveringWorkspaces = (*Worktrees)(nil) +) + +// BranchPrefix names every task branch, so a task branch is never evidence +// that another task's commits are safe. +const BranchPrefix = "basecamp-connect/" + +// NewWorktrees builds Worktrees. +func NewWorktrees(opts WorktreesOptions) (*Worktrees, error) { + if opts.Ledger == nil || opts.Root == "" || !filepath.IsAbs(opts.Root) { + return nil, errors.New("connector: worktrees need the ledger and an absolute root") + } + if opts.Git == "" { + opts.Git = "git" + } + if opts.Lookup == nil { + opts.Lookup = os.LookupEnv + } + if opts.Path == nil { + opts.Path = DefaultWorktreePath + } + if opts.Logger == nil { + opts.Logger = slog.New(slog.DiscardHandler) + } + env := driver.BuildEnv(driver.BaseEnv, opts.Lookup, map[string]string{ + // Never ask anyone anything, never take an optional lock a person's + // own git in the checkout would then wait on. + "GIT_TERMINAL_PROMPT": "0", + "GIT_OPTIONAL_LOCKS": "0", + "LC_ALL": "C", + }) + return &Worktrees{ + ledger: opts.Ledger, root: opts.Root, git: opts.Git, env: env, path: opts.Path, log: opts.Logger, + now: time.Now, off: opts.Off, walkLimit: WalkLimit, + red: driver.NewRedactor(opts.Redaction.With(driver.Redaction{Env: env, Dirs: []string{opts.Root}})), failures: map[string]prepareFailure{}, + }, nil +} + +// DefaultWorktreePath places a worktree under the connector's state +// directory, one directory per repository: never inside the checkout, where a +// task working in the route itself could edit another task's retained work, +// and `git add -A` in the checkout would pick it up. +func DefaultWorktreePath(root, repository, name string) string { + sum := sha256.Sum256([]byte(repository)) + return filepath.Join(root, safeName(filepath.Base(repository))+"-"+hex.EncodeToString(sum[:4]), name) +} + +var unsafeNameRunes = regexp.MustCompile(`[^A-Za-z0-9._-]+`) + +func safeName(s string) string { + s = unsafeNameRunes.ReplaceAllString(s, "-") + s = strings.Trim(s, ".-") + if len(s) > 40 { + s = s[:40] + } + if s == "" { + return "repo" + } + return s +} + +// PerTaskDirs implements PerTaskWorkspaces. +func (w *Worktrees) PerTaskDirs() bool { return !w.off } + +// Prepare implements Workspaces: a new worktree on a new task branch at the +// route's HEAD, and the route's place inside it. +// +// A failure holds the route, not the event: what stops a worktree (a route +// that is not a repository, one with no commit, a full disk) stops every +// event on it. The route waits PrepareBackoff, doubling up to +// PrepareBackoffMax, and RoutesWaiting tells the dispatcher to leave its +// records out, so they neither fill the disk and the ledger nor the window +// other routes' records are started from. +func (w *Worktrees) Prepare(ctx context.Context, route string, originatingEventID int64) (string, error) { + if w.off { + return route, nil + } + w.mu.Lock() + failure, failed := w.failures[route] + w.mu.Unlock() + if failed && w.now().Before(failure.until) { + return "", fmt.Errorf("connector: event %d on %s: %w", originatingEventID, route, ErrPrepareBackoff) + } + workDir, err := w.prepare(ctx, route, originatingEventID) + w.mu.Lock() + defer w.mu.Unlock() + if err != nil { + failure.count++ + delay := PrepareBackoff << min(failure.count-1, 10) + failure.until = w.now().Add(min(delay, PrepareBackoffMax)) + w.failures[route] = failure + return "", err + } + delete(w.failures, route) + return workDir, nil +} + +// RoutesWaiting implements WaitingWorkspaces: the routes still in a Prepare +// backoff. +func (w *Worktrees) RoutesWaiting() []string { + w.mu.Lock() + defer w.mu.Unlock() + now := w.now() + var out []string + for route, f := range w.failures { + if now.Before(f.until) { + out = append(out, route) + } + } + return out +} + +func (w *Worktrees) prepare(ctx context.Context, route string, originatingEventID int64) (string, error) { + if !filepath.IsAbs(route) { + return "", fmt.Errorf("connector: route %q is not absolute", route) + } + top, err := w.gitOut(ctx, route, "rev-parse", "--show-toplevel") + if err != nil { + return "", fmt.Errorf("connector: route %s is not in a git repository: %w", route, err) + } + repository := filepath.Clean(top) + rel, err := filepath.Rel(realPath(repository), realPath(route)) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("connector: route %s is not inside its repository", route) + } + base, err := w.gitOut(ctx, repository, "rev-parse", "--verify", "--end-of-options", "HEAD^{commit}") + if err != nil { + return "", fmt.Errorf("connector: route %s has no commit to branch from: %w", route, err) + } + suffix := make([]byte, 3) + if _, err := rand.Read(suffix); err != nil { + return "", err + } + name := strconv.FormatInt(originatingEventID, 10) + "-" + hex.EncodeToString(suffix) + path := w.path(w.root, repository, name) + if !filepath.IsAbs(path) { + return "", fmt.Errorf("connector: worktree path %q is not absolute", path) + } + workDir := filepath.Join(path, rel) + record := Worktree{ + Path: path, WorkDir: workDir, Route: route, Repository: repository, + Branch: BranchPrefix + name, BaseCommit: base, OriginatingEventID: originatingEventID, + State: WorktreeCreating, + } + id, err := w.ledger.BeginWorktree(ctx, record) + if err != nil { + return "", err + } + record.ID = id + + err = w.add(ctx, &record) + if err == nil { + record.AdminDir, err = w.gitOut(ctx, record.Path, "rev-parse", "--absolute-git-dir") + } + if err == nil { + err = w.ledger.WorktreeAdminDir(ctx, id, record.AdminDir) + } + if err == nil { + err = w.ledger.MoveWorktree(ctx, id, WorktreeLive, WorktreeCreating) + } + if err != nil { + // Whatever git left is judged like any finished worktree; a lock not + // had leaves the row for the next start. + settleCtx := context.WithoutCancel(ctx) + if unlock, lockErr := w.lock(settleCtx); lockErr == nil { + w.settle(settleCtx, record) + unlock() + } + return "", fmt.Errorf("connector: create a worktree for event %d: %w", originatingEventID, err) + } + return workDir, nil +} + +func (w *Worktrees) add(ctx context.Context, r *Worktree) error { + if err := os.MkdirAll(w.root, 0o700); err != nil { + return err + } + if err := setup.EnsurePrivateDir(filepath.Dir(r.Path)); err != nil { + return err + } + // The branch is created before the worktree and only if it does not + // exist, so the row's branch is this task's and deleting it later can + // never delete a branch someone else made (invariant 1). + if _, err := w.gitOut(ctx, r.Repository, "update-ref", "--end-of-options", "refs/heads/"+r.Branch, r.BaseCommit, ""); err != nil { + return err + } + if err := w.ledger.WorktreeBranchCreated(ctx, r.ID); err != nil { + return err + } + // The caller settles this record if anything below fails, and only a + // record that says the branch is ours lets it be deleted again. + r.BranchCreated = true + // The checkout runs in the new worktree, so the filters blanked are the + // ones its own configuration defines (an include on its branch among + // them), not the checkout's the route is in. + if _, err := w.gitOut(ctx, r.Repository, "worktree", "add", "--no-checkout", "--end-of-options", r.Path, r.Branch); err != nil { + return err + } + _, err := w.gitOut(ctx, r.Path, "reset", "--quiet", "--hard", "--end-of-options", r.BaseCommit) + return err +} + +// Finish implements Workspaces: the worktree a task worked in is kept, +// whatever is in it, and recorded as kept so `worktrees list` shows it and a +// prune can judge it. Nothing here removes anything. A directory that is not +// one of this connector's worktrees is left alone. +func (w *Worktrees) Finish(ctx context.Context, _ string, workDir string) error { + record, ok, err := w.ledger.WorktreeByWorkDir(ctx, workDir) + if err != nil || !ok { + return err + } + if record.State != WorktreeCreating && record.State != WorktreeLive { + return nil + } + unlock, err := w.lock(ctx) + if err != nil { + // The worktree is kept either way, but it is said to be kept: a row + // left live is a directory `worktrees list` does not show and no + // prune touches until the next start settles it. + return errors.Join(err, w.keepUnjudged(ctx, record)) + } + defer unlock() + if after := w.settle(ctx, record); after.State != WorktreeRetained && after.State != WorktreeRemoved { + // The worktree is where it was; the ledger could not say so, and the + // row is not one `worktrees list` shows or a prune touches. The next + // start settles it. + return fmt.Errorf("connector: worktree %s is kept, but the ledger could not record it; the next start does", record.Path) + } + return nil +} + +// keepUnjudged retains a worktree the connector could not judge, so the +// operator sees it in `worktrees list` and a prune judges it later. +func (w *Worktrees) keepUnjudged(ctx context.Context, r Worktree) error { + // Waiting for the lock is what used the caller's deadline up: the row is + // still recorded, on a deadline of its own. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := w.ledger.RetainWorktree(ctx, r.ID, RetainedUnverified, r.State); err != nil { + return fmt.Errorf("connector: worktree %s is kept, but the ledger could not record it; the next start does: %w", r.Path, err) + } + w.log.Info("connector: worktree retained", "path", r.Path, "branch", r.Branch, "reason", string(RetainedUnverified)) + return nil +} + +// Recover implements RecoveringWorkspaces: every worktree a crash left +// creating, live or removing with no live task in it is kept and recorded as +// kept, as a finished task's is, after a removal the crash interrupted has +// its names restored. It removes nothing. It runs in the connector that holds +// the instance lock, before anything is dispatched. +func (w *Worktrees) Recover(ctx context.Context) error { + unlock, err := w.lock(ctx) + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + // A prune holding the lock does not keep the connector from starting: + // what Recover would settle is no task's, nothing is dispatched into + // it, and the next start settles it. + w.log.Warn("connector: worktrees are locked by another process; recovery left for the next start") + return nil + } + if err != nil { + return err + } + defer unlock() + records, err := w.ledger.UnfinishedWorktrees(ctx) + if err != nil { + return err + } + var unrecorded []string + for _, r := range records { + if after := w.settle(ctx, r); after.State != WorktreeRetained && after.State != WorktreeRemoved { + unrecorded = append(unrecorded, r.Path) + } + } + if len(unrecorded) > 0 { + // Nothing was deleted — recovery deletes nothing — but the ledger + // does not say where these worktrees are, so nothing lists them and + // no prune touches them. Starting on that is starting blind. + return fmt.Errorf("connector: %d worktree(s) could not be recorded: %s", len(unrecorded), strings.Join(unrecorded, ", ")) + } + return nil +} + +// Retained lists the worktrees kept for the operator: those retained, and +// those a removal left mid-flight, which hold work until a start or a prune +// judges them again. +func (w *Worktrees) Retained(ctx context.Context) ([]Worktree, error) { + return w.ledger.Worktrees(ctx, WorktreeRetained, WorktreeRemoving) +} + +// PruneAction is what prune did with one retained worktree. +type PruneAction string + +const ( + PruneRemoved PruneAction = "removed" + PruneForced PruneAction = "forced" + PruneMissing PruneAction = "missing" + PruneKept PruneAction = "kept" +) + +// PruneResult is one retained worktree after prune. +type PruneResult struct { + Worktree Worktree + Action PruneAction + // Reason is why a kept worktree was kept. + Reason RetainedReason + // ForceRefused is a --force that could not go through: the worktree's + // work could not be kept by refs. + ForceRefused bool + // RetainedRefs are the refs a forced removal kept commits under. + RetainedRefs []string + // BranchDeletedAt is the commit the task branch stood at when a force on + // an orphaned worktree deleted it. Nothing judged what that branch + // reached, so this is what an operator needs to put it back + // (`git branch `) before git's own prune clears the + // record. + BranchDeletedAt string +} + +// RetainedRefPrefix names the refs a forced removal keeps commits under: the +// operator is told about each one, and nothing here deletes them. +const RetainedRefPrefix = "refs/basecamp-connect/retained/" + +// RemovingRefPrefix names the refs a removal holds a worktree's commits under +// while it deletes it. They are the connector's own bookkeeping, let go when +// the removal is over, and never counted as holding a commit for anybody: one +// a crash left behind holds its commits without making the next judgment +// think someone else does. +const RemovingRefPrefix = "refs/basecamp-connect/removing/" + +// ErrNotRetained is a --force naming a path that is no retained worktree. +var ErrNotRetained = errors.New("not a retained worktree") + +// Prune removes the retained worktrees the operator has dealt with: those now +// clean, with every commit they reach held elsewhere. It is the only thing +// that removes a worktree. One still holding work is kept unless its path is +// in force, and so is one whose directory something else removed — that row +// is kept as orphaned, with git's record and the task branch left as they +// are, until its path is in force. A path in force that is no retained +// worktree refuses the whole prune before anything is removed. +func (w *Worktrees) Prune(ctx context.Context, force []string) ([]PruneResult, error) { + unlock, err := w.lock(ctx) + if err != nil { + return nil, err + } + defer unlock() + records, err := w.ledger.Worktrees(ctx, WorktreeRetained, WorktreeRemoving) + if err != nil { + return nil, err + } + forced := map[string]bool{} + for _, p := range force { + clean := filepath.Clean(p) + if !slices.ContainsFunc(records, func(r Worktree) bool { return r.Path == clean }) { + return nil, fmt.Errorf("connector: %s: %w", p, ErrNotRetained) + } + forced[clean] = true + } + out := make([]PruneResult, 0, len(records)) + for _, r := range records { + out = append(out, w.pruneOne(ctx, r, forced[r.Path])) + } + return out, nil +} + +func (w *Worktrees) pruneOne(ctx context.Context, r Worktree, force bool) PruneResult { + var refs []string + by := RemovedByPrune + if force { + by = RemovedByPruneForced + } + var at string + after := w.settleKeeping(ctx, r, by, force, &refs, &at) + result := PruneResult{Worktree: after, RetainedRefs: refs, BranchDeletedAt: at} + gone := after.State == WorktreeRemoving && !exists(after.Path) && !exists(frozenName(after.Path)) + switch { + case force && (after.State == WorktreeRemoved || gone): + result.Action = PruneForced + case after.State == WorktreeRemoved && after.RemovedBy == RemovedMissing: + result.Action = PruneMissing + case after.State == WorktreeRemoved || gone: + // Removing and gone is a removal the ledger could not record yet. + result.Action = PruneRemoved + default: + result.Action, result.Reason = PruneKept, after.RetainedReason + // A moved worktree has nothing here to force; anything else kept + // under a force is a force refused. + result.ForceRefused = force && after.RetainedReason != RetainedMoved + } + return result +} + +// settle is what the connector does with a worktree of its own accord, at the +// end of a task and at recovery: it keeps it. Nothing the connector does +// removes a worktree — only an operator's `worktrees prune` does — so this +// restores a removal a crash left frozen, reconciles a row whose directory is +// no longer there, and otherwise retains the row for the operator. The caller +// holds the lock. It returns the row as it now stands. +func (w *Worktrees) settle(ctx context.Context, r Worktree) Worktree { + from := []WorktreeState{r.State} + if restored, ok := w.unfreeze(r); !ok { + w.log.Warn("connector: a frozen worktree could not be restored; kept", "path", r.Path) + return w.retain(ctx, r, RetainedUnverified, from) + } else if restored { + w.log.Info("connector: restored a worktree a removal left frozen", "path", r.Path) + } + if !exists(r.Path) { + return w.forget(ctx, r, from, nil, nil) + } + return w.retain(ctx, r, RetainedFinished, from) +} + +// forget reconciles a row whose worktree is not on disk: something outside +// the connector removed the directory. What is left is git's record of the +// worktree and the task branch, which reach whatever they reach. The +// connector does not judge that and does not delete any of it — that is the +// class of defect this stopped trying to get right — so the row is kept, +// said to be orphaned, and listed with its record and the refs in it. Only an +// operator's explicit discard (`worktrees prune --force `) deletes the +// branch, having been told what goes. +func (w *Worktrees) forget(ctx context.Context, r Worktree, from []WorktreeState, how *removal, at *string) Worktree { + if w.movedElsewhere(ctx, r) { + // Moved out from under the connector: its files are someone's. + return w.retain(ctx, r, RetainedMoved, from) + } + tip, err := w.branchTip(ctx, r) + if err != nil { + if how != nil && how.force { + // The repository cannot be read at all, so there is nothing here + // to delete and nothing to keep the row for: an operator who + // named it gets it closed rather than a row nothing can clear. + w.log.Warn("connector: a worktree's repository could not be read; the row is closed as the operator asked", "path", r.Path, "error", err) + return w.recordGone(ctx, r, from) + } + return w.retain(ctx, r, RetainedUnverified, from) + } + ours := tip != "" && r.BranchCreated && strings.HasPrefix(r.Branch, BranchPrefix) + if !ours { + // No branch of the connector's making is left. What may be left is + // git's record of the worktree, which reaches whatever it reaches: a + // row that never stored where that is asks the repository, and a row + // whose record cannot be looked for is kept, not closed. + record, err := w.findRecord(ctx, r) + switch { + case err != nil: + return w.retain(ctx, r, RetainedUnverified, from) + case record == "": + // Nothing of the connector's is left: no branch it made, no + // record. There is nothing to decide and nothing to delete. + return w.recordGone(ctx, r, from) + } + } + if how == nil || !how.force { + return w.retain(ctx, r, RetainedOrphaned, from) + } + // The operator named this worktree: the branch it made goes, at the + // commit it stands at, and git's record is left for `git worktree prune`. + // A branch that could not be deleted keeps the row, so nothing is left + // behind that nothing lists. + if ours { + if !w.deleteBranch(ctx, r, tip) { + return w.retain(ctx, r, RetainedOrphaned, from) + } + if at != nil { + *at = tip + } + } + return w.recordGone(ctx, r, from) +} + +// findRecord is git's record of this worktree — /.git/worktrees/ — +// found by the path it names, for a row that never stored where it is: "" when +// the repository has no record of this worktree. It answers where the record +// is and nothing about what it reaches. Anything unreadable is an error, never +// a "no". +func (w *Worktrees) findRecord(ctx context.Context, r Worktree) (string, error) { + if r.AdminDir != "" { + if _, err := os.Lstat(r.AdminDir); errors.Is(err, os.ErrNotExist) { + return "", nil + } else if err != nil { + return "", err + } + return r.AdminDir, nil + } + common, err := w.gitOut(ctx, r.Repository, "rev-parse", "--path-format=absolute", "--git-common-dir") + if err != nil { + return "", err + } + dir := filepath.Join(common, "worktrees") + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + admin := filepath.Join(dir, entry.Name()) + at, err := os.ReadFile(filepath.Join(admin, "gitdir")) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return "", err + } + named := strings.TrimSpace(string(at)) + if !filepath.IsAbs(named) { + named = filepath.Join(admin, named) + } + if samePath(filepath.Dir(named), r.Path) { + return admin, nil + } + } + return "", nil +} + +// recordGone records a row whose worktree is not on disk and has nothing left +// to decide. +func (w *Worktrees) recordGone(ctx context.Context, r Worktree, from []WorktreeState) Worktree { + gone := RemovedMissing + if r.State == WorktreeCreating { + gone = RemovedNeverCreated + } + if err := w.ledger.RemovedWorktree(ctx, r.ID, gone, from...); err != nil { + w.log.Warn("connector: recording a worktree gone", "path", r.Path, "error", err) + return r + } + r.State, r.RemovedBy = WorktreeRemoved, gone + return r +} + +func (w *Worktrees) settleKeeping(ctx context.Context, r Worktree, by RemovedBy, force bool, refs *[]string, at *string) Worktree { + from := []WorktreeState{r.State} + // A removal a crash interrupted: its names come back first, and it is + // judged as it stands. + if restored, ok := w.unfreeze(r); !ok { + w.log.Warn("connector: a frozen worktree could not be restored; kept", "path", r.Path) + return w.retain(ctx, r, RetainedUnverified, from) + } else if restored { + w.log.Info("connector: restored a worktree a removal left frozen", "path", r.Path) + } + + if !exists(r.Path) { + return w.forget(ctx, r, from, &removal{force: force}, at) + } + return w.removeWorktree(ctx, r, by, removal{force: force}, refs) +} + +// removal is how removeWorktree judges. +type removal struct { + // force is an operator's explicit discard: unheld commits are kept under + // refs and the worktree goes. + force bool +} + +// RemovingSuffix is what a removal adds to a worktree's name and to its +// record's while it judges them: a directory under it is a removal that is +// running, or one a crash left for the next start to restore. +const RemovingSuffix = ".removing" + +// frozenName is where removeWorktree moves a name while it judges. +func frozenName(path string) string { return path + RemovingSuffix } + +// removeWorktree is the one removal (the rule, in the type's doc). It claims +// the row, freezes the worktree, judges it frozen, and deletes the frozen copy +// or restores it and retains the row. The caller holds the lock and has seen +// the directory there. +func (w *Worktrees) removeWorktree(ctx context.Context, r Worktree, by RemovedBy, how removal, refs *[]string) Worktree { + from := []WorktreeState{r.State} + admin := r.AdminDir + if admin == "" { + // A row whose record's place was never stored: found, proven to be + // this worktree's own record, and stored before anything is renamed. + found, err := w.recordOf(ctx, r) + if err != nil { + return w.retain(ctx, r, RetainedUnverified, from) + } + if err := w.ledger.WorktreeAdminDir(ctx, r.ID, found); err != nil { + return w.retain(ctx, r, RetainedUnverified, from) + } + admin, r.AdminDir = found, found + } + if r.State != WorktreeRemoving { + if err := w.ledger.MoveWorktree(ctx, r.ID, WorktreeRemoving, from...); err != nil { + w.log.Warn("connector: claiming a worktree for removal", "path", r.Path, "error", err) + return r + } + r.State = WorktreeRemoving + } + removing := []WorktreeState{WorktreeRemoving} + + // Freeze: lock the record, rename it, then the directory. The lock is + // git's own: a frozen record's gitdir names a directory that is not there, + // and git's prune deletes such a record unless it is locked. + switch taken, err := lockRecord(admin); { + case err != nil: + return w.retain(ctx, r, RetainedUnverified, removing) + case !taken: + return w.retain(ctx, r, RetainedLocked, removing) + } + v := view{dir: frozenName(r.Path), gitDir: frozenName(admin)} + if err := os.Rename(admin, v.gitDir); err != nil { + unlockRecord(admin) + return w.retain(ctx, r, RetainedUnverified, removing) + } + if err := os.Rename(r.Path, v.dir); err != nil { + if os.Rename(v.gitDir, admin) == nil { + unlockRecord(admin) + } else { + w.log.Warn("connector: a worktree's record could not be restored; the next start restores it", "path", r.Path) + return r + } + return w.retain(ctx, r, RetainedUnverified, removing) + } + if w.whileFrozen != nil { + if err := w.whileFrozen(v.dir); err != nil { + // A test standing in for a crash: names stay frozen. + return r + } + } + + judged := w.judge(ctx, r, v, how) + if judged.reason == "" && how.force && len(judged.unheld) > 0 { + // A force keeps what nothing holds before anything else happens, and + // those refs hold it from here on: the transaction below verifies + // them with every other holder. + kept, err := w.keepCommits(ctx, r, judged.unheld) + if err != nil { + judged.reason = RetainedUnverified + } else { + if refs != nil { + *refs = append(*refs, kept...) + } + for i, ref := range kept { + judged.holds = append(judged.holds, hold{ref: ref, oid: judged.unheld[i], commit: judged.unheld[i]}) + } + } + } + if judged.reason != "" { + if !w.restore(r, v, admin) { + w.log.Warn("connector: a frozen worktree could not be restored; the next start restores it", "path", r.Path) + return r + } + return w.retain(ctx, r, judged.reason, removing) + } + + // Every commit the worktree reaches is held under a ref of the + // connector's own before anything is deleted, and let go only once the + // removal is over. Whatever else holds those commits — a remote branch a + // fetch prunes, a branch someone deletes — may go while the removal runs: + // it takes nothing with it. These refs hold nothing for anybody else (see + // RemovingRefPrefix), so one left by a crash cannot pass for a holder. + if _, err := w.anchor(ctx, r, judged.tips); err != nil { + w.log.Warn("connector: a worktree's commits could not be held for its removal; kept", "path", r.Path, "error", err) + if w.restore(r, v, admin) { + return w.retain(ctx, r, RetainedUnverified, removing) + } + return r + } + // The branch goes first, in the transaction that proves the judgment + // still stands: every ref the judgment leaned on is verified where it was + // found, so a fetch, a reset or a branch deleted since makes git refuse + // the whole thing and the worktree is kept instead. It is deleted only at + // a commit judged held, so a crash between the two leaves nothing + // unreachable, while the other order would leave a branch nothing later + // settles. + if !w.endBranch(ctx, r, judged) { + // The removal does not happen: its anchors go, each only while what + // the judgment found still holds its commit. + w.dropAnchors(ctx, r, judged) + if w.restore(r, v, admin) { + return w.retain(ctx, r, RetainedUnverified, removing) + } + w.log.Warn("connector: a frozen worktree could not be restored; the next start restores it", "path", r.Path) + return r + } + // Delete the frozen copy: the directory, then the record. + if err := os.RemoveAll(v.dir); err != nil { + w.log.Warn("connector: a frozen worktree could not be deleted; kept", "path", r.Path, "error", err) + // The branch went first: a worktree that comes back comes back whole, + // checked out on the branch it was checked out on. + w.putBranchBack(ctx, r, judged.tip) + w.dropAnchors(ctx, r, judged) + if w.restore(r, v, admin) { + return w.retain(ctx, r, RetainedUnverified, removing) + } + return r + } + if err := os.RemoveAll(v.gitDir); err != nil { + // The directory is gone but git's record of it is not, and it is + // still frozen and locked. The row stays removing, which is a row the + // next start restores and judges again, rather than a removed row + // nothing lists and nothing reconciles. + w.log.Warn("connector: a worktree's record could not be deleted; the next start restores it", "path", r.Path, "error", err) + return r + } + // The worktree is gone: the anchors of its held commits are let go, each + // only while the ref the judgment found still holds its commit. One that + // moved keeps its anchor, and the operator is told which. + if left := w.dropAnchors(ctx, r, judged); len(left) > 0 && refs != nil { + *refs = append(*refs, left...) + } + if err := w.ledger.RemovedWorktree(ctx, r.ID, by, removing...); err != nil { + // The worktree is gone; the row still says removing, and the next + // settle records it missing. Nobody is told it was kept. + w.log.Warn("connector: a worktree was removed but the ledger could not record it", "path", r.Path, "error", err) + return r + } + r.State, r.RemovedBy = WorktreeRemoved, by + return r +} + +// restore gives a frozen worktree its names back: the directory, then the +// record. +func (w *Worktrees) restore(r Worktree, v view, admin string) bool { + if exists(v.dir) && os.Rename(v.dir, r.Path) != nil { + return false + } + if exists(v.gitDir) && os.Rename(v.gitDir, admin) != nil { + return false + } + unlockRecord(admin) + return true +} + +// recordLockReason marks a lock on git's record of a worktree as the +// connector's own, taken while a removal holds it frozen. +const recordLockReason = "basecamp-connect: removal in progress\n" + +// lockRecord locks git's record of a worktree for the connector, the way +// `git worktree lock` does, if nobody holds a lock on it. taken is false for a +// lock someone else holds. +func lockRecord(admin string) (taken bool, err error) { + f, err := os.OpenFile(filepath.Join(admin, "locked"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if errors.Is(err, os.ErrExist) { + return ownRecordLock(filepath.Join(admin, "locked")), nil + } + if err != nil { + return false, err + } + if _, err := f.WriteString(recordLockReason); err != nil { + _ = f.Close() + _ = os.Remove(f.Name()) + return false, err + } + return true, f.Close() +} + +// ownRecordLock reports whether a record's lock file is the connector's. +func ownRecordLock(path string) bool { + content, err := os.ReadFile(path) + return err == nil && string(content) == recordLockReason +} + +// unlockRecord removes the connector's own lock on a record, never another's. +func unlockRecord(admin string) { + path := filepath.Join(admin, "locked") + if ownRecordLock(path) { + _ = os.Remove(path) + } +} + +// recordOf finds git's record of a worktree and proves it is this worktree's: +// under the repository's common directory's worktrees/, with a gitdir that +// names this worktree's .git. +func (w *Worktrees) recordOf(ctx context.Context, r Worktree) (string, error) { + admin, err := w.gitOut(ctx, r.Path, "rev-parse", "--absolute-git-dir") + if err != nil { + return "", err + } + common, err := w.gitOut(ctx, r.Repository, "rev-parse", "--path-format=absolute", "--git-common-dir") + if err != nil { + return "", err + } + if !samePath(filepath.Dir(admin), filepath.Join(common, "worktrees")) { + return "", fmt.Errorf("connector: %s is not a worktree record of %s", admin, r.Repository) + } + at, err := os.ReadFile(filepath.Join(admin, "gitdir")) + if err != nil { + return "", err + } + named := strings.TrimSpace(string(at)) + if !filepath.IsAbs(named) { + named = filepath.Join(admin, named) + } + if !samePath(filepath.Dir(named), r.Path) { + return "", fmt.Errorf("connector: %s is another worktree's record", admin) + } + return admin, nil +} + +// unfreeze restores the names of a worktree a crash left frozen. It reports +// whether it restored anything, and false in ok when a frozen name is there +// but cannot be put back. +func (w *Worktrees) unfreeze(r Worktree) (restored, ok bool) { + pairs := [][2]string{{frozenName(r.Path), r.Path}} + if r.AdminDir != "" { + pairs = append(pairs, [2]string{frozenName(r.AdminDir), r.AdminDir}) + } + for _, p := range pairs { + if !exists(p[0]) { + continue + } + if exists(p[1]) || os.Rename(p[0], p[1]) != nil { + return restored, false + } + restored = true + } + if r.AdminDir != "" { + unlockRecord(r.AdminDir) + } + return restored, true +} + +// view is how git is pointed at a worktree: its directory, and, for a frozen +// one, git's record of it by its frozen name. +type view struct { + dir string + gitDir string +} + +func (v view) args(args ...string) []string { + if v.gitDir == "" { + return append([]string{"-C", v.dir}, args...) + } + return append([]string{"-C", v.dir, "--git-dir", v.gitDir, "--work-tree", v.dir}, args...) +} + +// hold is a ref the connector keeps, the commit it pointed at when a judgment +// leaned on it, and the commit of the worktree it was found to hold. +type hold struct{ ref, oid, commit string } + +// judgment is what judge decided about a frozen worktree: the reason to keep +// it, or "" with the task branch's tip ("" when the branch is gone), the refs +// that hold every commit the removal would forget, and, for a force, the +// commits nothing holds. +type judgment struct { + reason RetainedReason + tip string + // tips is every commit the worktree reaches that removing it would + // forget; unheld are the ones nothing else holds, which only a force + // reaches; holds are the refs that hold the rest. + tips []string + unheld []string + holds []hold +} + +// pseudoRefs are the record's own refs outside refs/: what a reset, a fetch or +// an operation in progress left in /.git/worktrees/, and what goes +// with the record when it is deleted. +var pseudoRefs = []string{"ORIG_HEAD", "FETCH_HEAD", "MERGE_HEAD", "REBASE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "AUTO_MERGE", "BISECT_EXPECTED_REV", "MERGE_AUTOSTASH"} + +// autostashFiles are where a rebase keeps the commit it stashed away: not a +// ref, a file in the record naming one, and nothing else reaches it. +var autostashFiles = []string{filepath.Join("rebase-merge", "autostash"), filepath.Join("rebase-apply", "autostash")} + +// judge decides whether a frozen worktree holds anything that could be lost. +func (w *Worktrees) judge(ctx context.Context, r Worktree, v view, how removal) judgment { + gitPath := func(name string) string { return filepath.Join(v.gitDir, name) } + switch _, err := os.Lstat(gitPath("locked")); { + case err == nil && !ownRecordLock(gitPath("locked")): + return judgment{reason: RetainedLocked} + case err == nil: + case !errors.Is(err, os.ErrNotExist): + return judgment{reason: RetainedUnverified} + } + // A submodule's git data is never lost, and never forced away: no ref + // here can keep it. + switch entries, err := os.ReadDir(gitPath("modules")); { + case err == nil && len(entries) > 0: + return judgment{reason: RetainedDirty} + case err != nil && !errors.Is(err, os.ErrNotExist): + return judgment{reason: RetainedUnverified} + } + if !how.force { + for _, marker := range []string{"MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "BISECT_LOG", "rebase-merge", "rebase-apply", "sequencer"} { + switch _, err := os.Lstat(gitPath(marker)); { + case err == nil: + return judgment{reason: RetainedDirty} + case !errors.Is(err, os.ErrNotExist): + return judgment{reason: RetainedUnverified} + } + } + } + // The disk before any git command that could recurse: whatever is not a + // tracked file is work, and a submodule directory holding anything — a git + // directory and configuration a worker planted among it — is work git is + // never asked to look inside. + untracked, gitlinkContent, err := w.untrackedOnDisk(ctx, v) + switch { + case err != nil: + return judgment{reason: RetainedUnverified} + case gitlinkContent: + return judgment{reason: RetainedDirty} + case untracked && !how.force: + return judgment{reason: RetainedDirty} + } + // A checkout that never happened — a crash between `worktree add + // --no-checkout` and the checkout — holds git's .git file and nothing + // else, with an empty index. There is nothing in it to lose, though the + // rule below would read that index against HEAD as every file deleted. + bare, err := w.neverCheckedOut(ctx, v) + if err != nil { + return judgment{reason: RetainedUnverified} + } + if !how.force && !bare { + status, err := w.gitRawIn(ctx, v, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--ignored=traditional", "--ignore-submodules=all") + if err != nil { + return judgment{reason: RetainedUnverified} + } + if len(status) > 0 { + return judgment{reason: RetainedDirty} + } + // An index entry marked skip-worktree or assume-unchanged hides its + // edits from status. + entries, err := w.gitRawIn(ctx, v, "ls-files", "-v", "-z") + if err != nil { + return judgment{reason: RetainedUnverified} + } + for entry := range strings.SplitSeq(string(entries), "\x00") { + if entry == "" { + continue + } + if tag := entry[0]; tag == 'S' || (tag >= 'a' && tag <= 'z') { + return judgment{reason: RetainedDirty} + } + } + } + + tip, err := w.branchTip(ctx, r) + if err != nil { + return judgment{reason: RetainedUnverified} + } + // Every commit the worktree or its branch reaches, and that removing it + // would forget: HEAD, the branch, their reflogs, per-worktree refs. + var tips []string + // A HEAD that names no commit — a worker's `checkout --orphan`, or an + // unborn branch — reaches nothing through HEAD, and that is an answer, + // not doubt: what HEAD stood at before is still read from its reflog + // below. `--quiet` says it with exit 1 and nothing else, so a git the + // connector could not run still leaves the worktree unjudged. + head, err := w.gitRawIn(ctx, v, "rev-parse", "--quiet", "--verify", "--end-of-options", "HEAD^{commit}") + unborn := false + switch { + case err == nil: + tips = append(tips, strings.TrimSpace(string(head))) + case noSuchRevision(err): + unborn = true + default: + return judgment{reason: RetainedUnverified} + } + if tip != "" { + tips = append(tips, tip) + out, err := w.gitOut(ctx, r.Repository, "reflog", "show", "--format=%H", "refs/heads/"+r.Branch, "--") + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, strings.Fields(out)...) + } + // HEAD's reflog holds every commit it stood at, and an unborn HEAD is + // the one HEAD `reflog show` will not name — while the file still holds + // what it stood at before, which nothing else reaches. So it is read as + // the per-worktree reflogs below are, and an orphan loses no history. + if unborn { + logged, err := reflogFileTips(filepath.Join(v.gitDir, "logs", "HEAD")) + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, logged...) + } else { + out, err := w.gitRawIn(ctx, v, "reflog", "show", "--format=%H", "HEAD", "--") + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, strings.Fields(string(out))...) + } + perWorktree, err := w.gitRawIn(ctx, v, "for-each-ref", "--format=%(objectname)", "refs/worktree/", "refs/bisect/", "refs/rewritten/") + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, strings.Fields(string(perWorktree))...) + // Those refs' own reflogs, when the repository keeps them: git logs ref + // updates under refs/ only with core.logAllRefUpdates=always, and a + // per-worktree ref's log lives in the record and goes with it. + for _, dir := range []string{"refs/worktree", "refs/bisect", "refs/rewritten"} { + logged, err := reflogDirTips(filepath.Join(v.gitDir, "logs", filepath.FromSlash(dir))) + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, logged...) + } + // + // The record's pseudo-refs are its too, and go with it: ORIG_HEAD is what + // a reset left behind, and the rest are an operation's. They are read as + // the files they are, because some of them — FETCH_HEAD, and MERGE_HEAD + // in an octopus merge — name more than one commit, which `rev-parse` + // would reduce to the first. + named, err := pseudoRefTips(v.gitDir) + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, named...) + stashed, err := autostashTips(v.gitDir) + if err != nil { + return judgment{reason: RetainedUnverified} + } + tips = append(tips, stashed...) + // A reflog that is not there is not a reflog that holds nothing: with + // core.logAllRefUpdates off, or after an expire, what the worktree + // reached is unreadable, and what cannot be read is not judged clean — + // unless an operator names this worktree and forces it, which is a + // decision about work, not a judgment. Every repository keeps reflogs by + // default; one that does not would otherwise leave rows nothing could + // ever clear. + if !bare && !how.force { + if _, err := os.Lstat(filepath.Join(v.gitDir, "logs", "HEAD")); err != nil { + return judgment{reason: RetainedUnverified} + } + } + slices.Sort(tips) + // Only commits: a pseudo-ref or a ref pointed at a tree or a blob names + // no history, and asking what contains one is a question git answers + // with an error — which would keep the worktree for ever. A conflicted + // merge leaves exactly that in AUTO_MERGE. + commits, err := w.commitsAmong(ctx, v, slices.Compact(tips)) + if err != nil { + return judgment{reason: RetainedUnverified} + } + decided := judgment{tip: tip, tips: commits} + for _, commit := range decided.tips { + // The ref that holds it, and where that ref stands: the removal + // verifies each one again, in the transaction that ends the branch, + // so a holder that moved in between stops the removal. + ref, oid, err := w.holder(ctx, r, commit) + switch { + case err != nil: + return judgment{reason: RetainedUnverified} + case ref == "": + if !how.force { + return judgment{reason: RetainedUnpushed} + } + decided.unheld = append(decided.unheld, commit) + default: + decided.holds = append(decided.holds, hold{ref: ref, oid: oid, commit: commit}) + } + } + return decided +} + +// dropAnchors lets go of the refs a removal held its commits under, each only +// while the ref the judgment found still holds that commit. It returns the +// anchors that stay, because what held their commits moved. +func (w *Worktrees) dropAnchors(ctx context.Context, r Worktree, judged judgment) []string { + var left []string + for _, h := range judged.holds { + anchor := anchorRef(r, h.commit) + stdin := "start\nverify " + h.ref + " " + h.oid + "\ndelete " + anchor + " " + h.commit + "\nprepare\ncommit\n" + if err := w.gitStdin(ctx, r.Repository, stdin, "update-ref", "--stdin"); err != nil { + w.log.Info("connector: a commit of a removed worktree is kept under a ref: what held it moved", "ref", anchor, "path", r.Path) + left = append(left, anchor) + } + } + return left +} + +// retainedRef is where a commit of this worktree is kept for the operator. +func retainedRef(r Worktree, commit string) string { + return RetainedRefPrefix + safeName(filepath.Base(r.Path)) + "/" + commit +} + +// anchorRef is where a removal holds a commit of this worktree while it runs. +func anchorRef(r Worktree, commit string) string { + return RemovingRefPrefix + safeName(filepath.Base(r.Path)) + "/" + commit +} + +// keepCommits keeps each commit under refs/basecamp-connect/retained// +// , create-only; a ref already there at that commit is the same keep. +func (w *Worktrees) keepCommits(ctx context.Context, r Worktree, commits []string) ([]string, error) { + return w.holdUnder(ctx, r, commits, retainedRef) +} + +// anchor holds each commit under RemovingRefPrefix for as long as a removal +// runs. +func (w *Worktrees) anchor(ctx context.Context, r Worktree, commits []string) ([]string, error) { + return w.holdUnder(ctx, r, commits, anchorRef) +} + +func (w *Worktrees) holdUnder(ctx context.Context, r Worktree, commits []string, where func(Worktree, string) string) ([]string, error) { + refs := make([]string, 0, len(commits)) + var made []string + for _, commit := range commits { + ref := where(r, commit) + if _, err := w.gitOut(ctx, r.Repository, "update-ref", "--end-of-options", ref, commit, ""); err != nil { + at, atErr := w.gitOut(ctx, r.Repository, "rev-parse", "--verify", "--end-of-options", ref) + if atErr != nil || at != commit { + // Holding them all failed, so none is held: the ones this + // call made are let go again, and a ref that was already + // there — an earlier force's keep — is left alone. + for _, ref := range made { + if _, err := w.gitOut(ctx, r.Repository, "update-ref", "-d", "--end-of-options", ref); err != nil { + w.log.Warn("connector: a ref made to hold a commit could not be let go", "ref", ref, "error", err) + } + } + return nil, err + } + } else { + made = append(made, ref) + } + refs = append(refs, ref) + } + return refs, nil +} + +// movedElsewhere reports whether the repository still has a worktree on this +// row's branch somewhere else: someone moved it, and its files are work the +// connector neither judges nor forgets, so the row is kept. +func (w *Worktrees) movedElsewhere(ctx context.Context, r Worktree) bool { + // The repository's record of this worktree names where it is now, + // whatever it has checked out and whatever its branch is called. + if r.AdminDir != "" { + switch at, err := os.ReadFile(filepath.Join(r.AdminDir, "gitdir")); { + case err == nil: + // The record is the worktree's .git file, absolute or — with + // worktree.useRelativePaths — relative to the admin directory. + path := strings.TrimSpace(string(at)) + if !filepath.IsAbs(path) { + path = filepath.Join(r.AdminDir, path) + } + path = filepath.Dir(path) + return !samePath(path, r.Path) && exists(path) + case !errors.Is(err, os.ErrNotExist): + // The record cannot be read: assume it is still somewhere. + return true + } + return false + } + // A row from before the admin directory was recorded: its branch is the + // only handle left. + out, err := w.gitRaw(ctx, r.Repository, "worktree", "list", "--porcelain", "-z") + if err != nil { + return true + } + var current string + for field := range strings.SplitSeq(string(out), "\x00") { + switch { + case strings.HasPrefix(field, "worktree "): + current = strings.TrimPrefix(field, "worktree ") + case field == "branch refs/heads/"+r.Branch: + if !samePath(current, r.Path) && exists(current) { + return true + } + } + } + return false +} + +// reflogDirTips is every commit the reflogs under one directory of a record +// name. A directory that is not there is a repository that logs nothing +// there, which names nothing. +func reflogDirTips(dir string) ([]string, error) { + var tips []string + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + switch { + case errors.Is(err, os.ErrNotExist): + return nil + case err != nil: + return err + case d.IsDir(): + return nil + } + logged, err := reflogFileTips(path) + if err != nil { + return err + } + tips = append(tips, logged...) + return nil + }) + if err != nil { + return nil, err + } + return tips, nil +} + +// reflogFileTips is every commit a reflog file names, read as git writes it: +// one line per entry, the commit before it and the commit after it first. A +// reflog that is not there names nothing; one that cannot be read is an error, +// never an empty answer. +func reflogFileTips(path string) ([]string, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var tips []string + for line := range strings.SplitSeq(string(data), "\n") { + fields := strings.Fields(line) + // The two object names an entry starts with; the rest of the line is + // who, when and why, which name nothing. + for _, field := range fields[:min(2, len(fields))] { + if !isObjectName(field) { + continue + } + tips = append(tips, field) + } + } + return tips, nil +} + +// commitsAmong is the commits these object names reach: the name itself, or +// what an annotated tag points at, because that is the history the name +// keeps. A tree or a blob reaches no commit and neither does an object that +// is no longer there, and both are dropped — there is nothing in them to +// lose. Git answers once per name, in order; answering for fewer is an error, +// never a quiet drop. +func (w *Worktrees) commitsAmong(ctx context.Context, v view, oids []string) ([]string, error) { + if len(oids) == 0 { + return nil, nil + } + var asked strings.Builder + for _, oid := range oids { + asked.WriteString(oid + "^{commit}\n") + } + out, err := w.gitRawInStdin(ctx, v, asked.String(), "cat-file", "--batch-check=%(objectname) %(objecttype)") + if err != nil { + return nil, err + } + answers := strings.Split(strings.TrimSuffix(string(out), "\n"), "\n") + if len(answers) != len(oids) { + return nil, fmt.Errorf("connector: git answered for %d of %d object names", len(answers), len(oids)) + } + commits := make([]string, 0, len(oids)) + for _, answer := range answers { + if name, kind, ok := strings.Cut(answer, " "); ok && kind == "commit" { + commits = append(commits, name) + } + } + slices.Sort(commits) + return slices.Compact(commits), nil +} + +// pseudoRefTips is every object name the record's pseudo-refs hold: one per +// line, first field, as git writes FETCH_HEAD and the rest. A file that is +// not there names nothing; one that cannot be read is an error. +func pseudoRefTips(gitDir string) ([]string, error) { + var tips []string + for _, name := range pseudoRefs { + data, err := os.ReadFile(filepath.Join(gitDir, name)) + switch { + case errors.Is(err, os.ErrNotExist): + continue + case err != nil: + return nil, err + } + for line := range strings.SplitSeq(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && isObjectName(fields[0]) { + tips = append(tips, fields[0]) + } + } + } + return tips, nil +} + +// autostashTips is every commit an operation in progress stashed away in a +// record: git writes the object name to a file, and nothing else names it. +func autostashTips(gitDir string) ([]string, error) { + var tips []string + for _, name := range autostashFiles { + data, err := os.ReadFile(filepath.Join(gitDir, name)) + switch { + case errors.Is(err, os.ErrNotExist): + continue + case err != nil: + return nil, err + } + if oid := strings.TrimSpace(string(data)); isObjectName(oid) { + tips = append(tips, oid) + } + } + return tips, nil +} + +// isGitDir reports whether a directory is a repository's git data: git's own +// test is a HEAD, an objects directory and a refs directory. +func isGitDir(path string) bool { + for _, name := range []string{"HEAD", "objects", "refs"} { + if _, err := os.Lstat(filepath.Join(path, name)); err != nil { + return false + } + } + return true +} + +// isObjectName reports whether a field is an object name and not the zero one. +func isObjectName(field string) bool { + return len(field) >= 40 && strings.Trim(field, "0123456789abcdef") == "" && strings.Trim(field, "0") != "" +} + +// noSuchRevision reports whether git said a revision does not resolve, which +// `rev-parse --quiet` says with exit 1 and nothing else. Any other failure is +// a git that could not be run, which is never an answer about work. +func noSuchRevision(err error) bool { + var exitErr *exec.ExitError + return errors.As(err, &exitErr) && exitErr.ExitCode() == 1 +} + +// exists reports whether a path is anything but proven absent: a path that +// cannot be read counts as there, because an error is not evidence that work +// is gone. +func exists(path string) bool { + _, err := os.Lstat(path) + return !errors.Is(err, os.ErrNotExist) +} + +func (w *Worktrees) retain(ctx context.Context, r Worktree, reason RetainedReason, from []WorktreeState) Worktree { + if err := w.ledger.RetainWorktree(ctx, r.ID, reason, from...); err != nil { + w.log.Warn("connector: recording a worktree retained", "path", r.Path, "error", err) + return r + } + w.log.Info("connector: worktree retained", "path", r.Path, "branch", r.Branch, "reason", string(reason)) + r.State, r.RetainedReason = WorktreeRetained, reason + return r +} + +// neverCheckedOut reports whether a worktree's directory holds nothing but +// git's .git file and its index is empty: `git worktree add --no-checkout` +// ran and the checkout that follows it did not. +func (w *Worktrees) neverCheckedOut(ctx context.Context, v view) (bool, error) { + entries, err := os.ReadDir(v.dir) + if err != nil { + return false, err + } + if len(entries) != 1 || entries[0].Name() != ".git" || entries[0].IsDir() { + return false, nil + } + index, err := w.gitRawIn(ctx, v, "ls-files", "--stage", "-z") + if err != nil { + return false, err + } + return len(strings.TrimSpace(string(index))) == 0, nil +} + +// untrackedOnDisk reports whether a worktree's directory holds anything that +// is not a file git tracks (an untracked or ignored file, a directory git has +// no file in), and separately whether a submodule's directory, which the +// checkout left empty, holds anything at all. Symlinks are not followed. +func (w *Worktrees) untrackedOnDisk(ctx context.Context, v view) (untracked, gitlinkContent bool, err error) { + out, err := w.gitRawIn(ctx, v, "ls-files", "--stage", "-z") + if err != nil { + return false, false, err + } + files, gitlinks, dirs := map[string]bool{}, map[string]bool{}, map[string]bool{".": true} + for entry := range strings.SplitSeq(string(out), "\x00") { + meta, path, ok := strings.Cut(entry, "\t") + if !ok { + continue + } + if strings.HasPrefix(meta, "160000 ") { + gitlinks[path] = true + } else { + files[path] = true + } + for dir := filepath.Dir(filepath.FromSlash(path)); dir != "."; dir = filepath.Dir(dir) { + dirs[filepath.ToSlash(dir)] = true + } + } + // Bounded: a tree too big to read in time, or a filesystem call that never + // returns (a mount a worker left), is not proven clean, and must not hold + // the connector's shutdown. The walk runs apart and is abandoned at the + // deadline; a call stuck in the kernel keeps only its own goroutine. + deadline := time.Now().Add(w.walkLimit) + type verdict struct { + untracked, gitlink bool + err error + } + walked := make(chan verdict, 1) + go func() { + var found verdict + found.err = filepath.WalkDir(v.dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + if time.Now().After(deadline) { + return errors.New("connector: the worktree could not be read in time") + } + rel, err := filepath.Rel(v.dir, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + switch { + case rel == ".git" && !d.IsDir(): + // The worktree's link to its repository. + return nil + case filepath.Base(rel) == ".git": + // Git data of a repository inside the worktree — a submodule + // git someone initialized, a repository a worker made, or a + // .git the worktree's own was replaced with. No ref here can + // keep its commits, so it is never removed, forced or not. + found.gitlink = true + return filepath.SkipAll + case gitlinks[rel]: + if !d.IsDir() { + found.gitlink = true + return filepath.SkipAll + } + entries, err := os.ReadDir(path) + if err != nil { + return err + } + if len(entries) > 0 { + found.gitlink = true + return filepath.SkipAll + } + return filepath.SkipDir + case d.IsDir(): + // A directory that is itself a repository — `git init --bare`, + // or a clone with no worktree — is git data like any other, + // wherever it is: no ref here can keep its commits, so it is + // never removed, forced or not. The worktree's own directory + // is one of these to look at: a worker can make a repository + // of the root it works in. + if isGitDir(path) { + found.gitlink = true + return filepath.SkipAll + } + if !dirs[rel] { + found.untracked = true + } + return nil + case !files[rel]: + found.untracked = true + } + return nil + }) + walked <- found + }() + timer := time.NewTimer(time.Until(deadline)) + defer timer.Stop() + select { + case found := <-walked: + return found.untracked, found.gitlink, found.err + case <-timer.C: + return false, false, errors.New("connector: the worktree could not be read in time") + case <-ctx.Done(): + return false, false, ctx.Err() + } +} + +// holder is a ref the connector keeps that contains commit, and the commit it +// points at; "" when there is none. +func (w *Worktrees) holder(ctx context.Context, r Worktree, commit string) (string, string, error) { + own := RetainedRefPrefix + safeName(filepath.Base(r.Path)) + "/" + out, err := w.gitOut(ctx, r.Repository, "for-each-ref", "--format=%(refname) %(objectname)", "--contains", commit, "refs/remotes", "refs/heads", own) + if err != nil { + return "", "", err + } + for line := range strings.SplitSeq(out, "\n") { + ref, oid, ok := strings.Cut(line, " ") + switch { + case !ok, strings.HasPrefix(ref, "refs/heads/"+BranchPrefix): + case strings.HasPrefix(ref, "refs/remotes/"), strings.HasPrefix(ref, "refs/heads/"), strings.HasPrefix(ref, own): + return ref, oid, nil + } + } + return "", "", nil +} + +func (w *Worktrees) branchTip(ctx context.Context, r Worktree) (string, error) { + out, err := w.gitRaw(ctx, r.Repository, "for-each-ref", "--format=%(objectname)", "refs/heads/"+r.Branch) + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +// putBranchBack makes the task branch again, at the commit it was deleted at, +// for a removal that could not go through: what came back must be what was +// there. +func (w *Worktrees) putBranchBack(ctx context.Context, r Worktree, commit string) { + if commit == "" || !r.BranchCreated || !strings.HasPrefix(r.Branch, BranchPrefix) { + return + } + if _, err := w.gitOut(ctx, r.Repository, "update-ref", "--end-of-options", "refs/heads/"+r.Branch, commit, ""); err != nil { + w.log.Warn("connector: a task branch could not be made again for a worktree that stayed", "branch", r.Branch, "error", err) + } +} + +// deleteBranch deletes the task branch of a worktree whose directory is gone, +// at the commit it stands at and only when this row made it. An operator +// asked for it by naming the worktree, and was told what goes; nothing here +// judges what the branch reaches. +func (w *Worktrees) deleteBranch(ctx context.Context, r Worktree, commit string) bool { + if commit == "" || !r.BranchCreated || !strings.HasPrefix(r.Branch, BranchPrefix) { + return true + } + stdin := "start\ndelete refs/heads/" + r.Branch + " " + commit + "\nprepare\ncommit\n" + if err := w.gitStdin(ctx, r.Repository, stdin, "update-ref", "--stdin"); err != nil { + w.log.Warn("connector: a task branch could not be deleted", "branch", r.Branch, "error", err) + return false + } + return true +} + +// endBranch is the last thing a removal does before the frozen copy goes: one +// ref transaction that verifies every ref the judgment leaned on is still +// where it was found and deletes the task branch at the tip judged held. Git +// refuses the whole transaction if any of them moved, and the removal stops. +// What it proves is that the judgment still stands when the deleting starts; +// what keeps standing while the deleting runs is the anchors. +// It reports whether the judgment still stands. +func (w *Worktrees) endBranch(ctx context.Context, r Worktree, judged judgment) bool { + stdin := "start\n" + seen := map[string]string{} + for _, h := range judged.holds { + // One verify line per ref: git refuses a transaction that names a ref + // twice. A ref found at two different commits while the judgment ran + // is a ref that moved, and one of the two cannot be verified, so the + // worktree is kept. + if at, ok := seen[h.ref]; ok { + if at != h.oid { + w.log.Warn("connector: a worktree is kept: what held its commits moved while it was judged", "path", r.Path, "ref", h.ref) + return false + } + continue + } + seen[h.ref] = h.oid + stdin += "verify " + h.ref + " " + h.oid + "\n" + } + deleting := judged.tip != "" && r.BranchCreated && strings.HasPrefix(r.Branch, BranchPrefix) + if deleting { + stdin += "delete refs/heads/" + r.Branch + " " + judged.tip + "\n" + } + if !deleting && len(seen) == 0 { + // Nothing held anything and no branch of this row's making: the + // judgment leans on nothing that could have moved. + return true + } + stdin += "prepare\ncommit\n" + if err := w.gitStdin(ctx, r.Repository, stdin, "update-ref", "--stdin"); err != nil { + w.log.Warn("connector: a worktree is kept: what held its commits moved while it was judged", "path", r.Path, "branch", r.Branch, "error", err) + return false + } + return true +} + +// WalkLimit bounds how long reading a worktree's files may take before it is +// kept as unverified. +const WalkLimit = 2 * time.Minute + +// LockWait bounds how long a settling worktree waits for another remover's +// lock. Longer than a removal takes, short enough that a stuck prune cannot +// hold a task's end, and so the connector's shutdown, open: the row is +// reconciled on the next start instead. +const LockWait = 2 * time.Minute + +// lock takes the worktrees lock (invariant 4), waiting up to LockWait for +// another holder. +func (w *Worktrees) lock(ctx context.Context) (func(), error) { + ctx, cancel := context.WithTimeout(ctx, LockWait) + defer cancel() + if err := os.MkdirAll(w.root, 0o700); err != nil { + return nil, err + } + if err := setup.EnsurePrivateDir(w.root); err != nil { + return nil, err + } + path := filepath.Join(w.root, ".lock") + for { + unlock, err := setup.TryLockPrivate(path) + switch { + case err == nil: + return func() { _ = unlock() }, nil + case !errors.Is(err, setup.ErrLockHeld): + return nil, fmt.Errorf("connector: worktrees lock: %w", err) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } +} + +func (w *Worktrees) gitOut(ctx context.Context, dir string, args ...string) (string, error) { + out, err := w.gitRaw(ctx, dir, args...) + return strings.TrimSpace(string(out)), err +} + +// gitRaw runs git in dir with hooks, the fsmonitor and every configured +// content filter disabled, and a fixed environment (invariant 3). +func (w *Worktrees) gitRaw(ctx context.Context, dir string, args ...string) ([]byte, error) { + return w.gitRawIn(ctx, view{dir: dir}, args...) +} + +// gitRawIn is gitRaw for a view: a frozen worktree is reached through its +// record by its frozen name. +func (w *Worktrees) gitRawIn(ctx context.Context, v view, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + guard, err := w.filterOverrides(ctx, v) + if err != nil { + return nil, err + } + return w.run(ctx, guard, v.args(args...), args[0]) +} + +// safeGit is the configuration every git call runs with. +var safeGit = [][2]string{ + {"core.hooksPath", "/dev/null"}, + {"core.fsmonitor", "false"}, + // A reflog or log that verifies signatures runs gpg.program. + {"log.showSignature", "false"}, +} + +// filterOverrides blanks every content filter git's configuration defines +// for dir. A checkout runs a path's smudge, clean or process filter, which is +// a command from configuration a worker in the checkout could have edited; +// an empty command is no filter. Reading the configuration runs nothing. +// +// The overrides travel as GIT_CONFIG_KEY_n/GIT_CONFIG_VALUE_n, not `-c`, +// which splits at the first "=" and would miss a driver whose name has one. +func (w *Worktrees) filterOverrides(ctx context.Context, v view) ([][2]string, error) { + out, err := w.run(ctx, safeGit, v.args("config", "--name-only", "--get-regexp", `^filter\.`), "config") + var exitErr *exec.ExitError + if err != nil && (!errors.As(err, &exitErr) || exitErr.ExitCode() != 1) { + // Exit 1 is "no such keys"; anything else leaves filters unknown. + return nil, err + } + guard := slices.Clone(safeGit) + seen := map[string]bool{} + for key := range strings.SplitSeq(string(out), "\n") { + rest, ok := strings.CutPrefix(strings.TrimSpace(key), "filter.") + if !ok { + continue + } + i := strings.LastIndexByte(rest, '.') + if i <= 0 || seen[rest[:i]] { + continue + } + name := rest[:i] + seen[name] = true + for _, cmd := range []string{"clean", "smudge", "process"} { + guard = append(guard, [2]string{"filter." + name + "." + cmd, ""}) + } + // A blanked filter that is also required makes git die mid-checkout + // (git lfs install sets required for its own). + guard = append(guard, [2]string{"filter." + name + ".required", "false"}) + } + return guard, nil +} + +// gitStdin runs git in dir, guarded as gitRaw is, with input on stdin. +func (w *Worktrees) gitStdin(ctx context.Context, dir, input string, args ...string) error { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + guard, err := w.filterOverrides(ctx, view{dir: dir}) + if err != nil { + return err + } + _, err = w.runInput(ctx, guard, view{dir: dir}.args(args...), args[0], input) + return err +} + +// gitRawInStdin is gitStdin for a view, and gives back what git wrote: a +// frozen worktree is reached through its record by its frozen name. +func (w *Worktrees) gitRawInStdin(ctx context.Context, v view, input string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + guard, err := w.filterOverrides(ctx, v) + if err != nil { + return nil, err + } + return w.runInput(ctx, guard, v.args(args...), args[0], input) +} + +func (w *Worktrees) run(ctx context.Context, config [][2]string, args []string, what string) ([]byte, error) { + return w.runInput(ctx, config, args, what, "") +} + +func (w *Worktrees) runInput(ctx context.Context, config [][2]string, args []string, what, input string) ([]byte, error) { + cmd := exec.CommandContext(ctx, w.git, args...) //nolint:gosec // G204: git with the connector's own arguments + if input != "" { + cmd.Stdin = strings.NewReader(input) + } + env := slices.Clone(w.env) + env = append(env, "GIT_CONFIG_COUNT="+strconv.Itoa(len(config))) + for i, kv := range config { + env = append(env, "GIT_CONFIG_KEY_"+strconv.Itoa(i)+"="+kv[0], "GIT_CONFIG_VALUE_"+strconv.Itoa(i)+"="+kv[1]) + } + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if len(msg) > 200 { + msg = msg[:200] + } + return stdout.Bytes(), fmt.Errorf("git %s: %w: %s", what, err, w.red.Sanitize(msg)) + } + return stdout.Bytes(), nil +} + +func realPath(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return filepath.Clean(p) +} + +func samePath(a, b string) bool { + return a != "" && b != "" && realPath(a) == realPath(b) +} diff --git a/internal/connector/worktrees_test.go b/internal/connector/worktrees_test.go new file mode 100644 index 000000000..5b78e1cf1 --- /dev/null +++ b/internal/connector/worktrees_test.go @@ -0,0 +1,1899 @@ +package connector + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/connector/admission" + "github.com/basecamp/basecamp-cli/internal/connector/driver" +) + +// worktreeHarness is a repository with a bare remote, a ledger, and +// Worktrees placing worktrees under a private root. +type worktreeHarness struct { + t *testing.T + home string + repo string + remote string + root string + ledger *Ledger + wt *Worktrees +} + +func newWorktreeHarness(t *testing.T) *worktreeHarness { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } + dir := t.TempDir() + h := &worktreeHarness{ + t: t, + home: filepath.Join(dir, "home"), + repo: filepath.Join(dir, "repo"), + remote: filepath.Join(dir, "remote.git"), + root: filepath.Join(dir, "state", "worktrees"), + ledger: newTestLedger(t), + } + require.NoError(t, os.MkdirAll(h.home, 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(h.repo, "app"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Dir(h.root), 0o700)) + h.git(dir, "init", "-q", "--bare", "-b", "main", h.remote) + h.git(h.repo, "init", "-q", "-b", "main") + h.write(h.repo, "app/README", "hello\n") + h.git(h.repo, "add", ".") + h.git(h.repo, "commit", "-q", "-m", "init") + h.git(h.repo, "remote", "add", "origin", h.remote) + h.git(h.repo, "push", "-q", "origin", "main") + h.wt = h.worktrees("") + return h +} + +func (h *worktreeHarness) worktrees(gitBinary string) *Worktrees { + h.t.Helper() + w, err := NewWorktrees(WorktreesOptions{ + Ledger: h.ledger, + Root: h.root, + Git: gitBinary, + Lookup: h.lookup, + }) + require.NoError(h.t, err) + return w +} + +func (h *worktreeHarness) lookup(k string) (string, bool) { + switch k { + case "HOME": + return h.home, true + case "PATH": + return os.Getenv("PATH"), true + } + return "", false +} + +func (h *worktreeHarness) git(dir string, args ...string) string { + h.t.Helper() + cmd := exec.CommandContext(context.Background(), "git", append([]string{"-c", "user.name=Test", "-c", "user.email=test@example.invalid", "-c", "commit.gpgsign=false"}, args...)...) + cmd.Dir = dir + cmd.Env = []string{"HOME=" + h.home, "PATH=" + os.Getenv("PATH"), "GIT_CONFIG_NOSYSTEM=1"} + out, err := cmd.CombinedOutput() + require.NoError(h.t, err, "git %v: %s", args, out) + return strings.TrimSpace(string(out)) +} + +// gitConflicting runs a git command that is meant to stop with a conflict: +// what it leaves in the record is the point, not its exit status. +func (h *worktreeHarness) gitConflicting(dir string, args ...string) { + h.t.Helper() + cmd := exec.CommandContext(context.Background(), "git", append([]string{"-c", "user.name=Test", "-c", "user.email=test@example.invalid", "-c", "commit.gpgsign=false"}, args...)...) + cmd.Dir = dir + cmd.Env = []string{"HOME=" + h.home, "PATH=" + os.Getenv("PATH"), "GIT_CONFIG_NOSYSTEM=1"} + out, err := cmd.CombinedOutput() + require.Error(h.t, err, "git %v was meant to conflict: %s", args, out) +} + +func (h *worktreeHarness) write(dir, name, content string) { + h.t.Helper() + require.NoError(h.t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o700)) + require.NoError(h.t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600)) +} + +// prepare makes a worktree for the route "app" and returns the working +// directory and its row. +func (h *worktreeHarness) prepare(eventID int64) (string, Worktree) { + h.t.Helper() + workDir, err := h.wt.Prepare(context.Background(), filepath.Join(h.repo, "app"), eventID) + require.NoError(h.t, err) + row := h.row(workDir) + return workDir, row +} + +func (h *worktreeHarness) row(workDir string) Worktree { + h.t.Helper() + rows, err := h.ledger.Worktrees(context.Background()) + require.NoError(h.t, err) + for _, r := range rows { + if r.WorkDir == workDir { + return r + } + } + h.t.Fatalf("no worktree row for %s", workDir) + return Worktree{} +} + +// discard is what an operator does: the task ends (which only ever keeps the +// worktree), then `worktrees prune` judges it and removes what holds nothing. +func (h *worktreeHarness) discard(workDir string) Worktree { + h.t.Helper() + require.NoError(h.t, h.wt.Finish(context.Background(), filepath.Join(h.repo, "app"), workDir)) + _, err := h.wt.Prune(context.Background(), nil) + require.NoError(h.t, err) + return h.row(workDir) +} + +func (h *worktreeHarness) finish(workDir string) Worktree { + h.t.Helper() + require.NoError(h.t, h.wt.Finish(context.Background(), filepath.Join(h.repo, "app"), workDir)) + return h.row(workDir) +} + +func (h *worktreeHarness) branchExists(branch string) bool { + h.t.Helper() + return h.git(h.repo, "for-each-ref", "refs/heads/"+branch) != "" +} + +func TestPrepareMakesAWorktreeOnATaskBranchOutsideTheCheckout(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(17) + + assert.Equal(t, WorktreeLive, row.State) + assert.Equal(t, filepath.Join(row.Path, "app"), workDir, "the route's place inside the repository") + assert.True(t, strings.HasPrefix(row.Path, h.root+string(filepath.Separator)), "placed under the connector's root") + assert.False(t, strings.HasPrefix(row.Path, h.repo), "never inside the checkout") + assert.True(t, strings.HasPrefix(row.Branch, BranchPrefix+"17-")) + assert.Equal(t, h.git(h.repo, "rev-parse", "HEAD"), row.BaseCommit) + assert.FileExists(t, filepath.Join(workDir, "README")) + assert.Empty(t, h.git(h.repo, "status", "--porcelain"), "the checkout sees nothing of it") + + info, err := os.Stat(filepath.Dir(row.Path)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) +} + +// Invariant 1: a task's end keeps its worktree, whatever is in it; the +// operator's prune is what removes one with nothing to lose, with its branch. +func TestATasksEndKeepsItsWorktreeAndAPruneRemovesIt(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(1) + + kept := h.finish(workDir) + assert.Equal(t, WorktreeRetained, kept.State) + assert.Equal(t, RetainedFinished, kept.RetainedReason) + assert.True(t, exists(kept.Path), "the worktree is still there") + assert.True(t, h.branchExists(kept.Branch), "and so is its branch") + + row := h.discard(workDir) + assert.Equal(t, WorktreeRemoved, row.State) + assert.Equal(t, RemovedByPrune, row.RemovedBy) + assert.False(t, exists(row.Path)) + assert.False(t, h.branchExists(row.Branch)) +} + +// Invariant 1: uncommitted work survives the task's end and is listed as +// retained (the card's done-when). +func TestUncommittedWorkSurvivesTheTaskAndIsRetained(t *testing.T) { + for name, change := range map[string]func(h *worktreeHarness, workDir string){ + "modified": func(h *worktreeHarness, d string) { h.write(d, "README", "changed\n") }, + "untracked": func(h *worktreeHarness, d string) { h.write(d, "notes/new.txt", "draft\n") }, + "staged": func(h *worktreeHarness, d string) { + h.write(d, "staged.txt", "x\n") + h.git(d, "add", "staged.txt") + }, + "deleted": func(h *worktreeHarness, d string) { require.NoError(h.t, os.Remove(filepath.Join(d, "README"))) }, + "ignored": func(h *worktreeHarness, d string) { + exclude := h.git(d, "rev-parse", "--path-format=absolute", "--git-path", "info/exclude") + require.NoError(h.t, os.MkdirAll(filepath.Dir(exclude), 0o700)) + require.NoError(h.t, os.WriteFile(exclude, []byte("*.local\n"), 0o600)) + h.write(d, "report.local", "results\n") + }, + "skip-worktree": func(h *worktreeHarness, d string) { + h.git(d, "update-index", "--skip-worktree", "README") + h.write(d, "README", "hidden edit\n") + }, + "assume-unchanged": func(h *worktreeHarness, d string) { + h.git(d, "update-index", "--assume-unchanged", "README") + h.write(d, "README", "hidden edit\n") + }, + "merge in progress": func(h *worktreeHarness, d string) { + marker := h.git(d, "rev-parse", "--path-format=absolute", "--git-path", "MERGE_HEAD") + require.NoError(h.t, os.WriteFile(marker, []byte(h.git(d, "rev-parse", "HEAD")+"\n"), 0o600)) + }, + } { + t.Run(name, func(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(2) + change(h, workDir) + row := h.finish(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, RetainedFinished, row.RetainedReason) + // And an operator's prune keeps it too, now saying what is in it. + row = h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, RetainedDirty, row.RetainedReason) + assert.True(t, exists(workDir)) + assert.True(t, h.branchExists(row.Branch)) + + retained, err := h.wt.Retained(context.Background()) + require.NoError(t, err) + require.Len(t, retained, 1) + assert.Equal(t, row.Path, retained[0].Path) + }) + } +} + +// Invariant 1: a commit only this worktree holds keeps it; one a remote or +// the main line holds does not; another task's branch is no evidence. +func TestCommitsAreKeptUntilHeldElsewhere(t *testing.T) { + commit := func(h *worktreeHarness, d, name string) string { + h.write(d, name, name+"\n") + h.git(d, "add", name) + h.git(d, "commit", "-q", "-m", name) + return h.git(d, "rev-parse", "HEAD") + } + t.Run("unpushed", func(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(3) + commit(h, workDir, "work.txt") + row := h.discard(workDir) + assert.Equal(t, RetainedUnpushed, row.RetainedReason) + assert.True(t, exists(workDir)) + }) + t.Run("pushed", func(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(4) + commit(h, workDir, "work.txt") + h.git(workDir, "push", "-q", "origin", row.Branch) + row = h.discard(workDir) + assert.Equal(t, WorktreeRemoved, row.State) + assert.False(t, h.branchExists(row.Branch)) + }) + t.Run("merged", func(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(5) + commit(h, workDir, "work.txt") + h.git(h.repo, "merge", "-q", "--ff-only", row.Branch) + row = h.discard(workDir) + assert.Equal(t, WorktreeRemoved, row.State) + }) + t.Run("held only by another task's branch", func(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(6) + sha := commit(h, workDir, "work.txt") + h.git(h.repo, "branch", BranchPrefix+"99-other", sha) + row := h.discard(workDir) + assert.Equal(t, RetainedUnpushed, row.RetainedReason) + }) + t.Run("detached away from an unpushed branch", func(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(7) + commit(h, workDir, "work.txt") + h.git(workDir, "checkout", "-q", "--detach", row.BaseCommit) + row = h.discard(workDir) + assert.Equal(t, RetainedUnpushed, row.RetainedReason, "the task branch's commits count, wherever HEAD is") + }) +} + +func TestALockedWorktreeIsRetained(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(8) + h.git(h.repo, "worktree", "lock", row.Path) + row = h.discard(workDir) + assert.Equal(t, RetainedLocked, row.RetainedReason) + assert.True(t, exists(workDir)) +} + +// fakeGit is a git that runs the real one, except where told to fail or to +// do something first. +func fakeGit(t *testing.T, script string) string { + t.Helper() + gitPath, err := exec.LookPath("git") + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "git") + body := "#!/bin/sh\nREAL=" + gitPath + "\n" + script + "\nexec \"$REAL\" \"$@\"\n" + require.NoError(t, os.WriteFile(path, []byte(body), 0o700)) + return path +} + +// Invariant 1: a check that fails keeps the worktree. +func TestAFailedCheckRetains(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(9) + h.wt = h.worktrees(fakeGit(t, `for a in "$@"; do [ "$a" = status ] && exit 128; done`)) + row := h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, RetainedUnverified, row.RetainedReason) + assert.True(t, exists(workDir)) +} + +// Invariant 2: the branch is deleted only while it still points at the commit +// that was verified. +func TestABranchThatMovedIsNotDeleted(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(11) + // Between the check and the branch's deletion someone commits onto the + // task branch from elsewhere. + other := filepath.Join(t.TempDir(), "other") + h.git(h.repo, "worktree", "add", "-q", "--detach", other, row.BaseCommit) + h.write(other, "moved.txt", "x\n") + h.git(other, "add", "moved.txt") + h.git(other, "commit", "-q", "-m", "moved") + moved := h.git(other, "rev-parse", "HEAD") + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"update-ref --stdin"*) "$REAL" -C "`+h.repo+`" update-ref refs/heads/`+row.Branch+` `+moved+`;; esac`)) + row = h.discard(workDir) + // The judgment no longer stands, so the worktree is kept with it. + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, moved, h.git(h.repo, "rev-parse", "refs/heads/"+row.Branch)) + assert.True(t, exists(workDir), "the worktree is still there") +} + +// Invariant 6: the repository's hooks do not run. +func TestTheRepositorysHooksDoNotRun(t *testing.T) { + h := newWorktreeHarness(t) + marker := filepath.Join(t.TempDir(), "hook-ran") + hook := filepath.Join(h.repo, ".git", "hooks", "post-checkout") + require.NoError(t, os.WriteFile(hook, []byte("#!/bin/sh\ntouch "+marker+"\n"), 0o700)) + h.prepare(12) + assert.False(t, exists(marker)) +} + +// Invariant 1, on the disk itself: a file in a submodule's directory, which +// git neither reports nor refuses to remove, is work. +func TestWorkInASubmodulesDirectoryIsRetained(t *testing.T) { + h := newWorktreeHarness(t) + sub := filepath.Join(t.TempDir(), "sub") + require.NoError(t, os.MkdirAll(sub, 0o700)) + h.git(sub, "init", "-q", "-b", "main") + h.write(sub, "lib.txt", "lib\n") + h.git(sub, "add", ".") + h.git(sub, "commit", "-q", "-m", "sub") + h.git(h.repo, "-c", "protocol.file.allow=always", "submodule", "add", "-q", sub, "app/vendor") + h.git(h.repo, "commit", "-q", "-m", "submodule") + + workDir, _ := h.prepare(14) + h.write(workDir, "vendor/notes.txt", "notes\n") + row := h.discard(workDir) + assert.Equal(t, RetainedDirty, row.RetainedReason) + assert.True(t, exists(filepath.Join(workDir, "vendor", "notes.txt"))) +} + +// Invariant 1: a commit only the worktree's reflog still reaches is work. +func TestACommitOnlyTheReflogReachesIsRetained(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(15) + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "moved away from") + h.git(workDir, "checkout", "-q", row.Branch) + row = h.discard(workDir) + assert.Equal(t, RetainedUnpushed, row.RetainedReason) +} + +// Invariant 6: a filter whose name git's -c could not carry, or that only +// the task branch's configuration defines, does not run either. +func TestFiltersOutOfReachOfAScanStillDoNotRun(t *testing.T) { + for name, configure := range map[string]func(h *worktreeHarness, marker string){ + "name with =": func(h *worktreeHarness, marker string) { + h.write(h.repo, ".gitattributes", "*.txt filter=a=b\n") + h.git(h.repo, "config", "filter.a=b.smudge", "touch "+marker+"; cat") + }, + "defined on the task branch": func(h *worktreeHarness, marker string) { + h.write(h.repo, ".gitattributes", "*.txt filter=probe\n") + include := filepath.Join(h.home, "branch-filter.gitconfig") + h.write(h.home, "branch-filter.gitconfig", "[filter \"probe\"]\n\tsmudge = touch "+marker+"; cat\n") + h.git(h.repo, "config", "includeIf.onbranch:"+BranchPrefix+"**.path", include) + }, + } { + t.Run(name, func(t *testing.T) { + h := newWorktreeHarness(t) + marker := filepath.Join(t.TempDir(), "ran") + configure(h, marker) + h.write(h.repo, "app/data.txt", "data\n") + h.git(h.repo, "add", ".") + h.git(h.repo, "commit", "-q", "-m", "attributes") + h.prepare(16) + assert.False(t, exists(marker), "no filter ran") + }) + } +} + +// Invariant 6, at removal: git worktree remove reads the worktree's files +// under the task branch's own configuration, and a filter defined there does +// not run either. +func TestAFilterOnTheTaskBranchDoesNotRunAtRemoval(t *testing.T) { + h := newWorktreeHarness(t) + marker := filepath.Join(t.TempDir(), "ran") + h.write(h.repo, ".gitattributes", "*.txt filter=probe\n") + h.write(h.repo, "app/data.txt", "data\n") + h.git(h.repo, "add", ".") + h.git(h.repo, "commit", "-q", "-m", "attributes") + workDir, _ := h.prepare(96) + h.write(h.home, "branch-filter.gitconfig", "[filter \"probe\"]\n\tclean = touch "+marker+"; cat\n\tsmudge = touch "+marker+"; cat\n") + h.git(h.repo, "config", "includeIf.onbranch:"+BranchPrefix+"**.path", filepath.Join(h.home, "branch-filter.gitconfig")) + // A racy index entry makes status read the file through its clean filter. + require.NoError(t, os.Chtimes(filepath.Join(workDir, "data.txt"), time.Now().Add(time.Hour), time.Now().Add(time.Hour))) + + row := h.discard(workDir) + assert.False(t, exists(marker), "no filter ran") + assert.Equal(t, WorktreeRemoved, row.State) +} + +// A filter git lfs marks required does not break the checkout once blanked. +func TestARequiredFilterDoesNotBreakTheCheckout(t *testing.T) { + h := newWorktreeHarness(t) + h.write(h.repo, ".gitattributes", "*.bin filter=lfsish\n") + h.write(h.repo, "app/blob.bin", "blob\n") + h.git(h.repo, "add", ".") + h.git(h.repo, "commit", "-q", "-m", "blob") + h.git(h.repo, "config", "filter.lfsish.smudge", "cat") + h.git(h.repo, "config", "filter.lfsish.clean", "cat") + h.git(h.repo, "config", "filter.lfsish.required", "true") + + workDir, row := h.prepare(97) + assert.Equal(t, WorktreeLive, row.State) + assert.FileExists(t, filepath.Join(workDir, "blob.bin")) + assert.Equal(t, WorktreeRemoved, h.discard(workDir).State) +} + +// submoduleHarness is a worktree harness whose repository has a submodule at +// app/vendor, and the submodule's source. +func submoduleHarness(t *testing.T) (*worktreeHarness, string) { + t.Helper() + h := newWorktreeHarness(t) + sub := filepath.Join(t.TempDir(), "sub") + require.NoError(t, os.MkdirAll(sub, 0o700)) + h.git(sub, "init", "-q", "-b", "main") + h.write(sub, "lib.txt", "lib\n") + h.git(sub, "add", ".") + h.git(sub, "commit", "-q", "-m", "sub") + h.git(h.repo, "-c", "protocol.file.allow=always", "submodule", "add", "-q", sub, "app/vendor") + h.git(h.repo, "commit", "-q", "-m", "submodule") + return h, sub +} + +// Invariant 6 in a submodule's directory: a git directory and configuration a +// worker planted there never make the connector's git run a filter, because +// git is never asked to look inside it. +func TestAFilterPlantedInASubmoduleDoesNotRun(t *testing.T) { + h, sub := submoduleHarness(t) + workDir, _ := h.prepare(98) + marker := filepath.Join(t.TempDir(), "ran") + vendor := filepath.Join(workDir, "vendor") + clone := filepath.Join(t.TempDir(), "clone") + h.git(filepath.Dir(clone), "clone", "-q", sub, clone) + require.NoError(t, os.Rename(filepath.Join(clone, ".git"), filepath.Join(vendor, ".planted"))) + require.NoError(t, os.Rename(filepath.Join(clone, "lib.txt"), filepath.Join(vendor, "lib.txt"))) + h.write(vendor, ".git", "gitdir: .planted\n") + h.write(vendor, ".planted/info/attributes", "lib.txt filter=probe\n") + h.git(vendor, "config", "filter.probe.clean", "touch "+marker+"; cat") + require.NoError(t, os.Chtimes(filepath.Join(vendor, "lib.txt"), time.Now().Add(time.Hour), time.Now().Add(time.Hour))) + + row := h.discard(workDir) + assert.False(t, exists(marker), "no filter ran") + assert.Equal(t, RetainedDirty, row.RetainedReason) +} + +// Invariant 5 with a submodule: its commits live in git directories a forced +// removal would delete, so a worktree holding any is not forced. +func TestAForcedPruneKeepsASubmodulesCommits(t *testing.T) { + h, _ := submoduleHarness(t) + workDir, _ := h.prepare(99) + h.git(workDir, "-c", "protocol.file.allow=always", "submodule", "update", "-q", "--init") + vendor := filepath.Join(workDir, "vendor") + h.write(vendor, "more.txt", "more\n") + h.git(vendor, "add", ".") + h.git(vendor, "commit", "-q", "-m", "only copy") + subGitDir := h.git(vendor, "rev-parse", "--absolute-git-dir") + row := h.discard(workDir) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.True(t, results[0].ForceRefused) + assert.DirExists(t, subGitDir) +} + +// A checkout that never happened leaves nothing kept: an empty worktree is +// not work, and keeping it as dirty at every retry would fill the disk. +// A checkout that never happened is still not the connector's to delete: the +// row is kept, and the operator's prune removes it as a worktree holding +// nothing. +func TestAnUnpopulatedWorktreeIsKeptUntilAPrune(t *testing.T) { + h := newWorktreeHarness(t) + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"reset --quiet --hard"*) exit 128;; esac`)) + _, err := h.wt.Prepare(context.Background(), filepath.Join(h.repo, "app"), 100) + require.Error(t, err) + rows, err := h.ledger.Worktrees(context.Background()) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.Equal(t, WorktreeRetained, rows[0].State) + assert.True(t, exists(rows[0].Path)) + + h.wt = h.worktrees("") + results, err := h.wt.Prune(context.Background(), nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneRemoved, results[0].Action) + assert.False(t, exists(rows[0].Path)) + assert.False(t, h.branchExists(rows[0].Branch)) +} + +// A worktree whose directory was deleted is recorded missing, and the +// repository's own record of it is left for git: it can hold a submodule's +// only commits, a reflog, or a lock for a directory that is only away. +func TestAMissingWorktreesRepositoryRecordIsLeftAlone(t *testing.T) { + h, _ := submoduleHarness(t) + workDir, row := h.prepare(103) + h.git(workDir, "-c", "protocol.file.allow=always", "submodule", "update", "-q", "--init") + vendor := filepath.Join(workDir, "vendor") + h.write(vendor, "more.txt", "more\n") + h.git(vendor, "add", ".") + h.git(vendor, "commit", "-q", "-m", "only copy") + subGitDir := h.git(vendor, "rev-parse", "--absolute-git-dir") + require.NoError(t, os.RemoveAll(row.Path)) + + row = h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State, "a record holding a submodule's commits keeps the row") + assert.DirExists(t, row.AdminDir) + assert.DirExists(t, subGitDir, "the submodule's only commits survive") +} + +// A worktree someone moved is kept, not forgotten: its files are still +// somewhere, and the connector cannot judge them where it cannot find them. +func TestAMovedWorktreeIsKept(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(90) + moved := filepath.Join(t.TempDir(), "moved") + h.git(h.repo, "worktree", "move", row.Path, moved) + require.False(t, exists(workDir)) + + row = h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, RetainedMoved, row.RetainedReason) + assert.True(t, h.branchExists(row.Branch), "the branch the moved worktree has checked out") + assert.FileExists(t, filepath.Join(moved, "app", "README")) +} + +// A worktree moved with a detached HEAD is kept too: the repository's own +// record of it, not its branch, is what says where it is. +func TestAMovedWorktreeWithNoBranchIsKept(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(92) + h.git(workDir, "checkout", "-q", "--detach") + h.git(h.repo, "branch", "-q", "-D", row.Branch) + moved := filepath.Join(t.TempDir(), "moved") + h.git(h.repo, "worktree", "move", row.Path, moved) + + row = h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.FileExists(t, filepath.Join(moved, "app", "README")) +} + +// A worktree moved and then deleted is gone, not kept forever. +func TestAMovedWorktreeThatIsThenDeletedIsGone(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(91) + moved := filepath.Join(t.TempDir(), "moved") + h.git(h.repo, "worktree", "move", row.Path, moved) + require.NoError(t, os.RemoveAll(moved)) + + row = h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, RetainedOrphaned, row.RetainedReason, "the connector deletes none of what is left") + assert.True(t, h.branchExists(row.Branch)) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action, "the operator names it and is told what goes") + assert.False(t, h.branchExists(row.Branch)) + assert.Equal(t, WorktreeRemoved, h.row(workDir).State) +} + +// Invariant 1: a task branch the connector did not create is never deleted, +// however that worktree ends. +func TestABranchTheConnectorDidNotMakeIsNotDeleted(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + base := h.git(h.repo, "rev-parse", "HEAD") + branch := BranchPrefix + "80-taken" + h.git(h.repo, "branch", branch, base) + + record := Worktree{ + Path: filepath.Join(h.root, "repo", "80-taken"), WorkDir: filepath.Join(h.root, "repo", "80-taken"), + Route: filepath.Join(h.repo, "app"), Repository: h.repo, Branch: branch, BaseCommit: base, + OriginatingEventID: 80, State: WorktreeCreating, + } + id, err := h.ledger.BeginWorktree(ctx, record) + require.NoError(t, err) + record.ID = id + require.Error(t, h.wt.add(ctx, &record), "the branch is already someone's") + + unlock, err := h.wt.lock(ctx) + require.NoError(t, err) + settled := h.wt.settle(ctx, record) + unlock() + assert.Equal(t, WorktreeRemoved, settled.State) + assert.True(t, h.branchExists(branch), "someone else's branch survives") + assert.False(t, h.row(record.WorkDir).BranchCreated) +} + +// A worktree that cannot be made is not attempted again at every dispatch +// tick: each failure leaves a row and maybe a partial checkout. +func TestAFailedPrepareBacksOff(t *testing.T) { + h := newWorktreeHarness(t) + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"worktree add"*) exit 128;; esac`)) + clock := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) + h.wt.now = func() time.Time { return clock } + route := filepath.Join(h.repo, "app") + ctx := context.Background() + + _, err := h.wt.Prepare(ctx, route, 70) + require.Error(t, err) + _, err = h.wt.Prepare(ctx, route, 70) + require.ErrorIs(t, err, ErrPrepareBackoff) + rows, err := h.ledger.Worktrees(ctx) + require.NoError(t, err) + assert.Len(t, rows, 1, "the second call made nothing") + + clock = clock.Add(PrepareBackoff) + _, err = h.wt.Prepare(ctx, route, 70) + require.Error(t, err) + assert.NotErrorIs(t, err, ErrPrepareBackoff) + clock = clock.Add(PrepareBackoff) + _, err = h.wt.Prepare(ctx, route, 70) + require.ErrorIs(t, err, ErrPrepareBackoff, "the wait doubles") + + _, err = h.wt.Prepare(ctx, route, 71) + require.ErrorIs(t, err, ErrPrepareBackoff, "the route waits, whichever event asks") + assert.Equal(t, []string{route}, h.wt.RoutesWaiting()) + + clock = clock.Add(PrepareBackoffMax) + assert.Empty(t, h.wt.RoutesWaiting(), "a route whose wait is over is not held") +} + +// A route that cannot take a worktree never fills the window the dispatcher +// starts records from: a healthy route's record still starts. +func TestAFailingWorktreeRouteDoesNotStarveTheOthers(t *testing.T) { + h := newWorktreeHarness(t) + broken := filepath.Join(t.TempDir(), "not-a-repository") + require.NoError(t, os.MkdirAll(broken, 0o700)) + healthy := filepath.Join(h.repo, "app") + const brokenBucket = 777 + fake := newFakeDriver() + d := newDispatchHarness(t, fake, func(o *DispatcherOptions) { + o.Ledger = h.ledger + o.Workspaces = h.wt + }) + d.ledger = h.ledger + d.mu.Lock() + d.routes = map[int64]admission.Route{adapterBucketID: {Path: healthy}, brokenBucket: {Path: broken}} + d.mu.Unlock() + admit := func(id, bucket int64, route string) { + event := testEvent(id) + event.BucketID = bucket + _, err := h.ledger.RecordSeen(context.Background(), event, LanePoll) + require.NoError(t, err) + v := admittedVerdict(id, 0, "recording:"+strconv.FormatInt(id, 10)) + v.BucketID, v.Route = bucket, route + _, err = h.ledger.Admission().Commit(context.Background(), v) + require.NoError(t, err) + } + for id := int64(100); id < 110; id++ { + admit(id, brokenBucket, broken) + } + admit(200, adapterBucketID, healthy) + d.run(t) + select { + case s := <-fake.made: + assert.True(t, strings.HasPrefix(s.cfg.Cwd, h.root), "the healthy route's record started in its worktree") + case <-time.After(10 * time.Second): + t.Fatal("a route that cannot take a worktree starved a healthy one") + } +} + +// With worktrees off, a new task works in its route, and a worktree made +// while they were on is still recovered. +func TestWorktreesOffStillRecoversWhatWasMade(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, _ := h.prepare(72) + h.write(workDir, "wip.txt", "wip\n") + + off, err := NewWorktrees(WorktreesOptions{Ledger: h.ledger, Root: h.root, Lookup: h.lookup, Off: true}) + require.NoError(t, err) + assert.False(t, off.PerTaskDirs()) + route := filepath.Join(h.repo, "app") + dir, err := off.Prepare(ctx, route, 73) + require.NoError(t, err) + assert.Equal(t, route, dir) + require.NoError(t, off.Finish(ctx, route, route)) + + require.NoError(t, off.Recover(ctx)) + assert.Equal(t, RetainedFinished, h.row(workDir).RetainedReason) + assert.True(t, exists(filepath.Join(workDir, "wip.txt")), "the work is where it was") +} + +// Invariant 6: a content filter the repository's configuration defines does +// not run when the connector checks out or inspects a worktree. +func TestConfiguredContentFiltersDoNotRun(t *testing.T) { + h := newWorktreeHarness(t) + markers := t.TempDir() + h.write(h.repo, ".gitattributes", "*.txt filter=probe\n") + h.write(h.repo, "app/data.txt", "data\n") + h.git(h.repo, "add", ".") + h.git(h.repo, "commit", "-q", "-m", "attributes") + h.git(h.repo, "config", "filter.probe.smudge", "touch "+filepath.Join(markers, "smudge")+"; cat") + h.git(h.repo, "config", "filter.probe.clean", "touch "+filepath.Join(markers, "clean")+"; cat") + + workDir, _ := h.prepare(13) + h.write(workDir, "data.txt", "changed\n") + row := h.discard(workDir) + assert.Equal(t, RetainedDirty, row.RetainedReason) + entries, err := os.ReadDir(markers) + require.NoError(t, err) + assert.Empty(t, entries, "no filter ran") +} + +// Invariant 3: every row a crash can leave is settled on the next start under +// the same rules, and a worktree a live task works in is not touched. +func TestRecoverSettlesWhatACrashLeft(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + + // Crashed before git ran: a row and nothing on disk. + never := Worktree{Path: filepath.Join(h.root, "x", "20-aaaaaa"), WorkDir: filepath.Join(h.root, "x", "20-aaaaaa", "app"), Route: filepath.Join(h.repo, "app"), + Repository: h.repo, Branch: BranchPrefix + "20-aaaaaa", BaseCommit: h.git(h.repo, "rev-parse", "HEAD"), OriginatingEventID: 20} + neverID, err := h.ledger.BeginWorktree(ctx, never) + require.NoError(t, err) + + // Crashed between git and live, with work in it. + dirtyDir, dirty := h.prepare(21) + h.write(dirtyDir, "wip.txt", "wip\n") + // Crashed mid-removal of a clean one (its names were not frozen). + cleanDir, clean := h.prepare(22) + require.NoError(t, h.ledger.MoveWorktree(ctx, clean.ID, WorktreeRemoving, WorktreeLive)) + // A live task still works in this one. + liveDir, _ := h.prepare(23) + admitOn(t, h.ledger, 23, "recording:23") + _, err = h.ledger.LaunchTask(ctx, LaunchSpec{EventID: 23, Route: testRoute, WorkDir: liveDir, Driver: "fake"}) + require.NoError(t, err) + + require.NoError(t, h.wt.Recover(ctx)) + + rows, err := h.ledger.Worktrees(ctx) + require.NoError(t, err) + byID := map[int64]Worktree{} + for _, r := range rows { + byID[r.ID] = r + } + // Nothing on disk to keep: the row is reconciled, nothing is deleted. + assert.Equal(t, RemovedNeverCreated, byID[neverID].RemovedBy) + // Everything that is on disk is kept, whatever is in it. + assert.Equal(t, RetainedFinished, byID[dirty.ID].RetainedReason) + assert.True(t, exists(filepath.Join(dirtyDir, "wip.txt"))) + assert.Equal(t, WorktreeRetained, byID[clean.ID].State) + assert.True(t, exists(cleanDir), "a clean worktree a crash left is kept too") + assert.Equal(t, WorktreeLive, h.row(liveDir).State) +} + +// Invariant 4: a check-and-remove waits for the lock another remover holds. +func TestRemovalsTakeTheWorktreesLock(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(30) + unlock, err := h.wt.lock(context.Background()) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + err = h.wt.Finish(ctx, filepath.Join(h.repo, "app"), workDir) + require.ErrorIs(t, err, context.DeadlineExceeded) + // A worktree that could not be judged is kept, and said to be: a row left + // live is a directory nothing lists and no prune touches. + row := h.row(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.Equal(t, RetainedUnverified, row.RetainedReason) + unlock() + retained, err := h.wt.Retained(context.Background()) + require.NoError(t, err) + require.Len(t, retained, 1) + assert.Equal(t, row.Path, retained[0].Path) + // And the prune that follows judges it as any other kept worktree. + results, err := h.wt.Prune(context.Background(), nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneRemoved, results[0].Action) +} + +// Invariant 5: prune removes what the operator dealt with, keeps what still +// holds work, forces only what the operator names, and never reaches a +// worktree that is not retained. +func TestPruneRemovesOnlyWhatTheOperatorDealtWith(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + + dealtDir, dealt := h.prepare(40) + h.write(dealtDir, "done.txt", "x\n") + h.finish(dealtDir) + h.git(dealtDir, "add", "done.txt") + h.git(dealtDir, "commit", "-q", "-m", "done") + h.git(dealtDir, "push", "-q", "origin", dealt.Branch) + + keptDir, _ := h.prepare(41) + h.write(keptDir, "wip.txt", "wip\n") + h.finish(keptDir) + + goneDir, gone := h.prepare(42) + h.write(goneDir, "wip.txt", "wip\n") + h.finish(goneDir) + require.NoError(t, os.RemoveAll(gone.Path)) + + forcedDir, forced := h.prepare(43) + h.write(forcedDir, "c.txt", "c\n") + h.git(forcedDir, "add", "c.txt") + h.git(forcedDir, "commit", "-q", "-m", "c") + h.write(forcedDir, "wip.txt", "wip\n") + h.finish(forcedDir) + + liveDir, live := h.prepare(44) + h.write(liveDir, "wip.txt", "wip\n") + + _, err := h.wt.Prune(ctx, []string{live.Path}) + require.ErrorIs(t, err, ErrNotRetained, "a live worktree is never prune's") + assert.True(t, exists(filepath.Join(liveDir, "wip.txt"))) + assert.True(t, exists(filepath.Join(forcedDir, "wip.txt")), "a refused prune removes nothing") + + results, err := h.wt.Prune(ctx, []string{forced.Path}) + require.NoError(t, err) + actions := map[string]PruneResult{} + for _, r := range results { + actions[r.Worktree.Path] = r + } + require.Len(t, actions, 4) + assert.Equal(t, PruneRemoved, actions[dealt.Path].Action) + assert.False(t, exists(dealt.Path)) + assert.Equal(t, PruneKept, actions[h.row(keptDir).Path].Action) + assert.Equal(t, RetainedDirty, actions[h.row(keptDir).Path].Reason) + assert.True(t, exists(filepath.Join(keptDir, "wip.txt"))) + // A directory something outside the connector removed: said to be + // orphaned, and nothing of what it left is touched. + assert.Equal(t, PruneKept, actions[gone.Path].Action) + assert.Equal(t, RetainedOrphaned, actions[gone.Path].Reason) + assert.Equal(t, PruneForced, actions[forced.Path].Action) + assert.NotEmpty(t, actions[forced.Path].RetainedRefs, "an unpushed commit is kept under a ref") + for _, ref := range actions[forced.Path].RetainedRefs { + assert.NotEmpty(t, h.git(h.repo, "for-each-ref", ref), "the kept ref is there") + } + assert.False(t, exists(forced.Path)) + assert.Equal(t, WorktreeLive, h.row(liveDir).State) + assert.True(t, exists(filepath.Join(liveDir, "wip.txt"))) +} + +// Invariant 5: a forced prune keeps a commit only a detached HEAD holds, on a +// branch of its own. +func TestAForcedPruneKeepsADetachedHeadsCommit(t *testing.T) { + h := newWorktreeHarness(t) + workDir, _ := h.prepare(45) + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "detached") + commit := h.git(workDir, "rev-parse", "HEAD") + row := h.discard(workDir) + require.Equal(t, RetainedUnpushed, row.RetainedReason) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action) + require.NotEmpty(t, results[0].RetainedRefs) + assert.Contains(t, h.git(h.repo, "for-each-ref", "--format=%(objectname)", RetainedRefPrefix), commit) + assert.False(t, exists(row.Path)) +} + +// The card's done-when, through the dispatcher: a worker leaves uncommitted +// work, its task ends, and the worktree is retained and listed. +func TestADispatchedTasksUncommittedWorkIsRetained(t *testing.T) { + h := newWorktreeHarness(t) + route := filepath.Join(h.repo, "app") + fake := newFakeDriver() + fake.turn = func(s *fakeSession, _ int, _ string) (driver.PromptResult, error) { + require.NoError(t, os.WriteFile(filepath.Join(s.cfg.Cwd, "answer.txt"), []byte("work\n"), 0o600)) + return driver.PromptResult{Stop: driver.TurnEndTurn}, nil + } + d := newDispatchHarness(t, fake, func(o *DispatcherOptions) { + o.Ledger = h.ledger + o.Workspaces = h.wt + }) + d.ledger = h.ledger + d.routes = map[int64]admission.Route{adapterBucketID: {Path: route}} + for _, id := range []int64{50, 51} { + seenRecord(t, h.ledger, id) + v := admittedVerdict(id, 0, "recording:"+string(rune('a'+id-50))) + v.Route = route + _, err := h.ledger.Admission().Commit(context.Background(), v) + require.NoError(t, err) + } + stop := d.run(t) + a, b := <-fake.made, <-fake.made + assert.NotEqual(t, a.cfg.Cwd, b.cfg.Cwd, "two tasks on one route, each in its own worktree") + d.attemptsEnded(t, 2) + stop() + + retained, err := h.wt.Retained(context.Background()) + require.NoError(t, err) + require.Len(t, retained, 2) + for _, r := range retained { + assert.Equal(t, RetainedFinished, r.RetainedReason) + assert.NotZero(t, r.TaskID) + content, err := os.ReadFile(filepath.Join(r.WorkDir, "answer.txt")) + require.NoError(t, err) + assert.Equal(t, "work\n", string(content)) + } +} + +// Worktree states move along their edges only: nothing goes back to live, +// and nothing leaves removed. +func TestWorktreeStatesMoveAlongTheirEdgesOnly(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + _, row := h.prepare(60) + require.NoError(t, h.ledger.RetainWorktree(ctx, row.ID, RetainedDirty, WorktreeLive)) + _, err := h.ledger.db.ExecContext(ctx, `UPDATE worktrees SET state = 'live' WHERE id = ?`, row.ID) + require.Error(t, err) + require.NoError(t, h.ledger.RemovedWorktree(ctx, row.ID, RemovedMissing, WorktreeRetained)) + _, err = h.ledger.db.ExecContext(ctx, `UPDATE worktrees SET state = 'retained', removed_by = '', retained_reason = 'dirty' WHERE id = ?`, row.ID) + require.Error(t, err) + require.ErrorIs(t, h.ledger.MoveWorktree(ctx, row.ID, WorktreeRemoving, WorktreeRetained), ErrWorktreeState) +} + +// A path the connector cannot even look at is not proof that work is gone. +func TestAnUnreadablePathCountsAsThere(t *testing.T) { + dir := t.TempDir() + closed := filepath.Join(dir, "closed") + require.NoError(t, os.Mkdir(closed, 0o700)) + inside := filepath.Join(closed, "worktree") + require.NoError(t, os.Mkdir(inside, 0o700)) + require.NoError(t, os.Chmod(closed, 0o000)) + t.Cleanup(func() { _ = os.Chmod(closed, 0o700) }) + if _, err := os.Lstat(inside); err == nil { + t.Skip("this user can read through a closed directory") + } + + assert.True(t, exists(inside), "unreadable is not absent") + assert.False(t, exists(filepath.Join(dir, "never")), "absent is absent") +} + +// A moved worktree is found through the repository's record of it however +// that record spells the path. +func TestAMovedWorktreeIsFoundWithRelativePaths(t *testing.T) { + h := newWorktreeHarness(t) + h.git(h.repo, "config", "worktree.useRelativePaths", "true") + workDir, row := h.prepare(95) + moved := filepath.Join(t.TempDir(), "moved") + h.git(h.repo, "worktree", "move", row.Path, moved) + + row = h.discard(workDir) + assert.Equal(t, RetainedMoved, row.RetainedReason) + assert.True(t, h.branchExists(row.Branch)) + assert.FileExists(t, filepath.Join(moved, "app", "README")) +} + +// A moved worktree is not forced away either: there is nothing at the path to +// judge, and prune says so instead of trying. +func TestAMovedWorktreeIsNotForced(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(93) + moved := filepath.Join(t.TempDir(), "moved") + h.git(h.repo, "worktree", "move", row.Path, moved) + row = h.discard(workDir) + require.Equal(t, RetainedMoved, row.RetainedReason) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.Equal(t, RetainedMoved, results[0].Reason) + assert.False(t, results[0].ForceRefused, "prune says where it is, it does not try and fail") + assert.FileExists(t, filepath.Join(moved, "app", "README")) +} + +// A branch the connector made for a worktree that then failed to appear is +// its own to clean up. +// A `worktree add` that failed leaves a branch and no directory. The +// connector deletes neither: the row is kept, and an operator's prune clears +// both once the branch reaches nothing that is not held. +func TestAFailedAddKeepsItsBranchUntilAPrune(t *testing.T) { + h := newWorktreeHarness(t) + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"worktree add"*) exit 128;; esac`)) + _, err := h.wt.Prepare(context.Background(), filepath.Join(h.repo, "app"), 94) + require.Error(t, err) + rows, err := h.ledger.Worktrees(context.Background()) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.True(t, rows[0].BranchCreated) + assert.Equal(t, WorktreeRetained, rows[0].State) + assert.True(t, h.branchExists(rows[0].Branch), "the branch it made is not the connector's to delete") + + h.wt = h.worktrees("") + results, err := h.wt.Prune(context.Background(), nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.Equal(t, RetainedOrphaned, results[0].Reason) + assert.True(t, h.branchExists(rows[0].Branch), "a plain prune deletes nothing of it") + + forced, err := h.wt.Prune(context.Background(), []string{rows[0].Path}) + require.NoError(t, err) + require.Len(t, forced, 1) + assert.False(t, h.branchExists(rows[0].Branch), "the operator naming it clears it") +} + +// A removal the ledger could not record is still reported as a removal, and +// Finish says it was not recorded, rather than anyone being told it was kept. +func TestARemovalTheLedgerCouldNotRecordIsNotReportedKept(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, _ := h.prepare(104) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + _, err := h.ledger.db.ExecContext(ctx, `CREATE TRIGGER refuse_removed BEFORE UPDATE OF state ON worktrees +WHEN NEW.state = 'removed' BEGIN SELECT RAISE(ABORT, 'test: the ledger refuses'); END`) + require.NoError(t, err) + + results, err := h.wt.Prune(ctx, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneRemoved, results[0].Action, "gone is not reported kept") + row := h.row(workDir) + assert.Equal(t, WorktreeRemoving, row.State) + assert.False(t, exists(row.Path)) +} + +// A missing worktree whose record in the repository still reaches a commit +// nothing else holds is kept, so the operator hears of it before git's own +// prune takes it; the connector deletes nothing either way. +func TestAMissingWorktreeWhoseRecordHoldsACommitIsKept(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(105) + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "reflog only") + h.git(workDir, "checkout", "-q", row.Branch) + require.NoError(t, os.RemoveAll(row.Path)) + + row = h.discard(workDir) + assert.Equal(t, WorktreeRetained, row.State) + assert.DirExists(t, row.AdminDir) +} + +// A forced removal the ledger could not record is still reported forced. +func TestAForcedRemovalTheLedgerCouldNotRecordIsReportedForced(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, _ := h.prepare(106) + h.write(workDir, "wip.txt", "wip\n") + row := h.discard(workDir) + require.Equal(t, RetainedDirty, row.RetainedReason) + _, err := h.ledger.db.ExecContext(ctx, `CREATE TRIGGER refuse_removed BEFORE UPDATE OF state ON worktrees +WHEN NEW.state = 'removed' BEGIN SELECT RAISE(ABORT, 'test: the ledger refuses'); END`) + require.NoError(t, err) + + results, err := h.wt.Prune(ctx, []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action) + assert.False(t, results[0].ForceRefused) + assert.False(t, exists(row.Path)) +} + +// The worktree rule ("One worktree, one removal"), case by case: what counts +// as work, what happens to it, and that the check still holds when something +// tries to land work between the check and the removal. +func TestTheWorktreeRule(t *testing.T) { + commit := func(h *worktreeHarness, dir, name string) string { + h.write(dir, name, name+"\n") + h.git(dir, "add", name) + h.git(dir, "commit", "-q", "-m", name) + return h.git(dir, "rev-parse", "HEAD") + } + type rowT struct { + name string + // work makes the worktree's state; it returns a commit that must + // survive, if any. + work func(h *worktreeHarness, dir string, row Worktree) string + // frozen runs after the removal has frozen the worktree. + frozen func(t *testing.T, h *worktreeHarness, dir string, row Worktree) + force bool + // want is the row's state after; reason when retained. + want WorktreeState + reason RetainedReason + } + rows := []rowT{ + {name: "clean", want: WorktreeRemoved}, + {name: "modified file", work: func(h *worktreeHarness, d string, _ Worktree) string { h.write(d, "README", "x\n"); return "" }, want: WorktreeRetained, reason: RetainedDirty}, + {name: "untracked file", work: func(h *worktreeHarness, d string, _ Worktree) string { h.write(d, "new.txt", "x\n"); return "" }, want: WorktreeRetained, reason: RetainedDirty}, + {name: "ignored file", work: func(h *worktreeHarness, d string, _ Worktree) string { + exclude := h.git(d, "rev-parse", "--path-format=absolute", "--git-path", "info/exclude") + require.NoError(h.t, os.MkdirAll(filepath.Dir(exclude), 0o700)) + require.NoError(h.t, os.WriteFile(exclude, []byte("*.local\n"), 0o600)) + h.write(d, "notes.local", "x\n") + return "" + }, want: WorktreeRetained, reason: RetainedDirty}, + {name: "unpushed commit", work: func(h *worktreeHarness, d string, _ Worktree) string { return commit(h, d, "c.txt") }, want: WorktreeRetained, reason: RetainedUnpushed}, + {name: "commit only the reflog reaches", work: func(h *worktreeHarness, d string, row Worktree) string { + h.git(d, "checkout", "-q", "--detach") + sha := commit(h, d, "c.txt") + h.git(d, "checkout", "-q", row.Branch) + return sha + }, want: WorktreeRetained, reason: RetainedUnpushed}, + {name: "commit a per-worktree ref holds", work: func(h *worktreeHarness, d string, row Worktree) string { + sha := commit(h, d, "c.txt") + h.git(d, "update-ref", "refs/worktree/keep", sha) + h.git(d, "reset", "-q", "--hard", row.BaseCommit) + h.git(d, "reflog", "expire", "--expire=now", "--all") + return sha + }, want: WorktreeRetained, reason: RetainedUnpushed}, + {name: "stash", work: func(h *worktreeHarness, d string, _ Worktree) string { + h.write(d, "README", "stashed\n") + h.git(d, "stash", "-q") + return h.git(d, "rev-parse", "refs/stash") + }, want: WorktreeRemoved}, + {name: "locked", work: func(h *worktreeHarness, _ string, row Worktree) string { + h.git(h.repo, "worktree", "lock", row.Path) + return "" + }, want: WorktreeRetained, reason: RetainedLocked}, + {name: "a commit tried between the check and the removal", frozen: func(t *testing.T, h *worktreeHarness, dir string, row Worktree) { + for _, at := range []string{filepath.Join(row.Path, "app"), filepath.Join(dir, "app")} { + cmd := exec.CommandContext(context.Background(), "git", "-c", "user.name=T", "-c", "user.email=t@example.invalid", "commit", "-q", "--allow-empty", "-m", "late") + cmd.Dir = at + cmd.Env = []string{"HOME=" + h.home, "PATH=" + os.Getenv("PATH")} + assert.Error(t, cmd.Run(), "no commit lands in a frozen worktree (%s)", at) + } + }, want: WorktreeRemoved}, + {name: "a file written by path between the check and the removal", frozen: func(t *testing.T, _ *worktreeHarness, _ string, row Worktree) { + assert.Error(t, os.WriteFile(filepath.Join(row.Path, "app", "late.txt"), []byte("x"), 0o600), "the path does not reach a frozen worktree") + }, want: WorktreeRemoved}, + {name: "git's own prune while frozen", work: func(h *worktreeHarness, d string, row Worktree) string { + h.git(d, "checkout", "-q", "--detach") + sha := commit(h, d, "c.txt") + h.git(d, "checkout", "-q", row.Branch) + return sha + }, frozen: func(t *testing.T, h *worktreeHarness, _ string, _ Worktree) { + h.git(h.repo, "worktree", "prune", "--expire=now") + }, want: WorktreeRetained, reason: RetainedUnpushed}, + {name: "forced unpushed commit", work: func(h *worktreeHarness, d string, _ Worktree) string { return commit(h, d, "c.txt") }, force: true, want: WorktreeRemoved}, + {name: "forced commit only the reflog reaches", work: func(h *worktreeHarness, d string, row Worktree) string { + h.git(d, "checkout", "-q", "--detach") + sha := commit(h, d, "c.txt") + h.git(d, "checkout", "-q", row.Branch) + h.write(d, "wip.txt", "wip\n") + return sha + }, force: true, want: WorktreeRemoved}, + } + for _, tc := range rows { + t.Run(tc.name, func(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, row := h.prepare(300) + var keep string + if tc.work != nil { + keep = tc.work(h, workDir, row) + } + if tc.frozen != nil { + h.wt.whileFrozen = func(dir string) error { tc.frozen(t, h, dir, row); return nil } + } + // The task's end only ever keeps the worktree; the rule is what + // the operator's prune goes by. + var after Worktree + if tc.force { + h.wt.whileFrozen = nil + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + results, err := h.wt.Prune(ctx, []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + after = h.row(workDir) + } else { + require.Equal(t, WorktreeRetained, h.finish(workDir).State, "a task's end keeps its worktree") + after = h.discard(workDir) + } + assert.Equal(t, tc.want, after.State) + if tc.reason != "" { + assert.Equal(t, tc.reason, after.RetainedReason) + } + if tc.force && after.State == WorktreeRemoved { + assert.Equal(t, RemovedByPruneForced, after.RemovedBy) + } + if tc.want == WorktreeRetained { + assert.DirExists(t, workDir, "a kept worktree is where it was") + assert.NoDirExists(t, frozenName(row.Path)) + } else { + assert.NoDirExists(t, row.Path) + assert.NoDirExists(t, frozenName(row.Path)) + assert.NoDirExists(t, frozenName(row.AdminDir)) + } + if keep != "" { + assert.NoError(t, exec.CommandContext(ctx, "git", "-C", h.repo, "cat-file", "-e", keep+"^{commit}").Run()) + if tc.want == WorktreeRemoved { + refs := h.git(h.repo, "for-each-ref", "--contains", keep, "--format=%(refname)") + assert.NotEmpty(t, refs, "the commit is still reachable from a ref") + } + } + }) + } +} + +// A crash while a worktree is frozen leaves a removing row and frozen names; +// the next start restores them and judges again. +func TestACrashWhileFrozenIsRestoredOnTheNextStart(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, row := h.prepare(301) + h.write(workDir, "wip.txt", "wip\n") + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + // The crash happens inside an operator's prune, the only thing that + // freezes a worktree. + h.wt.whileFrozen = func(string) error { return errors.New("crash") } + _, err := h.wt.Prune(ctx, nil) + require.NoError(t, err) + require.DirExists(t, frozenName(row.Path)) + require.Equal(t, WorktreeRemoving, h.row(workDir).State) + + h.wt.whileFrozen = nil + require.NoError(t, h.wt.Recover(ctx)) + after := h.row(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedFinished, after.RetainedReason) + assert.FileExists(t, filepath.Join(workDir, "wip.txt")) + assert.DirExists(t, row.AdminDir) + assert.NoDirExists(t, frozenName(row.Path)) +} + +// A row whose record's place was never stored has it found, proven and stored +// before anything is renamed, so a crash while frozen is restored too. +func TestACrashWhileFrozenWithoutAStoredRecordIsRestored(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, row := h.prepare(302) + _, err := h.ledger.db.ExecContext(ctx, `UPDATE worktrees SET admin_dir = '' WHERE id = ?`, row.ID) + require.NoError(t, err) + h.write(workDir, "wip.txt", "wip\n") + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + h.wt.whileFrozen = func(string) error { return errors.New("crash") } + _, err = h.wt.Prune(ctx, nil) + require.NoError(t, err) + require.NotEmpty(t, h.row(workDir).AdminDir, "stored before the freeze") + + h.wt.whileFrozen = nil + require.NoError(t, h.wt.Recover(ctx)) + after := h.row(workDir) + assert.Equal(t, RetainedFinished, after.RetainedReason) + assert.DirExists(t, row.AdminDir) + assert.NoFileExists(t, filepath.Join(row.AdminDir, "locked"), "the connector's lock goes with the freeze") +} + +// Invariant 3: configuration that verifies signatures does not make the +// connector's git run a program. +func TestSignatureVerificationDoesNotRun(t *testing.T) { + h := newWorktreeHarness(t) + marker := filepath.Join(t.TempDir(), "gpg-ran") + script := filepath.Join(t.TempDir(), "gpg") + require.NoError(t, os.WriteFile(script, []byte("#!/bin/sh\ntouch "+marker+"\nexit 1\n"), 0o700)) + h.git(h.repo, "config", "log.showSignature", "true") + h.git(h.repo, "config", "gpg.program", script) + workDir, row := h.prepare(303) + // A commit carrying a signature header, as a worker could hand-make. + tree := h.git(workDir, "rev-parse", "HEAD^{tree}") + body := "tree " + tree + "\nparent " + row.BaseCommit + "\nauthor T 1 +0000\ncommitter T 1 +0000\ngpgsig -----BEGIN PGP SIGNATURE-----\n \n -----END PGP SIGNATURE-----\n\nsigned\n" + obj := filepath.Join(t.TempDir(), "commit") + require.NoError(t, os.WriteFile(obj, []byte(body), 0o600)) + signed := h.git(workDir, "hash-object", "-t", "commit", "-w", obj) + h.git(workDir, "reset", "-q", "--soft", signed) + + h.discard(workDir) + assert.NoFileExists(t, marker, "no signature program ran") +} + +// Invariant 4: a task branch is deleted in one ref transaction with a check +// that its holder has not moved; a holder moved in between keeps the branch. +// What holds a worktree's commits while it is being deleted is the +// connector's own refs, not someone else's: a branch deleted between the +// transaction and the deletion takes nothing with it. +func TestAHolderThatGoesWhileTheRemovalRunsTakesNothingWithIt(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(310) + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "checkout", "-q", row.Branch) + // Only this branch holds that commit, and it goes the moment the + // removal's transaction is through — while the directory and the record + // are being deleted. + h.git(h.repo, "branch", "keeper", sha) + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"update-ref --stdin"*) "$REAL" "$@"; rc=$?; "$REAL" -C "`+h.repo+`" branch -D keeper >/dev/null 2>&1; exit $rc;; esac`)) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRemoved, after.State) + assert.False(t, exists(row.Path)) + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", sha+"^{commit}").Run()) + refs := h.git(h.repo, "for-each-ref", "--contains", sha, "--format=%(refname)") + assert.Contains(t, refs, RemovingRefPrefix, "the commit is still held by a ref of the connector's own") +} + +// A repository a worker made is git data wherever it is, including the +// directory the task worked in: a force discards files, never commits. +func TestARepositoryMadeWhereGitTracksFilesIsNeverRemoved(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(321) + // A repository made where git tracks files: the worktree's own root, + // which the walk starts at and which is tracked by definition. + h.git(workDir, "init", "-q", "--bare", row.Path) + require.True(t, isGitDir(row.Path)) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.True(t, results[0].ForceRefused) + assert.True(t, exists(filepath.Join(row.Path, "objects")), "the repository a worker made is still there") +} + +// A row whose repository is gone: an operator who names it gets it closed, +// because there is nothing left anywhere to delete or to keep it for. +func TestAForceClosesARowWhoseRepositoryIsGone(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(322) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + require.NoError(t, os.RemoveAll(row.Path)) + require.NoError(t, os.RemoveAll(h.repo)) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action, "a row nothing can read is not a row nothing can close") + assert.Equal(t, WorktreeRemoved, h.row(workDir).State) +} + +// A row that never stored where git's record is does not get closed on the +// strength of not knowing: the repository is asked, and a record that is +// there keeps the row. +func TestARowThatDoesNotKnowWhereItsRecordIsKeepsIt(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(323) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + _, err := h.ledger.db.ExecContext(context.Background(), `UPDATE worktrees SET admin_dir = '' WHERE id = ?`, row.ID) + require.NoError(t, err) + // The directory and the branch go; git's record of the worktree stays. + require.NoError(t, os.RemoveAll(row.Path)) + h.git(h.repo, "update-ref", "-d", "refs/heads/"+row.Branch) + + results, err := h.wt.Prune(context.Background(), nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.Equal(t, RetainedOrphaned, results[0].Reason, "the record is still there") +} + +// A force on an orphan says where the branch stood, because nothing worked +// out what it reached. +func TestAForcedOrphanSaysWhereItsBranchStood(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(324) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + tip := h.git(workDir, "rev-parse", "HEAD") + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + require.NoError(t, os.RemoveAll(row.Path)) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action) + assert.Equal(t, tip, results[0].BranchDeletedAt, "what an operator needs to put it back") + assert.False(t, h.branchExists(row.Branch)) + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", tip+"^{commit}").Run()) +} + +// A force on an orphan whose branch could not be deleted keeps the row: an +// operator is never left with something nothing lists. +func TestAnOrphanWhoseBranchStaysKeepsItsRow(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(320) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + require.NoError(t, os.RemoveAll(row.Path)) + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"update-ref --stdin"*) exit 1;; esac`)) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.Equal(t, RetainedOrphaned, results[0].Reason) + assert.True(t, h.branchExists(row.Branch)) + assert.Equal(t, WorktreeRetained, h.row(workDir).State, "still listed") +} + +// A pseudo-ref can name more than one commit — FETCH_HEAD does, and an +// octopus MERGE_HEAD does — and every one of them goes with the record. +func TestEveryCommitAPseudoRefNamesIsJudged(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(319) + first := h.git(h.repo, "rev-parse", "HEAD") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + second := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "reset", "-q", "--hard", row.BaseCommit) + h.git(workDir, "reflog", "expire", "--expire=now", "--all") + h.git(h.repo, "reflog", "expire", "--expire=now", "--all") + // Nothing else in the record names it: the reset's ORIG_HEAD goes. + origHead := h.git(workDir, "rev-parse", "--path-format=absolute", "--git-path", "ORIG_HEAD") + require.NoError(t, os.Remove(origHead)) + // A fetch's FETCH_HEAD: the held commit first, the unheld one after it. + fetchHead := h.git(workDir, "rev-parse", "--path-format=absolute", "--git-path", "FETCH_HEAD") + require.NoError(t, os.WriteFile(fetchHead, []byte(first+"\t\tbranch 'main' of origin\n"+second+"\tnot-for-merge\tbranch 'other' of origin\n"), 0o600)) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedUnpushed, after.RetainedReason, "the second name is judged too") + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", second+"^{commit}").Run()) +} + +// A bare repository a worker made inside its worktree is git data too: a +// force discards files, never commits, and no ref here could keep these. +func TestABareRepositoryTheWorkerMadeIsNeverRemoved(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(318) + bare := filepath.Join(workDir, "scratch.git") + require.NoError(t, os.MkdirAll(bare, 0o700)) + h.git(workDir, "init", "-q", "--bare", bare) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.True(t, results[0].ForceRefused, "a force does not discard a repository's git data") + assert.True(t, exists(bare)) +} + +// A commit only an old per-worktree ref's reflog reaches, in a repository +// that logs every ref: the reflog lives in the record and goes with it. +func TestACommitOnlyAPerWorktreeRefsReflogReachesIsKept(t *testing.T) { + h := newWorktreeHarness(t) + h.git(h.repo, "config", "core.logAllRefUpdates", "always") + workDir, row := h.prepare(317) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + // A per-worktree ref pointed at it and was moved away; only its reflog + // reaches it now. + h.git(workDir, "update-ref", "refs/worktree/keep", sha) + h.git(workDir, "update-ref", "refs/worktree/keep", row.BaseCommit) + h.git(workDir, "reset", "-q", "--hard", row.BaseCommit) + h.git(workDir, "reflog", "expire", "--expire=now", "HEAD") + h.git(h.repo, "reflog", "expire", "--expire=now", "refs/heads/"+row.Branch) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedUnpushed, after.RetainedReason) + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", sha+"^{commit}").Run()) +} + +// A repository that keeps no reflogs tells the rule nothing about what a +// worktree reached: what cannot be read is not judged clean. +func TestAWorktreeWithNoReflogIsNotJudgedClean(t *testing.T) { + h := newWorktreeHarness(t) + h.git(h.repo, "config", "core.logAllRefUpdates", "false") + workDir, row := h.prepare(308) + // The commit only the reflog would reach, in a repository that keeps + // none: the worktree and its branch are all that hold it. + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "checkout", "-q", row.Branch) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedUnverified, after.RetainedReason) + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", sha+"^{commit}").Run(), "the commit is still there") + + // And the operator can still get rid of it: a force is a decision, not a + // judgment, so a row nothing can read is not a row nothing can clear. In + // a repository that keeps no reflogs there is nothing left pointing at + // that commit for anyone — the connector included — to keep. + results, err := h.wt.Prune(context.Background(), []string{after.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action) + assert.Equal(t, WorktreeRemoved, h.row(workDir).State) + assert.False(t, exists(after.Path)) +} + +// A commit only the record's ORIG_HEAD reaches goes with the record: it is +// judged like any other commit the worktree reaches. +func TestACommitOnlyOrigHeadReachesIsKept(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(309) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "reset", "-q", "--hard", row.BaseCommit) + h.git(workDir, "reflog", "expire", "--expire=now", "--all") + require.Equal(t, sha, h.git(workDir, "rev-parse", "ORIG_HEAD")) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedUnpushed, after.RetainedReason) + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", sha+"^{commit}").Run(), "the commit is still there") +} + +// The judgment leans on every ref that holds a commit the worktree reaches, +// not only the one holding its branch tip: a holder that moves between the +// check and the removal keeps the worktree, commit and all. +func TestAHolderOffTheBranchTipMustNotMoveEither(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(305) + h.write(workDir, "off.txt", "off\n") + h.git(workDir, "add", "off.txt") + h.git(workDir, "commit", "-q", "-m", "off the tip") + off := h.git(workDir, "rev-parse", "HEAD") + // Another branch holds that commit, and the task branch is rolled back to + // its base: the commit is reachable from the worktree's reflogs, and what + // holds it is not the branch tip's holder. + h.git(h.repo, "branch", "keeper", off) + h.git(workDir, "reset", "-q", "--hard", row.BaseCommit) + // Just before the removal's transaction, keeper is moved off it. + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"update-ref --stdin"*) "$REAL" -C "`+h.repo+`" update-ref refs/heads/keeper `+row.BaseCommit+`;; esac`)) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State, "the judgment no longer stands") + assert.True(t, exists(workDir), "the worktree is still there") + assert.Equal(t, off, h.git(workDir, "rev-parse", "HEAD@{1}"), "and the commit with it") + assert.Empty(t, h.git(h.repo, "for-each-ref", "--format=%(refname)", RetainedRefPrefix), + "a removal that did not happen leaves no ref of the connector's own behind") +} + +// The commit a worktree was made from is judged like any other: the route's +// branch usually holds it, but a route reset since is not evidence that it +// does, and the task branch is then the only thing reaching it. +func TestTheBaseCommitIsNotAssumedHeld(t *testing.T) { + h := newWorktreeHarness(t) + // A commit on the route's branch that was never pushed, and the worktree + // made from it. + h.write(h.repo, "app/base.txt", "base\n") + h.git(h.repo, "add", ".") + h.git(h.repo, "commit", "-q", "-m", "base") + base := h.git(h.repo, "rev-parse", "HEAD") + workDir, row := h.prepare(307) + require.Equal(t, base, row.BaseCommit) + // The route's branch is reset away: nothing but the task branch reaches + // that commit any more. + h.git(h.repo, "reset", "-q", "--hard", "HEAD~1") + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedUnpushed, after.RetainedReason) + assert.True(t, h.branchExists(row.Branch), "the branch reaching the commit is kept") + assert.Equal(t, base, h.git(h.repo, "rev-parse", "refs/heads/"+row.Branch)) +} + +// The commits are held before the branch that reaches them goes, not after: +// nothing between the two can leave a commit with no ref at all. +func TestTheCommitsAreHeldBeforeTheBranchGoes(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(314) + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "checkout", "-q", row.Branch) + h.git(h.repo, "branch", "keeper", sha) + // The moment the branch's transaction runs, every commit must already be + // held by a ref of the connector's own. + // Only the first transaction is looked at: that is the branch's. + held := filepath.Join(t.TempDir(), "held") + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"update-ref --stdin"*) [ -f `+held+` ] || "$REAL" -C "`+h.repo+`" for-each-ref --contains `+sha+` --format='%(refname)' refs/basecamp-connect/removing/ > `+held+`;; esac`)) + + after := h.discard(workDir) + require.Equal(t, WorktreeRemoved, after.State) + data, err := os.ReadFile(held) + require.NoError(t, err) + assert.Contains(t, string(data), RemovingRefPrefix, "the commit was held when the branch's transaction ran") +} + +// A worktree whose directory something outside the connector removed: the +// row says so, git's record and the task branch are left exactly as they are, +// and nothing about what they reach is judged. An operator who names it is +// told what goes and it goes. +func TestADirectoryRemovedFromUnderTheConnectorIsOrphanedNotJudged(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(315) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "reset", "-q", "--hard", row.BaseCommit) + require.Equal(t, sha, h.git(workDir, "rev-parse", "ORIG_HEAD")) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + require.NoError(t, os.RemoveAll(row.Path)) + + results, err := h.wt.Prune(context.Background(), nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.Equal(t, RetainedOrphaned, results[0].Reason) + assert.True(t, h.branchExists(row.Branch), "the branch is left alone") + assert.True(t, exists(row.AdminDir), "and so is git's record of the worktree") + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", sha+"^{commit}").Run()) + + // The explicit discard, naming it: the branch goes, the record is left + // for `git worktree prune`, and the row is closed. + forced, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, forced, 1) + assert.Equal(t, PruneForced, forced[0].Action) + assert.False(t, h.branchExists(row.Branch)) + assert.Equal(t, WorktreeRemoved, h.row(workDir).State) +} + +// The connector deletes no ref of its own accord either: a task whose +// directory is gone keeps its branch, and with it the commits only that +// branch's reflog reaches. +func TestATaskEndDeletesNoBranchOfItsOwnAccord(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(312) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + // The branch is back at its base, and only its own reflog reaches that + // commit; the directory is gone when the task ends. + h.git(h.repo, "update-ref", "refs/heads/"+row.Branch, row.BaseCommit) + require.NoError(t, os.RemoveAll(row.Path)) + + after := h.finish(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.True(t, h.branchExists(row.Branch), "the branch is the operator's to lose, not the connector's") + assert.NoError(t, exec.CommandContext(context.Background(), "git", "-C", h.repo, "cat-file", "-e", sha+"^{commit}").Run()) + assert.Equal(t, sha, h.git(h.repo, "rev-parse", row.Branch+"@{1}"), "its reflog still reaches the commit") +} + +// A prune of the same worktree is judged on what holds its commits, not on +// what a removal that stopped halfway left behind. +func TestAnAbandonedRemovalsRefsDoNotPassTheNextJudgment(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(313) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + sha := h.git(workDir, "rev-parse", "HEAD") + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + // A removal that got as far as holding the commits and then stopped: the + // refs it made are still there. + held, err := h.wt.anchor(context.Background(), row, []string{sha}) + require.NoError(t, err) + require.Len(t, held, 1) + require.Equal(t, sha, h.git(h.repo, "rev-parse", held[0])) + + results, err := h.wt.Prune(context.Background(), nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action, "the commit is still unpushed") + assert.Equal(t, RetainedUnpushed, results[0].Reason, "a ref the connector left behind holds nothing for anybody else") + assert.True(t, exists(row.Path)) +} + +func TestABranchWhoseHolderMovedIsNotDeleted(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(304) + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "c") + h.git(workDir, "push", "-q", "origin", row.Branch) + // Just before the transaction, the only holder, the remote-tracking ref, + // is reset away. + h.wt = h.worktrees(fakeGit(t, `case "$*" in *"update-ref --stdin"*) "$REAL" -C "`+h.repo+`" update-ref refs/remotes/origin/`+row.Branch+` `+row.BaseCommit+`;; esac`)) + after := h.discard(workDir) + // Nothing holds the commit any more: the worktree and its branch stay. + assert.Equal(t, WorktreeRetained, after.State) + assert.True(t, h.branchExists(row.Branch), "the branch holding the commit alone is kept") + assert.True(t, exists(workDir), "the worktree the branch is checked out in is kept") +} + +// Git data of a repository inside the worktree — one a worker made, not a +// submodule of the route's — is work no ref can keep: never removed, and a +// force is refused. +func TestARepositoryTheWorkerMadeIsNeverRemoved(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + workDir, _ := h.prepare(400) + nested := filepath.Join(workDir, "vendor", "lib") + require.NoError(t, os.MkdirAll(nested, 0o700)) + h.git(nested, "init", "-q", "-b", "main") + h.write(nested, "lib.txt", "lib\n") + h.git(nested, "add", ".") + h.git(nested, "commit", "-q", "-m", "only copy") + commit := h.git(nested, "rev-parse", "HEAD") + + row := h.discard(workDir) + require.Equal(t, RetainedDirty, row.RetainedReason) + results, err := h.wt.Prune(ctx, []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneKept, results[0].Action) + assert.True(t, results[0].ForceRefused) + assert.DirExists(t, filepath.Join(nested, ".git")) + assert.NoError(t, exec.CommandContext(ctx, "git", "-C", nested, "cat-file", "-e", commit+"^{commit}").Run()) +} + +// A crash between `worktree add --no-checkout` and the checkout leaves a +// directory that was never checked out: nothing in it to lose. +func TestAWorktreeThatWasNeverCheckedOutIsKeptThenPruned(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + base := h.git(h.repo, "rev-parse", "HEAD") + path := filepath.Join(h.root, "repo", "401-abcdef") + record := Worktree{ + Path: path, WorkDir: filepath.Join(path, "app"), Route: filepath.Join(h.repo, "app"), Repository: h.repo, + Branch: BranchPrefix + "401-abcdef", BaseCommit: base, OriginatingEventID: 401, State: WorktreeCreating, + } + id, err := h.ledger.BeginWorktree(ctx, record) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + h.git(h.repo, "update-ref", "refs/heads/"+record.Branch, base, "") + require.NoError(t, h.ledger.WorktreeBranchCreated(ctx, id)) + h.git(h.repo, "worktree", "add", "--no-checkout", "-q", path, record.Branch) + require.FileExists(t, filepath.Join(path, ".git")) + + // Recovery keeps it, as it keeps everything on disk. + require.NoError(t, h.wt.Recover(ctx)) + rows, err := h.ledger.Worktrees(ctx) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.Equal(t, WorktreeRetained, rows[0].State) + assert.True(t, exists(path)) + + // The operator's prune reads it for what it is: a checkout that never + // happened, holding nothing. + results, err := h.wt.Prune(ctx, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneRemoved, results[0].Action) + assert.False(t, exists(path)) +} + +// A frozen name already taken keeps the worktree, untouched. +func TestAFrozenNameAlreadyTakenKeepsTheWorktree(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(402) + require.NoError(t, os.Mkdir(frozenName(row.Path), 0o700)) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRetained, after.State) + assert.Equal(t, RetainedUnverified, after.RetainedReason) + assert.DirExists(t, workDir) + assert.NoFileExists(t, filepath.Join(row.AdminDir, "locked"), "no lock is left on the record") +} + +// A row without a stored record, in a repository that writes relative worktree +// paths, is still proven and removed. +func TestALegacyRowWithRelativePathsIsRemoved(t *testing.T) { + h := newWorktreeHarness(t) + ctx := context.Background() + h.git(h.repo, "config", "worktree.useRelativePaths", "true") + workDir, row := h.prepare(403) + _, err := h.ledger.db.ExecContext(ctx, `UPDATE worktrees SET admin_dir = '' WHERE id = ?`, row.ID) + require.NoError(t, err) + + after := h.discard(workDir) + assert.Equal(t, WorktreeRemoved, after.State) + assert.False(t, exists(row.Path)) + assert.NoDirExists(t, row.AdminDir) +} + +// A conflicted merge leaves AUTO_MERGE naming the tree ort merged to, which +// is an object no ref can be asked to contain. A force must still go through: +// a tree names no history, so there is nothing there to keep. +func TestAForcedPruneIsNotStoppedByAPseudoRefNamingATree(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(325) + h.git(h.repo, "branch", "theirs") + h.git(h.repo, "checkout", "-q", "theirs") + h.write(h.repo, "app/README", "theirs\n") + h.git(h.repo, "commit", "-q", "-am", "theirs") + h.git(h.repo, "checkout", "-q", "main") + h.write(workDir, "README", "ours\n") + h.git(workDir, "commit", "-q", "-am", "ours") + h.gitConflicting(workDir, "merge", "theirs") + autoMerge, err := os.ReadFile(filepath.Join(row.AdminDir, "AUTO_MERGE")) + require.NoError(t, err, "the conflicted merge left AUTO_MERGE in the record") + require.Equal(t, "tree", h.git(h.repo, "cat-file", "-t", strings.TrimSpace(string(autoMerge)))) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action, "reason: %s", results[0].Reason) + assert.False(t, exists(row.Path)) +} + +// A HEAD that names no commit — a worker's `checkout --orphan` — reaches +// nothing through HEAD. That is not a git the connector could not run, and a +// force is not refused over it: the worktree would otherwise be one no +// command could ever remove. +func TestAWorktreeWhoseHeadNamesNoCommitIsStillForced(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(326) + h.git(workDir, "checkout", "-q", "--detach") + h.write(workDir, "c.txt", "c\n") + h.git(workDir, "add", "c.txt") + h.git(workDir, "commit", "-q", "-m", "detached") + commit := h.git(workDir, "rev-parse", "HEAD") + h.git(workDir, "checkout", "-q", "--orphan", "fresh") + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action, "reason: %s", results[0].Reason) + assert.False(t, exists(row.Path)) + assert.Contains(t, h.git(h.repo, "for-each-ref", "--format=%(objectname)", RetainedRefPrefix), commit, + "what HEAD stood at before the orphan is kept: only its reflog still reaches it") +} + +// A per-worktree ref names whatever a worker put under it, and the judgment +// is about the commit that reaches: an annotated tag is history to keep, +// which is what asking git what contains it used to say. +func TestAPerWorktreeRefAtAnAnnotatedTagKeepsItsCommit(t *testing.T) { + h := newWorktreeHarness(t) + workDir, row := h.prepare(327) + // A commit nothing else reaches: its branch and its tag ref are gone, + // and the tag object is left only under the worktree's own ref. + h.git(h.repo, "checkout", "-q", "-b", "temp") + h.write(h.repo, "app/t.txt", "t\n") + h.git(h.repo, "add", "app/t.txt") + h.git(h.repo, "commit", "-q", "-m", "tagged") + tagged := h.git(h.repo, "rev-parse", "HEAD") + h.git(h.repo, "tag", "-a", "-m", "kept", "kept") + tag := h.git(h.repo, "rev-parse", "kept") + h.git(h.repo, "checkout", "-q", "main") + h.git(h.repo, "branch", "-q", "-D", "temp") + h.git(h.repo, "tag", "-d", "kept") + h.git(workDir, "update-ref", "refs/worktree/kept", tag) + require.Equal(t, WorktreeRetained, h.finish(workDir).State) + + results, err := h.wt.Prune(context.Background(), []string{row.Path}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, PruneForced, results[0].Action, "reason: %s", results[0].Reason) + assert.Contains(t, h.git(h.repo, "for-each-ref", "--format=%(objectname)", RetainedRefPrefix), tagged, + "the tag's commit is kept, not dropped with the tag") +} diff --git a/skills/basecamp-connect/SKILL.md b/skills/basecamp-connect/SKILL.md index e3899af6f..1b9f18d2d 100644 --- a/skills/basecamp-connect/SKILL.md +++ b/skills/basecamp-connect/SKILL.md @@ -137,6 +137,7 @@ widens trust. "222": { "path": "/home/me/Work/app", "class": "internal", "watch_completions": true } }, "driver": "spawn", + "worker": "claude", "concurrency": 2, "deadline": "45m0s", "worktrees": false @@ -155,9 +156,10 @@ widens trust. | `projects..class` | A label carried on the project's records: 1 to 40 lowercase letters, digits, `-` and `_`, starting with a letter or digit | `--class '='`; `--class '='` clears it | | `projects..watch_completions` | Every trusted completion in the project reaches the agent, without assigning it | `--watch-completions `, `--no-watch-completions ` | | `driver` | How workers are run: `spawn` (default) or `acp` | `--driver` | +| `worker` | Which coding agent a spawn worker is: `claude` (default) or `codex` | `--worker` | | `concurrency` | Workers at once, 1 to 32 (default 2) | `--concurrency` | | `deadline` | Time limit per task, 1m to 24h (default 45m) | `--deadline 90m` | -| `worktrees` | Each task gets its own git worktree of the routed directory | `--worktrees`, `--worktrees=false` | +| `worktrees` | Each task gets its own git worktree of the routed directory, kept when the task ends and removed only by `connect worktrees prune` | `--worktrees`, `--worktrees=false` | **Never edit connect.json by hand.** It is the trust anchor: setup verifies every person and route before writing it, writes it owner-only, and parses it @@ -312,7 +314,7 @@ project names up the same way as on first setup, and quote values by the Shell q | Trust only the operator, or project members | `--trust operator` / `--trust project` (leaving allowlist mode drops the list) | | Trust specific people | `--allow ` for each; the list you pass **replaces** the old one, so pass everyone who stays | | Change the operator | `--operator-profile ''` | -| Change workers | `--driver`, `--concurrency`, `--deadline`, `--worktrees` / `--worktrees=false` | +| Change workers | `--driver`, `--worker claude` / `--worker codex`, `--concurrency`, `--deadline`, `--worktrees` / `--worktrees=false` | | Replace the agent's credential (only with the person's consent: it rotates the secret) | `basecamp auth agent connect -P ''`, then setup with no flags to re-check | A class or watch setting needs the project routed first, in the same run or an diff --git a/skills/basecamp/SKILL.md b/skills/basecamp/SKILL.md index d3ad35c32..6b8de4a3a 100644 --- a/skills/basecamp/SKILL.md +++ b/skills/basecamp/SKILL.md @@ -1456,6 +1456,9 @@ basecamp auth agent connect -P agent # Connect this computer to a basecamp connect setup -P agent --operator-profile --route = # Set up a local agent connector on a connected profile (run `auth agent connect` first): verifies trust, checks token, identity, scope, ticket mint and project reads, then writes connect.json basecamp connect -P agent # Run the connector in the foreground: hear the agent's events, admit what a trusted person asks, and hand the work to a local coding agent that replies as the agent basecamp connect -P agent --project --shadow # Narrow it to one project, and watch without acting: an isolated state directory, nothing dispatched and nothing posted +basecamp connect setup -P agent --worker codex --worktrees # Run workers with Codex instead of Claude Code, and give each task its own git worktree +basecamp connect worktrees list -P agent --json # The worktrees the connector kept: every task's, with its size on disk, git's record of it, and why it is kept (finished, dirty, unpushed, locked, moved, unverified, orphaned) +basecamp connect worktrees prune -P agent # The only thing that removes a worktree: removes the kept ones that hold no work; --force removes one that does (on disk: every commit it reaches is kept under refs/basecamp-connect/retained/; orphaned: the task branch goes and the commit it stood at is reported) ``` `basecamp connect` runs until it is stopped: it is not a command to call for an @@ -1467,6 +1470,16 @@ refuses a second connector for the same agent, and takes `--project` (repeatable to hear and dispatch only those projects. Run it under a supervisor rather than from a session you will close. +With worktrees on, a task's worktree is kept when the task ends — the connector +removes none of its own accord — and listed by `connect worktrees list` with its +size. Removing them is the operator's call: `connect worktrees prune` removes +those that hold no work, and never pass `--force` for a path the operator did not +name. A worktree whose directory something else removed is reported as +`orphaned`: the connector leaves git's record of it and the task branch exactly +as they are, and only a force on its path deletes the branch. A Codex worker cannot commit (its sandbox cannot write the +worktree's git data), so with Codex every task that edits files leaves a kept +worktree. + **Before running ANY of the logins above, check `oauth_type`.** `basecamp auth status --json` reports it, and `agent` means the profile is a Basecamp agent: a principal with no person behind it, which authenticates with its OAuth client