diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b030ac..a7e5fa85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # Changelog -## 0.9.3 - Unreleased +## 0.9.3 - 2026-08-29 +- Fix repeated gzip-only portable initialization and add a locked, bounded, preservation-first `portable refresh` subscriber command with automatic Git maintenance disabled across portable operations. +- Preserve writable portable runtime data across refresh and ordinary reads, keep prepared raw replicas updating through publisher transitions, report gzip subscriber status from the usable artifact/runtime, and retain explicit Git config isolation with safe scope diagnostics. +- Reject credential-bearing portable remote URLs while retaining SSH transport usernames. +- Report actionable portable Git failure categories without exposing raw remote or credential-helper diagnostics. - Update the minimum Go toolchain to 1.26.7, SQLite to v1.57.0, and the current compatible Go dependency closure; keep Go 1.27 on hold until managed CodeQL supports it. - Refresh govulncheck, deadcode, the Dockerfile frontend, and the docs build's Node runtime. - Stabilize the coverage gate with direct tests for immutable SQLite reads, PR detail lookups, and vector scopes. diff --git a/README.md b/README.md index af3700d4..2be31190 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Octopool owns pooled live `gh` reads. Gitcrawl keeps local mirror, search, clust | Job | Command | Guide | | --- | --- | --- | | Check archive health | `gitcrawl status` / `gitcrawl doctor` | [Configuration](docs/configuration.md) | +| Refresh a portable subscriber | `gitcrawl portable refresh --expected-remote URL` | [Portable stores](docs/portable-stores.md#routine-subscriber-refresh) | | Mirror GitHub threads | `gitcrawl sync owner/repo` | [Sync](docs/sync.md) | | Search threads or indexed code | `gitcrawl search ...` | [Search](docs/search.md) | | Build and inspect clusters | `gitcrawl refresh`, `clusters`, `tui` | [Clustering](docs/clustering.md) | diff --git a/docs/automation.md b/docs/automation.md index 237f7b4f..76806fe9 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -60,6 +60,26 @@ Run `gitcrawl refresh owner/repo` on a cron, systemd timer, or `launchd` agent e For multiple repos, loop in a small shell script — gitcrawl is happy to run sequentially against a shared SQLite file. +### Portable subscribers + +After one-time initialization, schedule `portable refresh`, not repeated +`init` or `doctor`. It retains configuration and refuses unsafe checkout state: + +```bash +gitcrawl --config /path/to/config.toml portable refresh \ + --expected-remote https://github.com/example/archive-store.git \ + --git /absolute/path/to/git --timeout 2m \ + --min-free-bytes 2147483648 --max-growth-bytes 2147483648 \ + --json > refresh.json 2> refresh.log +``` + +Treat nonzero exits as refusals requiring inspection. Do not automatically +delete locks, temporary packs, backups or SQLite sidecars and retry. A `partial` +result identifies an interrupted advancement, not a rollback. A +`mirror_result` of `preserved-local` means local runtime writes remain intact. +See [Portable stores](/portable-stores/#routine-subscriber-refresh) for the +admission rules, sampled growth budget and exact JSON contract. + ## Agent recipes ### "Look up an issue without burning quota" diff --git a/docs/commands.md b/docs/commands.md index 66d02409..01efc987 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -136,6 +136,7 @@ octopool gh api repos/openclaw/openclaw/pulls/123 | Command | Purpose | Docs | | --- | --- | --- | +| `gitcrawl portable refresh --expected-remote URL [--store-dir PATH --portable-db PATH --branch main --git PATH --timeout 2m --min-free-bytes N --max-growth-bytes N --json]` | Validate and fast-forward a clean configured subscriber without regenerating config or invoking repair | [Portable stores](/portable-stores/#routine-subscriber-refresh) | | `gitcrawl portable prune [--body-chars --no-vacuum --include-sync-failures --no-publish --json]` | Build a compact portable v2 backup and (optionally) `VACUUM` for publishing | [Portable stores](/portable-stores/#publishing-gitcrawl-portable-prune) | | `gitcrawl portable export --profile current-state-v1 --output-dir PATH [--repository owner/repo --body-chars --database-name --public-path --max-bytes --json]` | Create a validated, optionally repository-scoped database-and-manifest generation without changing or publishing the active database | [Portable stores](/portable-stores/#derived-generations-gitcrawl-portable-export) | diff --git a/docs/portable-stores.md b/docs/portable-stores.md index fbec89e2..207b6d2b 100644 --- a/docs/portable-stores.md +++ b/docs/portable-stores.md @@ -55,9 +55,144 @@ size and SHA-256, runs SQLite `quick_check`, and only then atomically replaces the runtime mirror. Legacy raw SQLite stores continue to use the same manifest without compression fields. +Initialization validates portable arguments before invoking Git and validates +the artifact before saving configuration. Repeated initialization and a +publisher's raw-to-gzip transition do not require a raw `.db` in the checkout. +Use `init` for setup; it still regenerates configuration on success. + +## Routine subscriber refresh + +Use the strict subscriber command for scheduled updates: + +```bash +gitcrawl --config /path/to/config.toml portable refresh \ + --expected-remote https://github.com/example/archive-store.git \ + --git /absolute/path/to/git \ + --timeout 2m \ + --min-free-bytes 2147483648 \ + --max-growth-bytes 2147483648 \ + --json +``` + +This uses the configured logical database and its existing checkout. Optional +`--store-dir` and `--portable-db` assert that those configured paths match the +intended store; they do not reconfigure it. `--expected-remote` is required +because legacy configs do not pin an origin URL. `--branch` defaults to `main`; +the checkout must be on that branch and track the matching origin branch. +No credentials belong in the URL or command line; use Git's credential helper. +HTTP(S) URL userinfo and passwords in any URL are refused. SSH transport +usernames such as `ssh://git@host/repo` and `git@host:repo` remain supported. + +Refresh takes a nonblocking advisory lock for the canonical store path. All +Gitcrawl portable Git writers, legacy recovery, runtime promotion and CLI +writable-runtime sessions share that lease. Symlink aliases converge. Its +permanent sibling `.STORE.gitcrawl.lock` file is never removed or stolen based +on age; the operating system releases ownership when the process exits. +An occupied lock is a refusal, not a reason to kill another process. + +Strict refresh refuses dirty indexes/worktrees, **all untracked or ignored +checkout files**, hidden index entries, unrelated origins, divergent history, +active Git locks or operation state, orphan/temporary packs, hooks, filters, +attributes, submodules, linked worktrees, alternate object stores, config +includes/redirections and unsupported extensions. Resolve these deliberately +outside the subscriber command. It never resets, cleans, prunes, repacks, +reclones, deletes backups or sidecars, or invokes reader/doctor auto-repair. + +Admission examines every exposed Git configuration scope. Refusals identify +the scope and unsafe category without printing keys or values. Single-valued +hooks-path, fsmonitor, attributes-file and SSH-command settings overridden by +the portable runner are inert; actual checkout hooks/attributes and exposed +filter definitions still cause refusal. For a dedicated subscriber that must +exclude machine-wide filters, Unix callers can set +`GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1` for this invocation (Windows: +`GIT_CONFIG_GLOBAL=NUL GIT_CONFIG_NOSYSTEM=1`). `GIT_CONFIG_SYSTEM` may also point +to the platform null device. These scope-removal controls survive sanitization; +arbitrary config paths, repository redirection and config injection do not. +Disabling global configuration also removes global credential helpers, so the +remaining authentication configuration must support the intended remote. +No global files are changed, and repository-local safety checks still apply. + +Fetch requests only the intended branch into `FETCH_HEAD`, with no pruning, +tags, submodule recursion or remote-tracking ref updates. Both existing HEAD +and the tracking ref must be ancestors of the frozen fetched commit. Only its +manifest and selected artifact are extracted into private staging beside the +runtime mirror. Existing raw/gzip digest, expanded-size, SQLite `quick_check` +and semantic-identity validation run before a fast-forward checkout. Tracking +ref advancement uses compare-and-swap. Configuration, refs, store identity, +Git metadata, cleanliness and capacity are checked again at mutation boundaries. + +The validated runtime generation is atomically renamed outside the checkout. +A runtime with bytes differing from its recorded source digest, or any SQLite +sidecars, is preserved as `preserved-local`; it is never overwritten to make +the subscriber appear fresh. Newly materialized raw replicas record their digest +even without a manifest. Verified disposable replicas use immutable SQLite reads +so ordinary reads do not create sidecars that would claim local ownership. +Writable CLI opens record local ownership before +SQLite migrations or writes. Ordinary reads retain that ownership and the +original source identity across later publisher generations; local closures, +vectors and other runtime work survive repeated reads. A corrupt locally owned +runtime reports an error without replacement; a corrupt disposable replica +still follows the legacy recovery path. Other Gitcrawl CLI writers obey the lease, but +external Git/SQLite writers do not. Observable changes cause refusal; this is +not a universal filesystem transaction or protection against a hostile writer. + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--expected-remote URL` | required | Expected identity of `origin` | +| `--store-dir PATH` | configured store | Assert canonical store directory | +| `--portable-db PATH` | configured logical DB | Assert clean relative slash path; never the gzip path | +| `--branch NAME` | `main` | Expected local and origin branch | +| `--git PATH` | `GITCRAWL_PORTABLE_GIT`, then resolved process PATH | Absolute executable; no login shell is used | +| `--timeout DURATION` | `2m` | Total operation deadline, plus bounded process cleanup | +| `--min-free-bytes N` | `2147483648` (2 GiB) | Free-space reserve on store and staging filesystems | +| `--max-growth-bytes N` | `2147483648` (2 GiB) | Positive logical-file growth budget | +| `--json` | off | Structured success, no-op, refusal or partial result | + +Admission requires reserve **plus the full growth budget** available on both +filesystems. Growth is the sum of positive per-path size deltas under the +checkout (including `.git`) and runtime database directory, relative to +admission. Deleting/shrinking old files does not credit the budget. Metadata +scans run at stage boundaries and every 100 ms in flight, with a 200,000-file +scan limit; they never read or copy historical pack contents. Blob extraction +is capped at its frozen Git size, and manifest-based staging estimates allow +for inflation and semantic-identity copies. A budget is a sampled cancellation +boundary, **not an OS quota**: rapid Git or filesystem writes can overshoot +between scans. Reserve is also affected by unrelated processes. Use filesystem +quotas where a hard physical allocation ceiling is required. + +On cancellation, owned Git process groups receive a graceful termination +request, then are forced after at most 750 ms and reaped (pipe cleanup adds at +most one second). Windows starts Git suspended, assigns a kill-on-close job, +then resumes it; console break is best-effort before job termination. Unrelated +processes are never targeted. Only this operation's private staging is removed. +Filesystem and SQLite cancellation is cooperative; a kernel I/O stall can +delay cancellation or process reaping beyond the requested deadline. +Fetched objects and `FETCH_HEAD` may remain after failure. If checkout, tracking +ref or mirror advancement has already started, JSON reports `partial`; no +successful rollback is claimed, and an interrupted Git operation may need +operator inspection before another strict refresh. + +JSON reports `stage`, `result` (`updated`, `no-op`, `refused`, `partial`), +`before_commit`, `after_commit`, `target_commit`, `artifact_id`, `sha256`, +`artifact_bytes`, `mirror_destination`, `mirror_result`, `capacity` and +`elapsed_ms`. A refusal includes a bounded `reason` and exits nonzero. Stderr +contains stage/elapsed/result diagnostics; stdout remains data. A no-op means +the refs already match; a missing runtime may still be materialized. Git failures +use fixed, credential-safe repository, authentication, connection or disk-space +diagnostics when recognized, with general troubleshooting guidance otherwise; +raw remote/helper stderr is never included. + +All portable Git entry points explicitly set `maintenance.auto=false` and +`gc.auto=0`, including init, implicit reader fetch/merge and retained recovery +clone/reset helpers. Fetch also uses `--no-auto-maintenance`. Hooks, fsmonitor +and recursive submodule operations are disabled in the portable runner. +Unsupported Git options fail closed; there is no fallback that drops safety +flags. Set `GITCRAWL_PORTABLE_GIT` to an absolute path to select Git for legacy +portable commands as well. Gitcrawl does not change global or local Git policy. + ## How read-only commands behave -Read-only commands (`search`, `threads`, `clusters`, `cluster-detail`, `neighbors`, the TUI) refresh the portable-store checkout before reading, so they always see the latest published data: +Read-only commands (`search`, `threads`, `clusters`, `cluster-detail`, `neighbors`, the TUI) normally refresh the portable-store checkout before reading. A locally owned writable runtime continues serving its local data instead of accepting publisher replacement: - The refresh is best-effort and non-interactive - SSH attempts are bounded so an offline remote does not hang the CLI @@ -68,6 +203,23 @@ Read-only commands (`search`, `threads`, `clusters`, `cluster-detail`, `neighbor If the remote is unreachable, the read still answers from the local checkout. +`status` is observational: it uses an existing runtime without fetching, +repairing, migrating, or promoting anything. It reports `state: stale` when +that runtime's recorded source generation differs from the checkout, with a +warning for writable local state. Before a gzip-only subscriber has a runtime, +status validates and reads the artifact in disposable temporary storage; its +inventory reports the gzip path and compressed on-disk bytes, with a warning +explaining that distinction. It never reports an absent logical `.db` as an +empty, current subscriber. + +This is the **legacy reader recovery contract**, separate from strict +`portable refresh`: a marked malformed store can still be backed up, reset and +recloned with the existing recovery backoff. Its historical stale-index-lock +check requires both age and an external open-file probe. Do not use these +commands as a preservation-first subscriber verification step. They now share +the canonical store lease and maintenance suppression, but do not inherit +strict refresh's capacity limits or clean-only policy. + ## How write commands behave Write commands (`sync`, `embed`, `refresh`, `portable prune`, `cluster`, neighbor generation) open a **writable runtime mirror** alongside the portable checkout so new GitHub data, vectors, and overrides persist without partially mutating the published portable store. When this redirect engages, gitcrawl prints one stderr notice naming both the runtime mirror and the checkout database. diff --git a/internal/cli/app.go b/internal/cli/app.go index 65798fd7..45a6c9c8 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -143,6 +143,11 @@ func (a *App) Run(ctx context.Context, args []string) error { if releaseNotificationAllowed(rest) { a.maybeNotifyRelease(ctx, rest) } + // Writable runtimes retain ownership through SQLite Close. Readers release + // after mirror preparation, so a long-lived TUI does not block subscribers. + session := &portableCommandSession{} + ctx = context.WithValue(ctx, portableCommandKey{}, session) + defer session.close() switch rest[0] { case "version": @@ -3300,6 +3305,9 @@ func (a *App) runInit(ctx context.Context, args []string) error { return usageErr(err) } a.applyCommandJSON(*jsonOut) + if fs.NArg() != 0 { + return usageErr(fmt.Errorf("init does not take positional arguments")) + } localDBPath := strings.TrimSpace(*dbPath) isolatedRuntimeDir := strings.TrimSpace(*runtimeDir) portableStoreURL := strings.TrimSpace(*portableStore) @@ -3338,6 +3346,15 @@ func (a *App) runInit(ctx context.Context, args []string) error { TokenEnv: crawlremote.DefaultTokenEnv, } } else if portableStoreURL != "" { + if err := validatePortableRemote(portableStoreURL); err != nil { + return usageErr(err) + } + if err := validatePortableRelativePath(*portableDB); err != nil { + return usageErr(err) + } + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, portableOperationTimeout) + defer cancel() portableStoreDir = strings.TrimSpace(*storeDir) if portableStoreDir == "" { portableStoreDir = defaultPortableStoreDir(config.ResolvePath(a.configPath), portableStoreURL) @@ -3346,18 +3363,20 @@ func (a *App) runInit(ctx context.Context, args []string) error { if err != nil { return fmt.Errorf("resolve --store-dir path: %w", err) } + var release func() + ctx, release, err = acquirePortableOwner(ctx, portableStoreDir) + if err != nil { + return err + } + defer release() action, err := syncPortableStore(ctx, portableStoreURL, portableStoreDir) if err != nil { return err } portableStoreAction = action - relativeDB := filepath.Clean(filepath.FromSlash(strings.TrimLeft(strings.TrimSpace(*portableDB), "/"))) - if relativeDB == "." || filepath.IsAbs(relativeDB) || strings.HasPrefix(relativeDB, ".."+string(os.PathSeparator)) || relativeDB == ".." { - return usageErr(fmt.Errorf("invalid --portable-db %q", *portableDB)) - } - cfg.DBPath = filepath.Join(portableStoreDir, relativeDB) - if _, err := os.Stat(cfg.DBPath); err != nil { - return fmt.Errorf("portable database not found at %s: %w", cfg.DBPath, err) + cfg.DBPath = filepath.Join(portableStoreDir, filepath.FromSlash(*portableDB)) + if err := validatePortableSQLiteSourceFile(ctx, cfg.DBPath, cfg.DBPath); err != nil { + return fmt.Errorf("validate portable database: %w", err) } } if localDBPath != "" { @@ -3426,6 +3445,8 @@ func (a *App) runPortable(ctx context.Context, args []string) error { return a.runPortablePrune(ctx, args[1:]) case "export": return a.runPortableExport(ctx, args[1:]) + case "refresh": + return a.runPortableRefresh(ctx, args[1:]) default: return usageErr(fmt.Errorf("unknown portable subcommand %q", args[0])) } @@ -3693,12 +3714,23 @@ func safePathName(value string) string { } func syncPortableStore(ctx context.Context, remoteURL, dir string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, portableOperationTimeout) + defer cancel() if strings.TrimSpace(remoteURL) == "" { return "", fmt.Errorf("portable store URL is required") } if strings.TrimSpace(dir) == "" { return "", fmt.Errorf("portable store directory is required") } + if err := validatePortableRemote(remoteURL); err != nil { + return "", err + } + ctx, release, err := acquirePortableOwner(ctx, dir) + if err != nil { + return "", err + } + defer release() + dir = ctx.Value(portableOwnerKey{}).(*portableOwner).root gitDir := filepath.Join(dir, ".git") if info, err := os.Stat(gitDir); err == nil && info.IsDir() { if err := ensurePortableStoreRemote(ctx, remoteURL, dir); err != nil { @@ -3747,7 +3779,7 @@ func syncPortableStore(ctx context.Context, remoteURL, dir string) (string, erro if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { return "", fmt.Errorf("create portable store parent: %w", err) } - if err := runGit(ctx, "", "clone", "--depth", "1", remoteURL, dir); err != nil { + if err := runGit(ctx, "", "clone", "--depth", "1", "--", remoteURL, dir); err != nil { return "", err } if err := markPortableStoreCheckout(dir); err != nil { @@ -3849,13 +3881,19 @@ func isDirtyPortablePullError(err error) bool { } func fastForwardGitCheckout(ctx context.Context, dir string, quiet bool) error { + ctx, release, err := acquirePortableOwner(ctx, dir) + if err != nil { + return err + } + defer release() + dir = ctx.Value(portableOwnerKey{}).(*portableOwner).root branch := currentGitBranch(ctx, dir) remote := gitBranchRemote(ctx, dir, branch) - fetchArgs := []string{"-C", dir, "fetch", "--prune"} + fetchArgs := []string{"-C", dir, "fetch", "--no-auto-maintenance", "--no-prune", "--no-prune-tags", "--no-tags", "--no-recurse-submodules"} if quiet { fetchArgs = append(fetchArgs, "--quiet") } - fetchArgs = append(fetchArgs, remote) + fetchArgs = append(fetchArgs, "--", remote) if err := runGit(ctx, "", fetchArgs...); err != nil { return err } @@ -3870,11 +3908,11 @@ func fastForwardGitCheckout(ctx context.Context, dir string, quiet bool) error { return fmt.Errorf("resolve portable store upstream branch: remote %q has no HEAD", remote) } } - mergeArgs := []string{"-C", dir, "merge", "--ff-only"} + mergeArgs := []string{"-C", dir, "merge", "--ff-only", "--no-autostash", "--no-overwrite-ignore"} if quiet { mergeArgs = append(mergeArgs, "--quiet") } - mergeArgs = append(mergeArgs, target) + mergeArgs = append(mergeArgs, "--", target) return runGit(ctx, "", mergeArgs...) } @@ -3915,6 +3953,9 @@ func gitConfigValue(ctx context.Context, dir, key string) (string, error) { func runGit(ctx context.Context, workdir string, args ...string) error { out, err := runGitCommandOutput(ctx, workdir, args...) if err != nil { + if _, portable := ctx.Value(portableGitKey{}).(portableGitExecutable); portable { + return fmt.Errorf("portable git failed: %w", err) + } return fmt.Errorf("git %s failed: %w\n%s", strings.Join(args, " "), err, strings.TrimSpace(out)) } return nil @@ -3930,6 +3971,10 @@ func runGitCommandOutputWithEnv(ctx context.Context, workdir string, env []strin } func runGitCommandOutputWithEnvSeparate(ctx context.Context, workdir string, env []string, args ...string) (string, string, error) { + if _, portable := ctx.Value(portableGitKey{}).(portableGitExecutable); portable { + out, err := portableGitOutput(ctx, workdir, args...) + return out, "", err + } if err := ctx.Err(); err != nil { return "", "", err } @@ -4324,24 +4369,25 @@ func (a *App) runMetadata(args []string) error { manifest.Capabilities = []string{"metadata", "status", "doctor", "sync", "capture", "coverage", "search", "code-index", "tui", "portable", "remote", "cloud-publish", "clusters", "summaries", "embeddings"} manifest.Privacy = control.Privacy{ContainsPrivateMessages: true, ExportsSecrets: false, LocalOnlyScopes: []string{"github", "git", "sqlite", "portable"}} manifest.Commands = map[string]control.Command{ - "status": {Title: "Status", Argv: []string{"gitcrawl", "status", "--json"}, JSON: true}, - "remote-status": {Title: "Remote archive status", Argv: []string{"gitcrawl", "remote", "status", "--json"}, JSON: true}, - "remote-archives": {Title: "Remote archive list", Argv: []string{"gitcrawl", "remote", "archives", "--json"}, JSON: true}, - "remote-login": {Title: "Remote GitHub login", Argv: []string{"gitcrawl", "remote", "login", "--json"}, JSON: true, Mutates: true}, - "cloud-publish": {Title: "Publish cloud archive", Argv: []string{"gitcrawl", "cloud", "publish", "--json"}, JSON: true, Mutates: true}, - "whoami": {Title: "Remote identity", Argv: []string{"gitcrawl", "whoami", "--json"}, JSON: true}, - "check-update": {Title: "Check for updates", Argv: []string{"gitcrawl", "check-update", "--json"}, JSON: true}, - "doctor": {Title: "Doctor", Argv: []string{"gitcrawl", "doctor", "--json"}, JSON: true}, - "coverage": {Title: "Archive coverage", Argv: []string{"gitcrawl", "coverage", "--json"}, JSON: true}, - "sync": {Title: "Sync repository", Argv: []string{"gitcrawl", "sync", "--json"}, JSON: true, Mutates: true}, - "capture": {Title: "Export conversation capture", Argv: []string{"gitcrawl", "capture", "--json"}, JSON: true}, - "search": {Title: "Search", Argv: []string{"gitcrawl", "search", "--json"}, JSON: true}, - "code-index": {Title: "Code index", Argv: []string{"gitcrawl", "code", "index", "--json"}, JSON: true, Mutates: true}, - "tui": {Title: "Terminal cluster browser", Argv: []string{"gitcrawl", "tui"}}, - "tui-json": {Title: "Terminal cluster data", Argv: []string{"gitcrawl", "tui", "--json"}, JSON: true}, - "portable": {Title: "Portable store tools", Argv: []string{"gitcrawl", "portable", "prune", "--json"}, JSON: true, Mutates: true}, - "clusters": {Title: "Clusters", Argv: []string{"gitcrawl", "clusters", "--json"}, JSON: true}, - "legacy-sync-api": {Title: "Legacy sync-status alias", Argv: []string{"gitcrawl", "sync-status"}, Legacy: true, Deprecated: true}, + "status": {Title: "Status", Argv: []string{"gitcrawl", "status", "--json"}, JSON: true}, + "remote-status": {Title: "Remote archive status", Argv: []string{"gitcrawl", "remote", "status", "--json"}, JSON: true}, + "remote-archives": {Title: "Remote archive list", Argv: []string{"gitcrawl", "remote", "archives", "--json"}, JSON: true}, + "remote-login": {Title: "Remote GitHub login", Argv: []string{"gitcrawl", "remote", "login", "--json"}, JSON: true, Mutates: true}, + "cloud-publish": {Title: "Publish cloud archive", Argv: []string{"gitcrawl", "cloud", "publish", "--json"}, JSON: true, Mutates: true}, + "whoami": {Title: "Remote identity", Argv: []string{"gitcrawl", "whoami", "--json"}, JSON: true}, + "check-update": {Title: "Check for updates", Argv: []string{"gitcrawl", "check-update", "--json"}, JSON: true}, + "doctor": {Title: "Doctor", Argv: []string{"gitcrawl", "doctor", "--json"}, JSON: true}, + "coverage": {Title: "Archive coverage", Argv: []string{"gitcrawl", "coverage", "--json"}, JSON: true}, + "sync": {Title: "Sync repository", Argv: []string{"gitcrawl", "sync", "--json"}, JSON: true, Mutates: true}, + "capture": {Title: "Export conversation capture", Argv: []string{"gitcrawl", "capture", "--json"}, JSON: true}, + "search": {Title: "Search", Argv: []string{"gitcrawl", "search", "--json"}, JSON: true}, + "code-index": {Title: "Code index", Argv: []string{"gitcrawl", "code", "index", "--json"}, JSON: true, Mutates: true}, + "tui": {Title: "Terminal cluster browser", Argv: []string{"gitcrawl", "tui"}}, + "tui-json": {Title: "Terminal cluster data", Argv: []string{"gitcrawl", "tui", "--json"}, JSON: true}, + "portable": {Title: "Portable store tools", Argv: []string{"gitcrawl", "portable", "prune", "--json"}, JSON: true, Mutates: true}, + "portable-refresh": {Title: "Refresh portable subscriber", Argv: []string{"gitcrawl", "portable", "refresh", "--expected-remote", "URL", "--json"}, JSON: true, Mutates: true}, + "clusters": {Title: "Clusters", Argv: []string{"gitcrawl", "clusters", "--json"}, JSON: true}, + "legacy-sync-api": {Title: "Legacy sync-status alias", Argv: []string{"gitcrawl", "sync-status"}, Legacy: true, Deprecated: true}, } return a.writeOutput("metadata", manifest, false) } @@ -4373,22 +4419,11 @@ func (a *App) runStatus(ctx context.Context, args []string) error { if cfg.Remote.Enabled() && cfg.Remote.Mode == crawlremote.ModeCloud { return a.runRemoteStatusWithConfig(ctx, cfg) } - status := store.Status{DBPath: cfg.DBPath} - if _, err := os.Stat(cfg.DBPath); err == nil { - st, err := store.OpenReadOnly(ctx, cfg.DBPath) - if err != nil { - return err - } - defer st.Close() - status, err = st.Status(ctx) - if err != nil { - return err - } - } else if !errors.Is(err, os.ErrNotExist) { + status, err := a.localArchiveStatus(ctx, cfg) + if err != nil { return err } - status.DBPath = cfg.DBPath - return a.writeOutput("status", controlStatus(config.ResolvePath(a.configPath), cfg, status), false) + return a.writeOutput("status", status, false) } func (a *App) runRemoteStatusWithConfig(ctx context.Context, cfg config.Config) error { @@ -5313,6 +5348,7 @@ Core commands: gh moved to Octopool; prints migration note portable prune prune volatile payloads from a portable store portable export create an immutable derived portable generation + portable refresh safely refresh a configured portable subscriber tui [owner/repo] browse clusters in the terminal UI; repo is inferred when omitted No API server is provided. There is intentionally no serve command. @@ -5528,10 +5564,12 @@ The TUI quietly refreshes from the local store every 15 seconds and leaves the c const portableUsageText = `gitcrawl portable manages local portable-store snapshots. Usage: + gitcrawl portable refresh --expected-remote URL [--store-dir PATH] [--portable-db PATH] [--branch main] [--git PATH] [--timeout 2m] [--min-free-bytes N] [--max-growth-bytes N] [--json] gitcrawl portable prune [--body-chars N] [--no-vacuum] [--include-sync-failures] [--no-publish] [--json] gitcrawl portable export --profile current-state-v1 --output-dir PATH [--repository owner/repo] [--database-name NAME] [--public-path PATH] [--body-chars N] [--max-bytes N] [--compression gzip] [--max-archive-bytes N] [--json] Subcommands: + refresh validate and fast-forward a clean configured subscriber prune prune volatile payloads from the configured portable store export create a validated portable artifact in a new directory @@ -5545,4 +5583,9 @@ a complete database and manifest generation at a previously nonexistent path. With --compression gzip, it commits the gzip archive and manifest without the uncompressed database; --max-archive-bytes applies to that published archive. --repository semantically restricts the disposable snapshot to one owner/repo. +Refresh never regenerates config or invokes repair. It requires a matching +origin, clean checkout and manifest-backed artifact. --git requires an absolute +path (or set GITCRAWL_PORTABLE_GIT). Timeout defaults to 2m; free-space reserve +and measured growth budget each default to 2147483648 bytes. Refusals exit +nonzero; JSON distinguishes updated, no-op, refused and partial results. ` diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 5f3c2806..96c4e8f8 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -3849,7 +3849,7 @@ func TestPortableRuntimeRejectsManifestMismatchBeforeReplacingMirror(t *testing. if err := os.WriteFile(portableDBManifestPath(checkoutDB), data, 0o644); err != nil { t.Fatalf("write bad manifest: %v", err) } - if err := copySQLiteFileAtomicVerified(ctx, checkoutDB, mirrorPath); err == nil || !strings.Contains(err.Error(), "portable manifest mismatch") { + if _, err := copySQLiteFileAtomicVerified(ctx, checkoutDB, mirrorPath); err == nil || !strings.Contains(err.Error(), "portable manifest mismatch") { t.Fatalf("manifest mismatch should fail without copy, err=%v", err) } after, err := fileSHA256(mirrorPath) @@ -3862,7 +3862,7 @@ func TestPortableRuntimeRejectsManifestMismatchBeforeReplacingMirror(t *testing. if err := os.WriteFile(portableDBManifestPath(checkoutDB), []byte("{}\n"), 0o644); err != nil { t.Fatalf("write incomplete manifest: %v", err) } - if err := copySQLiteFileAtomicVerified(ctx, checkoutDB, mirrorPath); err == nil || !strings.Contains(err.Error(), "schema missing") { + if _, err := copySQLiteFileAtomicVerified(ctx, checkoutDB, mirrorPath); err == nil || !strings.Contains(err.Error(), "schema missing") { t.Fatalf("incomplete manifest should fail without copy, err=%v", err) } info, err := os.Stat(checkoutDB) diff --git a/internal/cli/gh_compat_helpers.go b/internal/cli/gh_compat_helpers.go index ff689187..1aa444f6 100644 --- a/internal/cli/gh_compat_helpers.go +++ b/internal/cli/gh_compat_helpers.go @@ -44,15 +44,6 @@ func (a *App) ghCommandCacheDir() (string, error) { return dir, nil } -func tryGHCommandCacheLock(path string) (*os.File, bool) { - lock, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return nil, false - } - _, _ = fmt.Fprintf(lock, "%d\n", os.Getpid()) - return lock, true -} - func writeAtomicFile(path string, data []byte, mode os.FileMode) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err diff --git a/internal/cli/portable_behavior_test.go b/internal/cli/portable_behavior_test.go new file mode 100644 index 00000000..cf315f4a --- /dev/null +++ b/internal/cli/portable_behavior_test.go @@ -0,0 +1,215 @@ +package cli + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/openclaw/crawlkit/control" + "github.com/openclaw/gitcrawl/internal/store" +) + +func (fixture portableRefreshFixture) command(t *testing.T, args ...string) []byte { + t.Helper() + app := New() + var stdout, stderr bytes.Buffer + app.Stdout, app.Stderr = &stdout, &stderr + if err := app.Run(context.Background(), append([]string{"--config", fixture.configPath}, args...)); err != nil { + t.Fatalf("%v: %v; %s", args, err, stderr.String()) + } + return stdout.Bytes() +} + +func portableTestDigest(t *testing.T, path string) [32]byte { + t.Helper() + digest, err := fileSHA256(path) + if err != nil { + t.Fatal(err) + } + return digest +} + +func TestPortableLocalWorkSurvivesRefreshAndReads(t *testing.T) { + for _, mode := range []string{"control", "refresh", "legacy-refresh"} { + t.Run(mode, func(t *testing.T) { + refresh := mode != "control" + fixture := newPortableRefreshFixture(t, true) + fixture.command(t, "threads", "openclaw/openclaw", "--json") + fixture.command(t, "close-thread", "openclaw/openclaw", "--number", "1", "--reason", "keep local decision", "--json") + // A second kind of local write must survive as well, not just the + // close fields. Use the existing writable store contract. + seedPortableThread(t, fixture.mirror, 3, "local thread") + before := portableTestDigest(t, fixture.mirror) + baseline := readPortableStoreRefreshState(portableStoreRefreshStatePath(fixture.mirror)) + if mode == "legacy-refresh" { + baseline.MirrorWritable = false + info, err := os.Stat(fixture.mirror) + if err != nil { + t.Fatal(err) + } + baseline.MirrorHealthSize = info.Size() + baseline.MirrorHealthModTime = info.ModTime().UTC().Format(time.RFC3339Nano) + if err := writePortableStoreRefreshState(portableStoreRefreshStatePath(fixture.mirror), baseline); err != nil { + t.Fatal(err) + } + } + readClosure := func() (string, string) { + t.Helper() + st, err := store.OpenReadOnly(context.Background(), fixture.mirror) + if err != nil { + t.Fatal(err) + } + defer st.Close() + var closedAt, reason sql.NullString + if err := st.DB().QueryRow(`select closed_at_local, close_reason_local from threads where number=1`).Scan(&closedAt, &reason); err != nil { + t.Fatal(err) + } + if !closedAt.Valid || !reason.Valid || closedAt.String == "" || reason.String != "keep local decision" { + t.Fatalf("lost closure: %v %v", closedAt, reason) + } + return closedAt.String, reason.String + } + closedAt, reason := readClosure() + if refresh { + fixture.advance(t, true) + result, err := fixture.refresh(t) + if err != nil || result.Result != "updated" || result.MirrorResult != "preserved-local" { + t.Fatalf("refresh: %+v %v", result, err) + } + if fmt.Sprintf("%x", before) == result.SHA256 { + t.Fatal("fixture did not publish a conflicting generation") + } + } + for range 3 { + if portableTestDigest(t, fixture.mirror) != before { + t.Fatal("runtime bytes changed") + } + var output struct { + Threads []store.Thread `json:"threads"` + } + if err := json.Unmarshal(fixture.command(t, "threads", "openclaw/openclaw", "--json"), &output); err != nil { + t.Fatal(err) + } + threads := output.Threads + if len(threads) != 1 || threads[0].Number != 3 || threads[0].Title != "local thread" { + t.Fatalf("local read changed: %+v", threads) + } + if at, why := readClosure(); at != closedAt || why != reason { + t.Fatal("closure fields changed") + } + if portableTestDigest(t, fixture.mirror) != before { + t.Fatal("ordinary read replaced local runtime") + } + } + state := readPortableStoreRefreshState(portableStoreRefreshStatePath(fixture.mirror)) + if state.MirrorHealthSourceSHA256 != baseline.MirrorHealthSourceSHA256 { + t.Fatal("local mirror was mislabeled as the latest source") + } + var status control.Status + if err := json.Unmarshal(fixture.command(t, "status", "--json"), &status); err != nil { + t.Fatal(err) + } + if refresh && status.State != "stale" { + t.Fatalf("preserved runtime reported fresh: %+v", status) + } + }) + } +} + +func TestPortableRuntimeCorruptionOwnership(t *testing.T) { + for _, writable := range []bool{false, true} { + t.Run(fmt.Sprintf("writable=%t", writable), func(t *testing.T) { + fixture := newPortableRefreshFixture(t, false) + fixture.command(t, "threads", "openclaw/openclaw", "--json") + if writable { + fixture.command(t, "close-thread", "openclaw/openclaw", "--number", "1", "--json") + } + if err := os.WriteFile(fixture.mirror, []byte("broken sqlite"), 0o600); err != nil { + t.Fatal(err) + } + before := portableTestDigest(t, fixture.mirror) + app := New() + app.Stdout, app.Stderr = &bytes.Buffer{}, &bytes.Buffer{} + err := app.Run(context.Background(), []string{"--config", fixture.configPath, "threads", "openclaw/openclaw", "--json"}) + if writable { + if err == nil || portableTestDigest(t, fixture.mirror) != before { + t.Fatalf("corrupt local work was overwritten: %v", err) + } + } else if err != nil || portableTestDigest(t, fixture.mirror) != portableTestDigest(t, filepath.Join(fixture.checkout, fixture.relative)) { + t.Fatalf("disposable replica did not recover: %v", err) + } + }) + } +} + +func TestPortableStatusGzipReadOnly(t *testing.T) { + fixture := newPortableRefreshFixture(t, true) + logical := filepath.Join(fixture.checkout, fixture.relative) + for _, materialized := range []bool{false, true} { + if materialized { + fixture.command(t, "threads", "openclaw/openclaw", "--json") + } + before := portableTestSnapshot(t, filepath.Dir(fixture.configPath)) + var status control.Status + if err := json.Unmarshal(fixture.command(t, "status", "--json"), &status); err != nil { + t.Fatal(err) + } + if countValue(status.Counts, "repositories") != 1 || countValue(status.Counts, "threads") != 1 || status.DatabaseBytes <= 0 { + t.Fatalf("empty populated subscriber status: %+v", status) + } + path := logical + ".gz" + if materialized { + path = fixture.mirror + } + if !sameExistingPath(status.DatabasePath, path) || status.DatabaseBytes != fileSize(path) { + t.Fatalf("wrong inventory: %+v", status) + } + if _, err := os.Stat(logical); !os.IsNotExist(err) { + t.Fatalf("status materialized logical database: %v", err) + } + after := portableTestSnapshot(t, filepath.Dir(fixture.configPath)) + if !bytes.Equal(before, after) { + t.Fatal("status changed checkout, runtime, metadata, or config") + } + } + // Status must report a broken runtime, not silently repair it from source. + if err := os.WriteFile(fixture.mirror, []byte("broken sqlite"), 0o600); err != nil { + t.Fatal(err) + } + before := portableTestSnapshot(t, filepath.Dir(fixture.configPath)) + app := New() + app.Stdout, app.Stderr = &bytes.Buffer{}, &bytes.Buffer{} + if err := app.Run(context.Background(), []string{"--config", fixture.configPath, "status", "--json"}); err == nil { + t.Fatal("corrupt runtime reported successful status") + } + if !bytes.Equal(before, portableTestSnapshot(t, filepath.Dir(fixture.configPath))) { + t.Fatal("status repaired the corrupt runtime") + } +} + +func portableTestSnapshot(t *testing.T, root string) []byte { + t.Helper() + files := map[string][32]byte{} + if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Type().IsRegular() { + files[path] = portableTestDigest(t, path) + } + return nil + }); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(files) + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/internal/cli/portable_capacity.go b/internal/cli/portable_capacity.go new file mode 100644 index 00000000..4fdb54d1 --- /dev/null +++ b/internal/cli/portable_capacity.go @@ -0,0 +1,143 @@ +package cli + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + "time" +) + +const portableDefaultReserve = uint64(2 << 30) +const portableDefaultGrowth = int64(2 << 30) + +type portableCapacity struct { + ReserveBytes uint64 `json:"reserve_bytes"` + GrowthLimit int64 `json:"growth_limit_bytes"` + PeakGrowth int64 `json:"peak_growth_bytes"` + FreeBefore uint64 `json:"minimum_free_before_bytes"` + FreeAfter uint64 `json:"minimum_free_observed_bytes"` +} + +type portableBudget struct { + mu sync.Mutex + roots []string + baseline map[string]int64 + observed portableCapacity +} + +// Measure positive per-path logical-byte growth. Deletions and shrinking old +// files cannot buy more budget. This walks metadata, never historical contents. +func portableFileSizes(ctx context.Context, roots []string) (map[string]int64, error) { + sizes := make(map[string]int64) + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err := ctx.Err(); err != nil { + return err + } + if os.IsNotExist(err) { + return nil // Git may have just renamed a task-owned temporary file. + } + if err != nil { + return err + } + if len(sizes) >= 200000 { + return fmt.Errorf("portable capacity scan exceeds 200000 files") + } + if entry.Type().IsRegular() { + info, err := entry.Info() + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + sizes[path] = info.Size() + } + return nil + }) + if err != nil { + return nil, err + } + } + return sizes, nil +} + +func newPortableBudget(ctx context.Context, roots []string, reserve uint64, growth int64) (*portableBudget, error) { + baseline, err := portableFileSizes(ctx, roots) + if err != nil { + return nil, err + } + budget := &portableBudget{roots: roots, baseline: baseline, observed: portableCapacity{ReserveBytes: reserve, GrowthLimit: growth}} + if err := budget.check(ctx); err != nil { + return budget, err + } + budget.observed.FreeBefore = budget.observed.FreeAfter + if budget.observed.FreeBefore < reserve || budget.observed.FreeBefore-reserve < uint64(growth) { + return budget, fmt.Errorf("insufficient capacity for free-space reserve plus growth budget") + } + return budget, nil +} + +func (budget *portableBudget) check(ctx context.Context) error { + budget.mu.Lock() + defer budget.mu.Unlock() + sizes, err := portableFileSizes(ctx, budget.roots) + if err != nil { + return err + } + var growth int64 + for path, size := range sizes { + if delta := size - budget.baseline[path]; delta > 0 { + if delta > budget.observed.GrowthLimit-growth { + budget.observed.PeakGrowth = max(budget.observed.PeakGrowth, growth+delta) + return fmt.Errorf("portable temporary growth budget exceeded") + } + growth += delta + } + } + budget.observed.PeakGrowth = max(budget.observed.PeakGrowth, growth) + for _, root := range budget.roots { + free, err := portableFreeBytes(root) + if err != nil { + return fmt.Errorf("measure portable free space: %w", err) + } + if budget.observed.FreeAfter == 0 || free < budget.observed.FreeAfter { + budget.observed.FreeAfter = free + } + if free < budget.observed.ReserveBytes { + return fmt.Errorf("portable free-space reserve crossed") + } + } + return nil +} + +func (budget *portableBudget) snapshot() portableCapacity { + budget.mu.Lock() + defer budget.mu.Unlock() + return budget.observed +} + +func (budget *portableBudget) monitor(ctx context.Context) (context.Context, func()) { + ctx, cancel := context.WithCancelCause(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := budget.check(ctx); err != nil { + cancel(err) + return + } + } + } + }() + return ctx, func() { cancel(nil); <-done } +} diff --git a/internal/cli/portable_capacity_test.go b/internal/cli/portable_capacity_test.go new file mode 100644 index 00000000..68735f93 --- /dev/null +++ b/internal/cli/portable_capacity_test.go @@ -0,0 +1,102 @@ +package cli + +import ( + "context" + "errors" + "math" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +func TestPortableCapacityScansUniquePathsAndCancellation(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "nested") + first, second := filepath.Join(root, "first"), filepath.Join(nested, "second") + writePortableSafetyFile(t, first, []byte("abc")) + writePortableSafetyFile(t, second, []byte("12345")) + roots := []string{root, nested, filepath.Join(root, "already-gone")} + sizes, err := portableFileSizes(context.Background(), roots) + if err != nil || !reflect.DeepEqual(sizes, map[string]int64{first: 3, second: 5}) { + t.Fatalf("overlapping roots or vanished paths miscounted: %v, %v", sizes, err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if sizes, err := portableFileSizes(ctx, roots); !errors.Is(err, context.Canceled) || sizes != nil { + t.Fatalf("canceled scan returned usable accounting: %v, %v", sizes, err) + } + if budget, err := newPortableBudget(ctx, roots, 0, 8); !errors.Is(err, context.Canceled) || budget != nil { + t.Fatalf("canceled admission returned a budget: %+v, %v", budget, err) + } +} + +func TestPortableCapacityAdmissionAndMeasurementFailures(t *testing.T) { + root := t.TempDir() + for _, test := range []struct { + name string + roots []string + reserve uint64 + growth int64 + want string + }{ + {"reserve", []string{root}, math.MaxUint64, 1, "free-space reserve crossed"}, + {"reserve-plus-growth", []string{root}, 0, math.MaxInt64, "reserve plus growth budget"}, + {"missing-filesystem", []string{filepath.Join(root, "missing")}, 0, 1, "measure portable free space"}, + } { + t.Run(test.name, func(t *testing.T) { + budget, err := newPortableBudget(context.Background(), test.roots, test.reserve, test.growth) + if err == nil || !strings.Contains(err.Error(), test.want) || budget == nil { + t.Fatalf("expected %q refusal with diagnostics, got %+v, %v", test.want, budget, err) + } + snapshot := budget.snapshot() + if snapshot.ReserveBytes != test.reserve || snapshot.GrowthLimit != test.growth || snapshot.PeakGrowth != 0 { + t.Fatalf("refusal lost requested limits: %+v", snapshot) + } + if test.name == "reserve-plus-growth" && (snapshot.FreeBefore == 0 || snapshot.FreeAfter != snapshot.FreeBefore) { + t.Fatalf("admission omitted measured capacity: %+v", snapshot) + } + }) + } +} + +func TestPortableCapacityPeakAndMonitor(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "runtime.db") + writePortableSafetyFile(t, path, []byte("base")) + budget, err := newPortableBudget(context.Background(), []string{root}, 0, 4) + if err != nil { + t.Fatal(err) + } + writePortableSafetyFile(t, path, []byte("base1234")) + if err := budget.check(context.Background()); err != nil { + t.Fatalf("exact growth limit refused: %v", err) + } + writePortableSafetyFile(t, path, []byte("b")) + if err := budget.check(context.Background()); err != nil { + t.Fatal(err) + } + if got := budget.snapshot().PeakGrowth; got != 4 { + t.Fatalf("shrinking erased peak growth: %d", got) + } + // Five new bytes exceed the budget even though the old file shrank. + writePortableSafetyFile(t, filepath.Join(root, "new"), []byte("12345")) + ctx, stop := budget.monitor(context.Background()) + defer stop() + select { + case <-ctx.Done(): + if cause := context.Cause(ctx); cause == nil || !strings.Contains(cause.Error(), "growth budget exceeded") { + t.Fatalf("monitor lost capacity refusal: %v", cause) + } + case <-time.After(3 * time.Second): + t.Fatal("monitor did not cancel excessive growth") + } + if got := budget.snapshot().PeakGrowth; got != 5 { + t.Fatalf("monitor did not record observed growth: %d", got) + } + if data, err := os.ReadFile(filepath.Join(root, "new")); err != nil || string(data) != "12345" { + t.Fatalf("monitor deleted data to satisfy budget: %q, %v", data, err) + } +} diff --git a/internal/cli/portable_checkout.go b/internal/cli/portable_checkout.go new file mode 100644 index 00000000..fbe4b0f6 --- /dev/null +++ b/internal/cli/portable_checkout.go @@ -0,0 +1,265 @@ +package cli + +import ( + "context" + "crypto/sha256" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" +) + +type portableCheckout struct { + root string + branch string + remote string + head string + tracking string + configHash [32]byte + rootInfo os.FileInfo + gitInfo os.FileInfo +} + +func portableRef(ctx context.Context, root, ref string) (string, error) { + value, err := portableGitOutput(ctx, root, "rev-parse", "--verify", ref+"^{commit}") + if err != nil { + return "", fmt.Errorf("resolve portable commit: %w", err) + } + value = strings.TrimSpace(value) + if len(value) != 40 && len(value) != 64 { + return "", fmt.Errorf("invalid portable commit identity") + } + for _, char := range value { + if !strings.ContainsRune("0123456789abcdef", char) { + return "", fmt.Errorf("invalid portable commit identity") + } + } + return value, nil +} + +func inspectPortableCheckout(ctx context.Context, root, remote, branch string) (portableCheckout, error) { + checkout := portableCheckout{root: root, remote: remote, branch: branch} + var err error + checkout.rootInfo, err = os.Lstat(root) + if err != nil || !checkout.rootInfo.IsDir() { + return checkout, fmt.Errorf("portable store must be an existing directory") + } + checkout.gitInfo, err = os.Lstat(filepath.Join(root, ".git")) + if err != nil || !checkout.gitInfo.IsDir() { + return checkout, fmt.Errorf("portable store requires its own .git directory") + } + if err := checkPortableMetadata(ctx, root); err != nil { + return checkout, err + } + if ok, err := probePortableStoreGitWorktree(ctx, root); err != nil || !ok { + return checkout, fmt.Errorf("portable store identity could not be verified") + } + checkout.configHash, err = portableConfigIdentity(ctx, root) + if err != nil { + return checkout, err + } + actualRemote, err := portableGitOutput(ctx, root, "config", "--get-all", "remote.origin.url") + if err != nil || strings.Count(strings.TrimSpace(actualRemote), "\n") != 0 || !sameGitRemote(strings.TrimSpace(actualRemote), remote) { + return checkout, fmt.Errorf("origin does not match --expected-remote") + } + actualBranch, err := portableGitOutput(ctx, root, "symbolic-ref", "--quiet", "HEAD") + if err != nil || strings.TrimSpace(actualBranch) != "refs/heads/"+branch { + return checkout, fmt.Errorf("portable checkout is not on the expected branch") + } + branchRemote, _ := portableGitOutput(ctx, root, "config", "--get-all", "branch."+branch+".remote") + branchMerge, _ := portableGitOutput(ctx, root, "config", "--get-all", "branch."+branch+".merge") + if strings.TrimSpace(branchRemote) != "origin" || strings.TrimSpace(branchMerge) != "refs/heads/"+branch { + return checkout, fmt.Errorf("portable branch must track its matching origin branch") + } + checkout.head, err = portableRef(ctx, root, "HEAD") + if err != nil { + return checkout, err + } + checkout.tracking, err = portableRef(ctx, root, "refs/remotes/origin/"+branch) + if err != nil { + return checkout, err + } + if err := portableCheckoutClean(ctx, root); err != nil { + return checkout, err + } + return checkout, nil +} + +func portableCheckoutClean(ctx context.Context, root string) error { + // Inspect index modes before status can descend into a submodule or apply + // worktree attributes. No transport or checkout is needed for this check. + flags, err := portableGitOutput(ctx, root, "ls-files", "--stage", "-v", "-z") + if err != nil { + return err + } + for _, entry := range strings.Split(flags, "\x00") { + if entry == "" { + continue + } + metadata, path, ok := strings.Cut(entry, "\t") + fields := strings.Fields(metadata) + if !ok || len(fields) != 4 || fields[0] != "H" || fields[3] != "0" || (fields[1] != "100644" && fields[1] != "100755") { + return fmt.Errorf("portable index has hidden, unresolved, linked or submodule entries") + } + if filepath.Base(path) == ".gitattributes" || filepath.Base(path) == ".gitmodules" { + return fmt.Errorf("strict portable refresh does not accept attributes or submodules") + } + } + output, err := portableGitOutput(ctx, root, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--ignored=matching", "--ignore-submodules=all") + if err != nil { + return fmt.Errorf("inspect portable worktree: %w", err) + } + if output != "" { + return fmt.Errorf("portable store is not clean (index, worktree, untracked or ignored files)") + } + return nil +} + +func portableConfigIdentity(ctx context.Context, root string) ([32]byte, error) { + var digest [32]byte + output, err := portableGitOutput(ctx, root, "config", "--null", "--list", "--show-scope") + if err != nil { + return digest, fmt.Errorf("read portable Git configuration: %w", err) + } + parts := strings.Split(strings.TrimSuffix(output, "\x00"), "\x00") + if len(parts)%2 != 0 { + return digest, fmt.Errorf("unsupported Git configuration output") + } + for index := 0; index < len(parts); index += 2 { + scope := parts[index] + switch scope { + case "command": + continue + case "system", "global", "local", "worktree": + default: + return digest, fmt.Errorf("unsupported Git configuration scope") + } + key, value, _ := strings.Cut(parts[index+1], "\n") + key = strings.ToLower(key) + // The runner overrides these single-valued settings on every call. + if key == "core.hookspath" || key == "core.fsmonitor" || key == "core.attributesfile" || key == "core.sshcommand" { + continue + } + if strings.HasPrefix(key, "filter.") { + return digest, fmt.Errorf("unsupported portable Git configuration (%s scope: filters)", scope) + } + if key == "core.gitproxy" || key == "core.worktree" || key == "core.sparsecheckout" || key == "core.sparsecheckoutcone" || + strings.HasPrefix(key, "submodule.") || strings.HasPrefix(key, "extensions.") || + strings.HasPrefix(key, "include") || strings.HasPrefix(key, "url.") || key == "core.alternaterefscommand" || + (strings.HasPrefix(key, "remote.") && !strings.HasPrefix(key, "remote.origin.")) { + // Never print key names: URL subsections can contain credentials. + return digest, fmt.Errorf("unsupported portable Git configuration (%s scope: redirection, submodules or extensions)", scope) + } + if strings.HasPrefix(key, "remote.origin.") && key != "remote.origin.url" && key != "remote.origin.fetch" && key != "remote.origin.tagopt" { + return digest, fmt.Errorf("unsupported portable Git configuration (%s scope: origin options)", scope) + } + if key == "core.bare" && value != "false" { + return digest, fmt.Errorf("portable checkout cannot be bare") + } + } + return sha256.Sum256([]byte(output)), nil +} + +func checkPortableMetadata(ctx context.Context, root string) error { + gitDir := filepath.Join(root, ".git") + entries := 0 + return filepath.WalkDir(gitDir, func(path string, entry fs.DirEntry, err error) error { + if err := ctx.Err(); err != nil { + return err + } + entries++ + if entries > 200000 { + return fmt.Errorf("portable metadata scan exceeds 200000 entries") + } + if err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 || (!entry.IsDir() && !entry.Type().IsRegular()) { + return fmt.Errorf("portable Git metadata contains a link or special file") + } + rel, err := filepath.Rel(gitDir, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + name := entry.Name() + if strings.HasSuffix(name, ".lock") || strings.HasSuffix(name, ".pid") || strings.HasPrefix(name, "tmp_") || + rel == "MERGE_HEAD" || rel == "CHERRY_PICK_HEAD" || rel == "REVERT_HEAD" || rel == "BISECT_LOG" || + rel == "rebase-apply" || rel == "rebase-merge" || rel == "sequencer" || rel == "worktrees" || rel == "modules" || + rel == "commondir" || rel == "info/grafts" || rel == "info/sparse-checkout" || rel == "info/attributes" || + rel == "objects/info/alternates" || rel == "objects/info/http-alternates" || strings.HasPrefix(rel, "refs/replace/") { + return fmt.Errorf("portable Git metadata contains competing or unsupported state") + } + if strings.HasPrefix(rel, "hooks/") && !entry.IsDir() && !strings.HasSuffix(name, ".sample") { + return fmt.Errorf("portable checkout contains a hook") + } + if strings.HasPrefix(rel, "objects/pack/") && (strings.HasSuffix(name, ".pack") || strings.HasSuffix(name, ".idx")) { + paired := strings.TrimSuffix(path, filepath.Ext(path)) + ".idx" + if strings.HasSuffix(name, ".idx") { + paired = strings.TrimSuffix(path, ".idx") + ".pack" + } + if _, err := os.Lstat(paired); err != nil { + return fmt.Errorf("portable store contains an unresolved orphan pack") + } + } + return nil + }) +} + +func (checkout portableCheckout) recheck(ctx context.Context, head, tracking string) error { + owner, ok := ctx.Value(portableOwnerKey{}).(*portableOwner) + if !ok { + return fmt.Errorf("portable ownership missing") + } + if err := owner.check(); err != nil { + return err + } + current, err := inspectPortableCheckout(ctx, checkout.root, checkout.remote, checkout.branch) + if err != nil { + return err + } + if !os.SameFile(checkout.rootInfo, current.rootInfo) || !os.SameFile(checkout.gitInfo, current.gitInfo) || + current.configHash != checkout.configHash || current.head != head || current.tracking != tracking { + return fmt.Errorf("portable checkout identity or refs changed during refresh") + } + return nil +} + +type portableTreeEntry struct { + oid string + size int64 +} + +func portableCommitTree(ctx context.Context, root, commit string, limit int64) (map[string]portableTreeEntry, error) { + output, err := portableGitOutput(ctx, root, "ls-tree", "-r", "-l", "-z", "--full-tree", commit) + if err != nil { + return nil, err + } + entries := make(map[string]portableTreeEntry) + var total int64 + for _, record := range strings.Split(output, "\x00") { + if record == "" { + continue + } + metadata, path, ok := strings.Cut(record, "\t") + fields := strings.Fields(metadata) + if !ok || len(fields) != 4 || (fields[0] != "100644" && fields[0] != "100755") || fields[1] != "blob" { + return nil, fmt.Errorf("portable tree contains a symlink, submodule or unsupported entry") + } + if err := validatePortableRelativePath(path); err != nil { + return nil, err + } + if filepath.Base(path) == ".gitattributes" || filepath.Base(path) == ".gitmodules" { + return nil, fmt.Errorf("strict portable refresh does not accept attributes or submodules") + } + size, err := strconv.ParseInt(fields[3], 10, 64) + if err != nil || size < 0 || size > limit-total { + return nil, fmt.Errorf("portable checkout tree exceeds growth budget") + } + total += size + entries[path] = portableTreeEntry{oid: fields[2], size: size} + } + return entries, nil +} diff --git a/internal/cli/portable_config_protocol_unix_test.go b/internal/cli/portable_config_protocol_unix_test.go new file mode 100644 index 00000000..97f660f3 --- /dev/null +++ b/internal/cli/portable_config_protocol_unix_test.go @@ -0,0 +1,96 @@ +//go:build !windows + +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "os" + "path/filepath" + "testing" +) + +func TestPortableConfigProtocolFailsClosed(t *testing.T) { + root := t.TempDir() + output := filepath.Join(root, "config-output") + wrapper := filepath.Join(root, "git") + writePortableSafetyFile(t, wrapper, []byte("#!/bin/sh\ncat \"$GITCRAWL_TEST_CONFIG_OUTPUT\"\n")) + if err := os.Chmod(wrapper, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_TEST_CONFIG_OUTPUT", output) + ctx, err := portableGitContext(context.Background(), wrapper) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct{ name, output, want string }{ + {"unpaired", "local\x00", "unsupported Git configuration output"}, + {"unknown-scope", "future\x00core.bare\nfalse\x00", "unsupported Git configuration scope"}, + {"bare", "local\x00core.bare\ntrue\x00", "portable checkout cannot be bare"}, + {"filter", "worktree\x00filter.synthetic.clean\nprivate-value\x00", "unsupported portable Git configuration (worktree scope: filters)"}, + {"url-redirection", "global\x00url.https://example.invalid/private-subsection.insteadof\nprivate-value\x00", "unsupported portable Git configuration (global scope: redirection, submodules or extensions)"}, + {"remote-command", "local\x00remote.origin.uploadpack\nprivate-value\x00", "unsupported portable Git configuration (local scope: origin options)"}, + } { + t.Run(test.name, func(t *testing.T) { + writePortableSafetyFile(t, output, []byte(test.output)) + digest, err := portableConfigIdentity(ctx, root) + if err == nil || err.Error() != test.want || digest != ([32]byte{}) { + t.Fatalf("unsafe config received identity or leaked data: digest=%x, err=%v", digest, err) + } + }) + } + // The trusted command scope may neutralize an unsafe single-valued setting; + // its exact bytes still participate in the identity used at recheck. + valid := "local\x00core.bare\nfalse\x00command\x00core.hookspath\n/dev/null\x00" + writePortableSafetyFile(t, output, []byte(valid)) + first, err := portableConfigIdentity(ctx, root) + if err != nil || first != sha256.Sum256([]byte(valid)) { + t.Fatalf("valid config identity: %x, %v", first, err) + } + changed := valid + "local\x00user.name\nchanged\x00" + writePortableSafetyFile(t, output, []byte(changed)) + second, err := portableConfigIdentity(ctx, root) + if err != nil || second == first || second != sha256.Sum256([]byte(changed)) { + t.Fatalf("changed safe config did not change identity: %x, %v", second, err) + } +} + +func TestPortableMetadataRejectsLinksAndOrphanPairs(t *testing.T) { + for _, name := range []string{"paired", "orphan-pack", "orphan-index", "linked-metadata"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + pack := filepath.Join(root, ".git", "objects", "pack", "pack-fixture") + writePortableSafetyFile(t, pack+".pack", []byte("historical pack")) + writePortableSafetyFile(t, pack+".idx", []byte("historical index")) + want := "" + switch name { + case "orphan-pack": + if err := os.Remove(pack + ".idx"); err != nil { + t.Fatal(err) + } + want = "portable store contains an unresolved orphan pack" + case "orphan-index": + if err := os.Remove(pack + ".pack"); err != nil { + t.Fatal(err) + } + want = "portable store contains an unresolved orphan pack" + case "linked-metadata": + target := filepath.Join(t.TempDir(), "unrelated") + writePortableSafetyFile(t, target, []byte("do not follow")) + if err := os.Symlink(target, filepath.Join(root, ".git", "config")); err != nil { + t.Fatal(err) + } + want = "portable Git metadata contains a link or special file" + } + before := portableTestSnapshot(t, root) + err := checkPortableMetadata(context.Background(), root) + if want == "" && err != nil || want != "" && (err == nil || err.Error() != want) { + t.Fatalf("metadata admission: want %q, got %v", want, err) + } + if !bytes.Equal(before, portableTestSnapshot(t, root)) { + t.Fatal("metadata inspection removed historical objects") + } + }) + } +} diff --git a/internal/cli/portable_config_test.go b/internal/cli/portable_config_test.go new file mode 100644 index 00000000..52e30219 --- /dev/null +++ b/internal/cli/portable_config_test.go @@ -0,0 +1,76 @@ +package cli + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPortableGitEnvironmentIsolation(t *testing.T) { + for _, key := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0", "GIT_CONFIG_VALUE_0", "GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM", "GIT_ASKPASS", "SSH_ASKPASS", "GIT_SSH_COMMAND"} { + t.Setenv(key, "untrusted-input") + } + readEnv := func() map[string]string { + result := map[string]string{} + for _, entry := range portableGitEnv() { + key, value, _ := strings.Cut(entry, "=") + result[key] = value + if value == "untrusted-input" { + t.Fatalf("retained unsafe input: %s", key) + } + } + return result + } + readEnv() + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) + t.Setenv("GIT_CONFIG_NOSYSTEM", "true") + env := readEnv() + for key, value := range map[string]string{"GIT_CONFIG_GLOBAL": os.DevNull, "GIT_CONFIG_SYSTEM": os.DevNull, "GIT_CONFIG_NOSYSTEM": "true", "GIT_TERMINAL_PROMPT": "0", "GIT_ATTR_NOSYSTEM": "1", "GCM_INTERACTIVE": "never", "GIT_NO_REPLACE_OBJECTS": "1"} { + if env[key] != value { + t.Fatalf("lost isolation control %s", key) + } + } +} + +func TestPortableRefreshRejectsArgumentsBeforeConfigOrGit(t *testing.T) { + root := t.TempDir() + configPath := filepath.Join(root, "config.toml") + writePortableSafetyFile(t, configPath, []byte("deliberately invalid config: must not be read")) + for _, test := range []struct { + name string + args []string + want string + }{ + {"unknown-flag", []string{"--unknown-flag"}, "flag provided but not defined"}, + {"bad-duration", []string{"--timeout", "later"}, "invalid value"}, + {"positional", []string{"unexpected"}, "no positional arguments"}, + {"zero-timeout", []string{"--timeout", "0"}, "positive timeout and byte limits"}, + {"zero-reserve", []string{"--min-free-bytes", "0"}, "positive timeout and byte limits"}, + {"negative-growth", []string{"--max-growth-bytes=-1"}, "positive timeout and byte limits"}, + {"missing-remote", nil, "--expected-remote is required"}, + {"remote-userinfo", []string{"--expected-remote", "https://git@example.invalid/store.git"}, "--expected-remote is required"}, + {"remote-password-field", []string{"--expected-remote", "ssh://git:@example.invalid/store.git"}, "--expected-remote is required"}, + {"unsafe-path", []string{"--expected-remote", "https://example.invalid/store.git", "--portable-db", "../archive.db"}, "clean relative slash path"}, + {"relative-git", []string{"--expected-remote", "https://example.invalid/store.git", "--git", "relative-git"}, "absolute path"}, + {"unsafe-branch", []string{"--expected-remote", "https://example.invalid/store.git", "--git", os.Args[0], "--branch=-option"}, "invalid --branch"}, + } { + t.Run(test.name, func(t *testing.T) { + app := New() + app.configPath = configPath + var stdout, stderr bytes.Buffer + app.Stdout, app.Stderr = &stdout, &stderr + before := portableTestSnapshot(t, root) + err := app.runPortableRefresh(context.Background(), test.args) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("expected argument refusal %q, got %v", test.want, err) + } + if stdout.Len() != 0 || stderr.Len() != 0 || !bytes.Equal(before, portableTestSnapshot(t, root)) { + t.Fatal("invalid arguments reached refresh or changed config/runtime") + } + }) + } +} diff --git a/internal/cli/portable_git.go b/internal/cli/portable_git.go new file mode 100644 index 00000000..03e8e1cc --- /dev/null +++ b/internal/cli/portable_git.go @@ -0,0 +1,163 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" +) + +const portableGitOutputLimit = 8 << 20 +const portableGitGrace = 750 * time.Millisecond + +func portableGitArgs(args []string) []string { + controls := []string{ + "-c", "maintenance.auto=false", "-c", "gc.auto=0", + "-c", "core.hooksPath=" + os.DevNull, "-c", "core.fsmonitor=false", + "-c", "submodule.recurse=false", "-c", "fetch.recurseSubmodules=false", + "-c", "fetch.writeCommitGraph=false", "-c", "gc.writeCommitGraph=false", + "-c", "merge.autostash=false", "-c", "core.askPass=", + "-c", "core.autocrlf=false", "-c", "core.eol=lf", + "-c", "core.protectHFS=true", "-c", "core.protectNTFS=true", + "-c", "core.attributesFile=" + os.DevNull, + "-c", "protocol.ext.allow=never", + } + return append(controls, args...) +} + +func portableGitEnv() []string { + var env []string + for _, entry := range os.Environ() { + name, value, _ := strings.Cut(entry, "=") + // Preserve only config selectors that remove an entire scope. Other + // paths and GIT_CONFIG_* injection still cannot select Git's inputs. + if (name == "GIT_CONFIG_GLOBAL" || name == "GIT_CONFIG_SYSTEM") && value == os.DevNull || + name == "GIT_CONFIG_NOSYSTEM" && (value == "1" || strings.EqualFold(value, "true")) { + env = append(env, entry) + continue + } + if strings.HasPrefix(strings.ToUpper(name), "GIT_") || name == "SSH_ASKPASS" { + continue + } + env = append(env, entry) + } + return append(env, "GIT_TERMINAL_PROMPT=0", "GIT_OPTIONAL_LOCKS=0", "GIT_NO_REPLACE_OBJECTS=1", "GIT_ATTR_NOSYSTEM=1", + "GIT_LFS_SKIP_SMUDGE=1", "GCM_INTERACTIVE=never", "GIT_SSH_COMMAND=ssh -o BatchMode=yes -o ConnectTimeout=10") +} + +type portableLimitedWriter struct { + writer io.Writer + left int64 +} + +func (writer *portableLimitedWriter) Write(data []byte) (int, error) { + if int64(len(data)) > writer.left { + return 0, fmt.Errorf("portable Git output exceeds limit") + } + written, err := writer.writer.Write(data) + writer.left -= int64(written) + return written, err +} + +// Git diagnostics are deliberately not returned: transport errors can contain +// credential-bearing URLs, helper output, or unbounded remote messages. +func runPortableGit(ctx context.Context, workdir string, output io.Writer, args ...string) error { + if _, bounded := ctx.Deadline(); !bounded { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, portableOperationTimeout) + defer cancel() + } + if err := ctx.Err(); err != nil { + return err + } + executable, ok := ctx.Value(portableGitKey{}).(portableGitExecutable) + if !ok { + return fmt.Errorf("portable Git executable was not resolved") + } + info, err := os.Stat(executable.path) + if err != nil || !os.SameFile(info, executable.info) || info.Size() != executable.info.Size() || !info.ModTime().Equal(executable.info.ModTime()) { + return fmt.Errorf("portable Git executable changed during operation") + } + cmd := exec.Command(executable.path, portableGitArgs(args)...) + cmd.Dir = workdir + cmd.Env = portableGitEnv() + cmd.Stdout = output + var diagnostic portableGitDiagnostic + cmd.Stderr = &diagnostic + cmd.WaitDelay = time.Second + if err := configurePortableProcess(cmd); err != nil { + return err + } + defer cleanupCommandGroup(cmd) + if err := cmd.Start(); err != nil { + return fmt.Errorf("start portable Git: %w", err) + } + defer killPortableProcess(cmd) + if err := attachPortableProcess(cmd); err != nil { + killPortableProcess(cmd) + _ = cmd.Wait() + return fmt.Errorf("contain portable Git: %w", err) + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + if err != nil { + // Return fixed classifications, never raw remote/helper diagnostics. + // Keep the legacy recovery classifiers intact. + message := diagnostic.String() + lower := strings.ToLower(message) + if strings.Contains(message, "index.lock") && strings.Contains(lower, "file exists") { + return fmt.Errorf("index.lock file exists: %w", err) + } + if strings.Contains(message, "Your local changes") || strings.Contains(message, "would be overwritten by merge") { + return fmt.Errorf("Your local changes would be overwritten by merge: %w", err) + } + switch { + case strings.Contains(lower, "does not appear to be a git repository") || strings.Contains(lower, "repository") && strings.Contains(lower, "does not exist") || strings.Contains(lower, "repository not found"): + return fmt.Errorf("remote repository unavailable; verify the remote path and read access: %w", err) + case strings.Contains(lower, "authentication failed") || strings.Contains(lower, "permission denied (publickey") || strings.Contains(lower, "could not read username"): + return fmt.Errorf("Git authentication failed; check the credential helper or SSH identity and repository access: %w", err) + case strings.Contains(lower, "no space left on device"): + return fmt.Errorf("insufficient disk space for Git; restore free-space headroom before retrying: %w", err) + case strings.Contains(lower, "could not resolve host") || strings.Contains(lower, "failed to connect") || strings.Contains(lower, "connection timed out") || strings.Contains(lower, "connection refused"): + return fmt.Errorf("Git connection failed; check network connectivity and remote availability: %w", err) + } + return fmt.Errorf("Git command failed; verify Git version, repository state and remote access: %w", err) + } + return err + case <-ctx.Done(): + terminatePortableProcess(cmd) + timer := time.NewTimer(portableGitGrace) + defer timer.Stop() + select { + case <-done: + <-timer.C // Children may still be running their graceful cleanup. + killPortableProcess(cmd) + case <-timer.C: + killPortableProcess(cmd) + <-done + } + return context.Cause(ctx) + } +} + +type portableGitDiagnostic struct{ bytes.Buffer } + +func (diagnostic *portableGitDiagnostic) Write(data []byte) (int, error) { + size := len(data) + if remaining := 8192 - diagnostic.Len(); remaining > 0 { + _, _ = diagnostic.Buffer.Write(data[:min(size, remaining)]) + } + return size, nil +} + +func portableGitOutput(ctx context.Context, root string, args ...string) (string, error) { + var output bytes.Buffer + err := runPortableGit(ctx, root, &portableLimitedWriter{writer: &output, left: portableGitOutputLimit}, args...) + return output.String(), err +} diff --git a/internal/cli/portable_io.go b/internal/cli/portable_io.go new file mode 100644 index 00000000..c0a86d0d --- /dev/null +++ b/internal/cli/portable_io.go @@ -0,0 +1,37 @@ +package cli + +import ( + "context" + "crypto/sha256" + "io" + "os" +) + +type portableValidationDirKey struct{} + +type portableContextReader struct { + ctx context.Context + reader io.Reader +} + +func (reader portableContextReader) Read(data []byte) (int, error) { + if err := reader.ctx.Err(); err != nil { + return 0, context.Cause(reader.ctx) + } + return reader.reader.Read(data) +} + +func portableFileSHA256(ctx context.Context, path string) ([32]byte, error) { + var digest [32]byte + file, err := os.Open(path) + if err != nil { + return digest, err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, portableContextReader{ctx: ctx, reader: file}); err != nil { + return digest, err + } + copy(digest[:], hash.Sum(nil)) + return digest, nil +} diff --git a/internal/cli/portable_mirror.go b/internal/cli/portable_mirror.go new file mode 100644 index 00000000..3104c623 --- /dev/null +++ b/internal/cli/portable_mirror.go @@ -0,0 +1,158 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func openPortableMirrorReadOnly(ctx context.Context, path, source string) (*store.Store, error) { + mirror, err := inspectPortableMirror(ctx, path, source) + if err != nil { + return nil, err + } + if err := mirror.recheck(ctx); err != nil { + return nil, err + } + if mirror.exists && !mirror.preserve { + // A normal SQLite read of a WAL-mode replica creates sidecars that + // would falsely claim local ownership on the next refresh. + return store.OpenReadOnlyImmutable(ctx, path) + } + return store.OpenReadOnly(ctx, path) +} + +type portableMirror struct { + path string + exists bool + info os.FileInfo + parent os.FileInfo + state os.FileInfo + stateSHA [32]byte + digest [32]byte + preserve bool + sidecars map[string]os.FileInfo +} + +func inspectPortableMirror(ctx context.Context, path, source string) (portableMirror, error) { + mirror := portableMirror{path: path, sidecars: make(map[string]os.FileInfo)} + var err error + mirror.parent, err = os.Stat(filepath.Dir(path)) + if err != nil { + return mirror, err + } + statePath := portableStoreRefreshStatePath(path) + if _, err := os.Lstat(statePath + ".lock"); !errors.Is(err, os.ErrNotExist) { + return mirror, fmt.Errorf("legacy runtime refresh lock exists or cannot be inspected") + } + var state portableStoreRefreshState + if info, err := os.Lstat(statePath); err == nil { + if !info.Mode().IsRegular() || info.Size() > 1<<20 { + return mirror, fmt.Errorf("invalid runtime refresh metadata") + } + data, err := os.ReadFile(statePath) + if err != nil { + return mirror, err + } + if err := json.Unmarshal(data, &state); err != nil { + return mirror, fmt.Errorf("invalid runtime refresh metadata") + } + mirror.state = info + mirror.stateSHA, err = portableFileSHA256(ctx, statePath) + if err != nil { + return mirror, err + } + } else if !errors.Is(err, os.ErrNotExist) { + return mirror, err + } + info, err := os.Lstat(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return mirror, err + } + if err == nil { + if !info.Mode().IsRegular() { + return mirror, fmt.Errorf("runtime mirror is not a regular file") + } + mirror.exists, mirror.info = true, info + mirror.digest, err = portableFileSHA256(ctx, path) + if err != nil { + return mirror, err + } + sourceSHA := state.MirrorHealthSourceSHA256 + if sourceSHA == "" { + manifest, _, err := readPortableDBManifest(portableDBManifestPath(source)) + if err != nil { + return mirror, err + } + sourceSHA = manifest.SHA256 + } + mirror.preserve = state.MirrorWritable || !strings.EqualFold(fmt.Sprintf("%x", mirror.digest), sourceSHA) + } + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + info, err := os.Lstat(path + suffix) + if err == nil { + mirror.sidecars[suffix] = info + mirror.preserve = true + } else if !errors.Is(err, os.ErrNotExist) { + return mirror, err + } + } + return mirror, nil +} + +func (mirror portableMirror) recheck(ctx context.Context) error { + parent, err := os.Stat(filepath.Dir(mirror.path)) + if err != nil || !os.SameFile(parent, mirror.parent) { + return fmt.Errorf("runtime mirror directory changed during refresh") + } + statePath := portableStoreRefreshStatePath(mirror.path) + if _, err := os.Lstat(statePath + ".lock"); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("legacy runtime refresh lock appeared or cannot be inspected") + } + state, stateErr := os.Lstat(statePath) + if mirror.state == nil { + if !errors.Is(stateErr, os.ErrNotExist) { + return fmt.Errorf("runtime refresh metadata appeared during refresh") + } + } else { + if stateErr != nil || !os.SameFile(state, mirror.state) { + return fmt.Errorf("runtime refresh metadata changed during refresh") + } + digest, err := portableFileSHA256(ctx, statePath) + if err != nil || digest != mirror.stateSHA { + return fmt.Errorf("runtime refresh metadata changed during refresh") + } + } + info, err := os.Lstat(mirror.path) + if !mirror.exists { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("runtime mirror appeared during refresh") + } + } else { + if err != nil || !os.SameFile(info, mirror.info) || info.Size() != mirror.info.Size() || !info.ModTime().Equal(mirror.info.ModTime()) { + return fmt.Errorf("runtime mirror changed during refresh") + } + digest, err := portableFileSHA256(ctx, mirror.path) + if err != nil || digest != mirror.digest { + return fmt.Errorf("runtime mirror changed during refresh") + } + } + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + info, err := os.Lstat(mirror.path + suffix) + old, existed := mirror.sidecars[suffix] + if existed { + if err != nil || !os.SameFile(info, old) || !info.ModTime().Equal(old.ModTime()) || info.Size() != old.Size() { + return fmt.Errorf("runtime SQLite sidecar changed during refresh") + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("runtime SQLite sidecar appeared during refresh") + } + } + return nil +} diff --git a/internal/cli/portable_mirror_test.go b/internal/cli/portable_mirror_test.go new file mode 100644 index 00000000..06839a4c --- /dev/null +++ b/internal/cli/portable_mirror_test.go @@ -0,0 +1,203 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func writePortableSafetyFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +func writePortableSafetyState(t *testing.T, path string, state portableStoreRefreshState) { + t.Helper() + data, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + writePortableSafetyFile(t, portableStoreRefreshStatePath(path), data) +} + +func TestPortableMirrorPreservationIdentity(t *testing.T) { + for _, name := range []string{"missing", "replica", "legacy-manifest", "unknown-source", "local-bytes", "writable", "wal", "shm", "journal"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + source, mirror := filepath.Join(root, "source.db"), filepath.Join(root, "runtime.db") + writePortableSafetyFile(t, source, []byte("published generation")) + portableTestManifest(t, source, "source.db", false) + manifest, _, err := readPortableDBManifest(portableDBManifestPath(source)) + if err != nil { + t.Fatal(err) + } + if name != "missing" { + writePortableSafetyFile(t, mirror, []byte("published generation")) + } + state := portableStoreRefreshState{MirrorHealthSourceSHA256: strings.ToUpper(manifest.SHA256)} + preserve := false + switch name { + case "legacy-manifest": + state.MirrorHealthSourceSHA256 = "" + case "unknown-source": + state.MirrorHealthSourceSHA256 = "" + if err := os.Remove(portableDBManifestPath(source)); err != nil { + t.Fatal(err) + } + preserve = true + case "local-bytes": + writePortableSafetyFile(t, mirror, []byte("local maintainer work")) + preserve = true + case "writable": + state.MirrorWritable, preserve = true, true + case "wal", "shm", "journal": + writePortableSafetyFile(t, mirror+"-"+name, []byte("pending local write")) + preserve = true + } + writePortableSafetyState(t, mirror, state) + before := portableTestSnapshot(t, root) + inspected, err := inspectPortableMirror(context.Background(), mirror, source) + if err != nil || inspected.exists != (name != "missing") || inspected.preserve != preserve { + t.Fatalf("mirror ownership: exists=%t preserve=%t err=%v", inspected.exists, inspected.preserve, err) + } + if err := inspected.recheck(context.Background()); err != nil { + t.Fatalf("unchanged mirror rejected: %v", err) + } + if !bytes.Equal(before, portableTestSnapshot(t, root)) { + t.Fatal("inspection changed runtime, source, sidecars or ownership metadata") + } + }) + } +} + +func TestPortableMirrorRejectsInvalidMetadata(t *testing.T) { + for _, name := range []string{"malformed", "oversized", "directory", "runtime-directory", "bad-source-manifest"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + path, source := filepath.Join(root, "runtime.db"), filepath.Join(root, "source.db") + statePath := portableStoreRefreshStatePath(path) + want := "invalid runtime refresh metadata" + switch name { + case "malformed": + writePortableSafetyFile(t, statePath, []byte("{broken")) + case "oversized": + writePortableSafetyFile(t, statePath, bytes.Repeat([]byte(" "), (1<<20)+1)) + case "directory": + if err := os.Mkdir(statePath, 0o700); err != nil { + t.Fatal(err) + } + case "runtime-directory": + if err := os.Mkdir(path, 0o700); err != nil { + t.Fatal(err) + } + want = "runtime mirror is not a regular file" + case "bad-source-manifest": + writePortableSafetyFile(t, path, []byte("local bytes")) + writePortableSafetyFile(t, portableDBManifestPath(source), []byte("{broken")) + want = "manifest" + } + before := portableTestSnapshot(t, root) + if _, err := inspectPortableMirror(context.Background(), path, source); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected %q refusal, got %v", want, err) + } + if !bytes.Equal(before, portableTestSnapshot(t, root)) { + t.Fatal("invalid runtime state was repaired or deleted during inspection") + } + }) + } +} + +func TestPortableMirrorRecheckDetectsConcurrentChanges(t *testing.T) { + for _, name := range []string{"parent", "metadata-created", "metadata-replaced", "metadata-content", "runtime-created", "runtime-removed", "runtime-replaced", "runtime-content", "sidecar-created", "sidecar-removed", "sidecar-grown"} { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "runtime", "archive.db") + writePortableSafetyFile(t, filepath.Join(filepath.Dir(path), "sentinel"), []byte("keep")) + if name != "runtime-created" { + writePortableSafetyFile(t, path, []byte("old")) + } + statePath := portableStoreRefreshStatePath(path) + if name != "metadata-created" { + writePortableSafetyState(t, path, portableStoreRefreshState{MirrorWritable: true, LastSuccess: "old"}) + } + if name == "sidecar-removed" || name == "sidecar-grown" { + writePortableSafetyFile(t, path+"-wal", []byte("old")) + } + mirror, err := inspectPortableMirror(context.Background(), path, filepath.Join(root, "source.db")) + if err != nil { + t.Fatal(err) + } + if err := mirror.recheck(context.Background()); err != nil { + t.Fatalf("baseline rejected: %v", err) + } + want := "runtime mirror changed" + switch name { + case "parent": + if err := os.Rename(filepath.Dir(path), filepath.Join(root, "previous-runtime")); err != nil { + t.Fatal(err) + } + writePortableSafetyFile(t, path, []byte("new")) + want = "runtime mirror directory changed" + case "metadata-created", "metadata-replaced": + if name == "metadata-replaced" { + if err := os.Rename(statePath, statePath+".previous"); err != nil { + t.Fatal(err) + } + } + writePortableSafetyState(t, path, portableStoreRefreshState{MirrorWritable: true}) + want = "runtime refresh metadata" + case "metadata-content": + writePortableSafetyState(t, path, portableStoreRefreshState{MirrorWritable: true, LastSuccess: "new"}) + if err := os.Chtimes(statePath, mirror.state.ModTime(), mirror.state.ModTime()); err != nil { + t.Fatal(err) + } + want = "runtime refresh metadata changed" + case "runtime-created": + writePortableSafetyFile(t, path, []byte("new")) + want = "runtime mirror appeared" + case "runtime-removed": + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + case "runtime-replaced": + if err := os.Rename(path, path+".previous"); err != nil { + t.Fatal(err) + } + writePortableSafetyFile(t, path, []byte("old")) + case "runtime-content": + // Same inode, size and mtime: the digest must catch this write. + writePortableSafetyFile(t, path, []byte("new")) + if err := os.Chtimes(path, mirror.info.ModTime(), mirror.info.ModTime()); err != nil { + t.Fatal(err) + } + case "sidecar-created": + writePortableSafetyFile(t, path+"-journal", []byte("pending transaction")) + want = "runtime SQLite sidecar appeared" + case "sidecar-removed": + if err := os.Remove(path + "-wal"); err != nil { + t.Fatal(err) + } + want = "runtime SQLite sidecar changed" + case "sidecar-grown": + writePortableSafetyFile(t, path+"-wal", []byte("new transaction")) + want = "runtime SQLite sidecar changed" + } + before := portableTestSnapshot(t, root) + if err := mirror.recheck(context.Background()); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected %q refusal, got %v", want, err) + } + if !bytes.Equal(before, portableTestSnapshot(t, root)) { + t.Fatal("recheck modified concurrent writer's state") + } + }) + } +} diff --git a/internal/cli/portable_owner.go b/internal/cli/portable_owner.go new file mode 100644 index 00000000..848b0e05 --- /dev/null +++ b/internal/cli/portable_owner.go @@ -0,0 +1,220 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +const portableOperationTimeout = 2 * time.Minute + +type portableOwnerKey struct{} +type portableGitKey struct{} +type portableCommandKey struct{} + +type portableCommandSession struct { + mu sync.Mutex + owner *portableOwner + retain bool +} + +func (session *portableCommandSession) close() { + session.mu.Lock() + defer session.mu.Unlock() + if session.owner != nil { + _ = session.owner.file.Close() + } +} + +type portableGitExecutable struct { + path string + info os.FileInfo +} + +type portableOwner struct { + root string + file *os.File + git portableGitExecutable +} + +// Resolve existing ancestors too: a first clone and later operations must use +// the same lock, including when the store is reached through a directory link. +func canonicalPortablePath(path string) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(abs) + if err == nil { + return resolved, nil + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + if info, statErr := os.Lstat(abs); statErr == nil && info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(abs) + if err != nil { + return "", err + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(abs), target) + } + return canonicalPortablePath(target) + } + parent := filepath.Dir(abs) + if parent == abs { + return "", err + } + resolved, err = canonicalPortablePath(parent) + return filepath.Join(resolved, filepath.Base(abs)), err +} + +func portableGitContext(ctx context.Context, executable string) (context.Context, error) { + if executable == "" { + if _, ok := ctx.Value(portableGitKey{}).(portableGitExecutable); ok { + return ctx, nil + } + executable = os.Getenv("GITCRAWL_PORTABLE_GIT") + } + if executable == "" { + var err error + executable, err = exec.LookPath("git") + if err != nil { + return ctx, fmt.Errorf("resolve portable Git executable: %w", err) + } + } else if !filepath.IsAbs(executable) { + return ctx, fmt.Errorf("portable Git executable must be an absolute path") + } + executable, err := filepath.Abs(executable) + if err != nil { + return ctx, err + } + executable, err = filepath.EvalSymlinks(executable) + if err != nil { + return ctx, fmt.Errorf("resolve portable Git executable: %w", err) + } + info, err := os.Stat(executable) + if err != nil || !info.Mode().IsRegular() { + return ctx, fmt.Errorf("portable Git executable is not a regular file") + } + return context.WithValue(ctx, portableGitKey{}, portableGitExecutable{path: executable, info: info}), nil +} + +// The permanent sibling lock survives recovery renames. Never unlink it: doing +// so would allow two owners to lock different inodes at the same pathname. +func acquirePortableOwner(ctx context.Context, root string) (context.Context, func(), error) { + root, err := canonicalPortablePath(root) + if err != nil { + return ctx, nil, err + } + session, _ := ctx.Value(portableCommandKey{}).(*portableCommandSession) + if session != nil { + session.mu.Lock() + defer session.mu.Unlock() + if session.owner != nil { + ctx = context.WithValue(ctx, portableOwnerKey{}, session.owner) + ctx = context.WithValue(ctx, portableGitKey{}, session.owner.git) + } + } + if owner, ok := ctx.Value(portableOwnerKey{}).(*portableOwner); ok { + if owner.root != root { + return ctx, nil, fmt.Errorf("portable operation already owns a different store") + } + return ctx, func() {}, owner.check() + } + ctx, err = portableGitContext(ctx, "") + if err != nil { + return ctx, nil, err + } + if err := ctx.Err(); err != nil { + return ctx, nil, err + } + if err := os.MkdirAll(filepath.Dir(root), 0o755); err != nil { + return ctx, nil, err + } + lockPath := filepath.Join(filepath.Dir(root), "."+filepath.Base(root)+".gitcrawl.lock") + if info, err := os.Lstat(lockPath); err == nil && !info.Mode().IsRegular() { + return ctx, nil, fmt.Errorf("portable ownership lock is not a regular file") + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return ctx, nil, err + } + file, err := openPortableLockFile(lockPath) + if err != nil { + return ctx, nil, err + } + owner := &portableOwner{root: root, file: file, git: ctx.Value(portableGitKey{}).(portableGitExecutable)} + if err := owner.check(); err != nil { + _ = file.Close() + return ctx, nil, err + } + if err := lockPortableFile(file); err != nil { + _ = file.Close() + return ctx, nil, fmt.Errorf("portable store is busy (ownership lock): %w", err) + } + if session != nil && session.retain { + session.owner = owner + return context.WithValue(ctx, portableOwnerKey{}, owner), func() {}, nil + } + return context.WithValue(ctx, portableOwnerKey{}, owner), func() { + _ = file.Close() + }, nil +} + +func (owner *portableOwner) check() error { + opened, err := owner.file.Stat() + if err != nil { + return err + } + current, err := os.Lstat(owner.file.Name()) + if err != nil || !current.Mode().IsRegular() || !os.SameFile(opened, current) { + return fmt.Errorf("portable ownership lock changed") + } + return nil +} + +func validatePortableRelativePath(value string) error { + if value == "" || strings.TrimSpace(value) != value || strings.ContainsAny(value, "\\\x00\r\n\t:<>\"|?*") || filepath.IsAbs(value) { + return fmt.Errorf("portable database must be a clean relative slash path") + } + for _, part := range strings.Split(value, "/") { + if part == "" || part == "." || part == ".." || strings.EqualFold(part, ".git") || strings.HasSuffix(part, ".") || strings.HasSuffix(part, " ") { + return fmt.Errorf("portable database must be a clean relative slash path outside Git metadata") + } + name, _, _ := strings.Cut(strings.ToUpper(part), ".") + if name == "CON" || name == "PRN" || name == "AUX" || name == "NUL" || len(name) == 4 && (strings.HasPrefix(name, "COM") || strings.HasPrefix(name, "LPT")) && name[3] >= '1' && name[3] <= '9' { + return fmt.Errorf("portable path contains a reserved device name") + } + } + return nil +} + +func validatePortableRemote(value string) error { + if value == "" || strings.TrimSpace(value) != value || strings.HasPrefix(value, "-") || strings.ContainsAny(value, "\x00\r\n") || strings.Contains(value, "::") { + return fmt.Errorf("invalid portable remote") + } + if strings.Contains(value, "://") { + parsed, err := url.Parse(value) + if err != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return fmt.Errorf("invalid portable remote URL") + } + if parsed.User != nil { + _, password := parsed.User.Password() + if parsed.Scheme != "ssh" || password || parsed.User.Username() == "" { + return fmt.Errorf("credentials are not allowed in portable remote URLs") + } + } + switch parsed.Scheme { + case "https", "http", "ssh", "git", "file": + default: + return fmt.Errorf("unsupported portable remote transport") + } + } + return nil +} diff --git a/internal/cli/portable_owner_test.go b/internal/cli/portable_owner_test.go new file mode 100644 index 00000000..ff1887e1 --- /dev/null +++ b/internal/cli/portable_owner_test.go @@ -0,0 +1,118 @@ +package cli + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPortableOwnerRejectsCrossStoreAndCanceledAcquisition(t *testing.T) { + // Resolving the test executable is sufficient: ownership never runs Git. + t.Setenv("GITCRAWL_PORTABLE_GIT", os.Args[0]) + root := t.TempDir() + first, second := filepath.Join(root, "first"), filepath.Join(root, "second") + ctx, release, err := acquirePortableOwner(context.Background(), first) + if err != nil { + t.Fatal(err) + } + defer release() + if _, otherRelease, err := acquirePortableOwner(ctx, second); err == nil || !strings.Contains(err.Error(), "already owns a different store") || otherRelease != nil { + t.Fatalf("nested ownership changed stores: release=%t, %v", otherRelease != nil, err) + } + if _, competingRelease, err := acquirePortableOwner(context.Background(), first); err == nil { + competingRelease() + t.Fatal("cross-store refusal released the original lease") + } + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if _, canceledRelease, err := acquirePortableOwner(canceled, second); !errors.Is(err, context.Canceled) || canceledRelease != nil { + t.Fatalf("canceled acquisition: release=%t, %v", canceledRelease != nil, err) + } + if _, err := os.Stat(filepath.Join(root, ".second.gitcrawl.lock")); !os.IsNotExist(err) { + t.Fatalf("refused acquisition created a second lock: %v", err) + } + owner := ctx.Value(portableOwnerKey{}).(*portableOwner) + if err := owner.check(); err != nil { + t.Fatalf("original owner no longer valid: %v", err) + } + release() + if err := owner.check(); !errors.Is(err, os.ErrClosed) { + t.Fatalf("released file still represents ownership: %v", err) + } + if _, err := os.Stat(owner.file.Name()); err != nil { + t.Fatalf("release removed permanent lock: %v", err) + } +} + +func TestPortableGitExecutableSelectionAndRevalidation(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "selected-git") + writePortableSafetyFile(t, path, []byte("original executable")) + t.Setenv("GITCRAWL_PORTABLE_GIT", "relative-git") + if _, err := portableGitContext(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "absolute path") { + t.Fatalf("relative environment selector accepted: %v", err) + } + for _, test := range []struct{ path, want string }{ + {filepath.Join(root, "missing"), "resolve portable Git executable"}, + {root, "not a regular file"}, + } { + if _, err := portableGitContext(context.Background(), test.path); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("invalid selector %q: %v", test.path, err) + } + } + ctx, err := portableGitContext(context.Background(), path) + if err != nil { + t.Fatal(err) + } + // A pinned executable is not reselected from a changed environment. + reused, err := portableGitContext(ctx, "") + if err != nil || reused != ctx { + t.Fatalf("pinned executable was re-resolved: %v", err) + } + if err := os.Rename(path, path+".previous"); err != nil { + t.Fatal(err) + } + writePortableSafetyFile(t, path, []byte("replacement executable")) + before := portableTestSnapshot(t, root) + if err := runPortableGit(ctx, root, io.Discard, "status"); err == nil || err.Error() != "portable Git executable changed during operation" { + t.Fatalf("replaced executable not refused before start: %v", err) + } + if string(before) != string(portableTestSnapshot(t, root)) { + t.Fatal("executable refusal mutated files") + } + if err := runPortableGit(context.Background(), root, io.Discard, "status"); err == nil || err.Error() != "portable Git executable was not resolved" { + t.Fatalf("unresolved executable accepted: %v", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if err := runPortableGit(canceled, root, io.Discard, "status"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled command attempted executable validation/start: %v", err) + } +} + +func TestPortablePathAndRemoteAdmission(t *testing.T) { + for _, path := range []string{"data/archive.db", "archive.db", "data/COM10.db"} { + if err := validatePortableRelativePath(path); err != nil { + t.Fatalf("valid logical path %q rejected: %v", path, err) + } + } + for _, path := range []string{"", "/archive.db", "data\\archive.db", "../archive.db", "data//archive.db", "data/./archive.db", "data/.GiT/config", "data/archive.", "data/archive /db", "NUL.db", "data/com1.db", "LPT9/archive.db", "data/archive\n.db"} { + if err := validatePortableRelativePath(path); err == nil { + t.Fatalf("unsafe logical path %q accepted", path) + } + } + for _, remote := range []string{"https://example.invalid/archive.git", "ssh://git@example.invalid/archive.git", "git@example.invalid:archive.git", "file:///synthetic/archive.git"} { + if err := validatePortableRemote(remote); err != nil { + t.Fatalf("valid remote %q rejected: %v", remote, err) + } + } + for _, remote := range []string{"", " --upload-pack=helper", "-option", "ext::helper", "https://example.invalid/archive?token=synthetic", "https://example.invalid/archive#fragment", "https://%invalid/archive", "ftp://example.invalid/archive", "ssh://example.invalid/archive\nargument", "https://git@example.invalid/archive.git", "http://git@example.invalid/archive.git", "git://git@example.invalid/archive.git", "file://git@example.invalid/archive.git", "ssh://git:@example.invalid/archive.git", "ssh://@example.invalid/archive.git"} { + if err := validatePortableRemote(remote); err == nil { + t.Fatalf("unsafe remote %q accepted", remote) + } + } +} diff --git a/internal/cli/portable_platform_unix.go b/internal/cli/portable_platform_unix.go new file mode 100644 index 00000000..f1d54469 --- /dev/null +++ b/internal/cli/portable_platform_unix.go @@ -0,0 +1,44 @@ +//go:build !windows + +package cli + +import ( + "os" + "os/exec" + "syscall" + + "golang.org/x/sys/unix" +) + +func lockPortableFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) +} + +func openPortableLockFile(path string) (*os.File, error) { + descriptor, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(descriptor), path), nil +} + +func portableFreeBytes(path string) (uint64, error) { + var stats unix.Statfs_t + err := unix.Statfs(path, &stats) + return uint64(stats.Bavail) * uint64(stats.Bsize), err +} + +func configurePortableProcess(cmd *exec.Cmd) error { + configureCommandGroup(cmd) + return nil +} + +func attachPortableProcess(cmd *exec.Cmd) error { return nil } + +func killPortableProcess(cmd *exec.Cmd) { killCommandGroup(cmd) } + +func terminatePortableProcess(cmd *exec.Cmd) { + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM) + } +} diff --git a/internal/cli/portable_platform_windows.go b/internal/cli/portable_platform_windows.go new file mode 100644 index 00000000..2de659d0 --- /dev/null +++ b/internal/cli/portable_platform_windows.go @@ -0,0 +1,98 @@ +//go:build windows + +package cli + +import ( + "fmt" + "os" + "os/exec" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +func lockPortableFile(file *os.File) error { + return windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &windows.Overlapped{}) +} + +func openPortableLockFile(path string) (*os.File, error) { + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + handle, err := windows.CreateFile(name, windows.GENERIC_READ|windows.GENERIC_WRITE, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, nil, windows.OPEN_ALWAYS, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, 0) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(handle), path), nil +} + +func portableFreeBytes(path string) (uint64, error) { + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + var free uint64 + err = windows.GetDiskFreeSpaceEx(name, &free, nil, nil) + return free, err +} + +func configurePortableProcess(cmd *exec.Cmd) error { + configureCommandGroup(cmd) + if _, ok := commandJobs.Load(cmd); !ok { + return fmt.Errorf("create portable Git process job") + } + // Assign the suspended process before it can create uncontained children. + cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_SUSPENDED | windows.CREATE_NEW_PROCESS_GROUP} + return nil +} + +func attachPortableProcess(cmd *exec.Cmd) error { + job, ok := commandJobs.Load(cmd) + if !ok { + return fmt.Errorf("portable Git job missing") + } + process, err := windows.OpenProcess(windows.PROCESS_SET_QUOTA|windows.PROCESS_TERMINATE, false, uint32(cmd.Process.Pid)) + if err != nil { + return err + } + defer windows.CloseHandle(process) + if err := windows.AssignProcessToJobObject(job.(windows.Handle), process); err != nil { + return err + } + snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPTHREAD, 0) + if err != nil { + return err + } + defer windows.CloseHandle(snapshot) + entry := windows.ThreadEntry32{Size: uint32(unsafe.Sizeof(windows.ThreadEntry32{}))} + for err = windows.Thread32First(snapshot, &entry); err == nil; err = windows.Thread32Next(snapshot, &entry) { + if entry.OwnerProcessID != uint32(cmd.Process.Pid) { + continue + } + thread, err := windows.OpenThread(windows.THREAD_SUSPEND_RESUME, false, entry.ThreadID) + if err != nil { + return err + } + _, resumeErr := windows.ResumeThread(thread) + _ = windows.CloseHandle(thread) + return resumeErr + } + return fmt.Errorf("find suspended portable Git thread: %w", err) +} + +func terminatePortableProcess(cmd *exec.Cmd) { + if cmd.Process != nil { + _ = windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(cmd.Process.Pid)) + } +} + +func killPortableProcess(cmd *exec.Cmd) { + if job, ok := commandJobs.Load(cmd); ok { + _ = windows.TerminateJobObject(job.(windows.Handle), 1) + } + if cmd.Process != nil { + _ = cmd.Process.Kill() + } +} diff --git a/internal/cli/portable_refresh.go b/internal/cli/portable_refresh.go new file mode 100644 index 00000000..3d636d54 --- /dev/null +++ b/internal/cli/portable_refresh.go @@ -0,0 +1,438 @@ +package cli + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/openclaw/gitcrawl/internal/config" +) + +type portableRefreshOptions struct { + StoreDir string + PortableDB string + ExpectedRemote string + Branch string + Git string + Timeout time.Duration + Reserve uint64 + Growth int64 +} + +type portableRefreshResult struct { + Stage string `json:"stage"` + Result string `json:"result"` + BeforeCommit string `json:"before_commit,omitempty"` + AfterCommit string `json:"after_commit,omitempty"` + TargetCommit string `json:"target_commit,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + SHA256 string `json:"sha256,omitempty"` + ArtifactBytes int64 `json:"artifact_bytes,omitempty"` + Mirror string `json:"mirror_destination,omitempty"` + MirrorResult string `json:"mirror_result,omitempty"` + Capacity portableCapacity `json:"capacity"` + ElapsedMS int64 `json:"elapsed_ms"` + Reason string `json:"reason,omitempty"` +} + +func (a *App) runPortableRefresh(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("portable refresh", flag.ContinueOnError) + fs.SetOutput(io.Discard) + options := portableRefreshOptions{} + fs.StringVar(&options.StoreDir, "store-dir", "", "configured store checkout") + fs.StringVar(&options.PortableDB, "portable-db", "", "logical relative database path") + fs.StringVar(&options.ExpectedRemote, "expected-remote", "", "required expected origin URL") + fs.StringVar(&options.Branch, "branch", "main", "expected origin branch") + fs.StringVar(&options.Git, "git", "", "absolute Git executable (or GITCRAWL_PORTABLE_GIT)") + fs.DurationVar(&options.Timeout, "timeout", portableOperationTimeout, "total operation deadline") + fs.Uint64Var(&options.Reserve, "min-free-bytes", portableDefaultReserve, "minimum free-space reserve") + fs.Int64Var(&options.Growth, "max-growth-bytes", portableDefaultGrowth, "maximum observed temporary growth") + jsonOut := fs.Bool("json", false, "write JSON output") + values := map[string]bool{"store-dir": true, "portable-db": true, "expected-remote": true, "branch": true, "git": true, "timeout": true, "min-free-bytes": true, "max-growth-bytes": true} + if err := fs.Parse(normalizeCommandArgs(args, values)); err != nil { + return usageErr(err) + } + a.applyCommandJSON(*jsonOut) + if fs.NArg() != 0 || options.Timeout <= 0 || options.Reserve == 0 || options.Growth <= 0 { + return usageErr(fmt.Errorf("portable refresh accepts no positional arguments and requires positive timeout and byte limits")) + } + if err := validatePortableRemote(options.ExpectedRemote); err != nil { + return usageErr(fmt.Errorf("--expected-remote is required and must identify the intended origin")) + } + if options.PortableDB != "" { + if err := validatePortableRelativePath(options.PortableDB); err != nil { + return usageErr(err) + } + } + ctx, cancel := context.WithTimeout(ctx, options.Timeout) + defer cancel() + ctx, err := portableGitContext(ctx, options.Git) + if err != nil { + return usageErr(err) + } + if strings.HasPrefix(options.Branch, "-") || strings.ContainsAny(options.Branch, "\x00\r\n") { + return usageErr(fmt.Errorf("invalid --branch")) + } + result, refreshErr := a.refreshPortable(ctx, options) + if err := a.writeOutput("portable refresh", result, true); err != nil { + return errors.Join(refreshErr, err) + } + return refreshErr +} + +func (a *App) refreshPortable(ctx context.Context, options portableRefreshOptions) (result portableRefreshResult, retErr error) { + started := time.Now() + result.Stage, result.Result = "admission", "refused" + result.Capacity = portableCapacity{ReserveBytes: options.Reserve, GrowthLimit: options.Growth} + var checkout portableCheckout + advanced := false + defer func() { + result.ElapsedMS = time.Since(started).Milliseconds() + exitCode := 0 + if retErr != nil { + exitCode = 1 + // Never claim rollback: merge or promotion can have completed before + // an external writer, deadline or I/O error interrupts the next stage. + if advanced { + result.Result = "partial" + } + result.Reason = portableRefreshReason(retErr) + retErr = fmt.Errorf("portable refresh %s: %s", result.Stage, result.Reason) + } + fmt.Fprintf(a.Stderr, "gitcrawl: portable refresh: stage=%s result=%s elapsed=%s exit=%d\n", result.Stage, result.Result, time.Since(started).Round(time.Millisecond), exitCode) + }() + stage := func(name string) { + result.Stage = name + fmt.Fprintf(a.Stderr, "gitcrawl: portable refresh: stage=%s elapsed=%s\n", name, time.Since(started).Round(time.Millisecond)) + } + if _, err := portableGitOutput(ctx, "", "check-ref-format", "refs/heads/"+options.Branch); err != nil { + return result, fmt.Errorf("validate portable branch: %w", err) + } + cfg, err := config.LoadRuntime(a.configPath) + if err != nil { + return result, fmt.Errorf("could not load configured portable database") + } + if cfg.Remote.Enabled() && cfg.Remote.Mode != "local" { + return result, fmt.Errorf("portable refresh requires a local configured database") + } + configPath := config.ResolvePath(a.configPath) + configStamp, err := portableFileSHA256(ctx, configPath) + if err != nil { + return result, err + } + source, err := canonicalPortablePath(cfg.DBPath) + if err != nil { + return result, err + } + root, ok, err := portableStoreRoot(ctx, source) + if err != nil || !ok { + return result, fmt.Errorf("configured database is not in a portable checkout") + } + root, err = canonicalPortablePath(root) + if err != nil { + return result, err + } + if options.StoreDir != "" { + expected, err := a.absoluteInitPath(options.StoreDir) + if err != nil { + return result, err + } + expected, err = canonicalPortablePath(expected) + if err != nil || expected != root { + return result, fmt.Errorf("--store-dir does not match the configured database store") + } + } + relative, err := filepath.Rel(root, source) + if err != nil { + return result, err + } + relative = filepath.ToSlash(relative) + if err := validatePortableRelativePath(relative); err != nil { + return result, err + } + if options.PortableDB != "" && options.PortableDB != relative { + return result, fmt.Errorf("--portable-db does not match the configured logical database") + } + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return result, err + } + defer release() + checkout, err = inspectPortableCheckout(ctx, root, options.ExpectedRemote, options.Branch) + result.BeforeCommit, result.AfterCommit = checkout.head, checkout.head + if err != nil { + return result, err + } + if _, err := portableCommitTree(ctx, root, checkout.head, options.Growth); err != nil { + return result, err + } + // Keep the configured spelling for the established runtime destination; + // canonicalization binds ownership, not a migration of writable mirrors. + mirrorPath, err := a.portableRuntimeDBPath(ctx, cfg.DBPath) + if err != nil { + return result, err + } + mirrorPath, err = canonicalPortablePath(mirrorPath) + if err != nil { + return result, err + } + if pathWithin(root, mirrorPath) || pathWithin(filepath.Dir(mirrorPath), root) { + return result, fmt.Errorf("runtime mirror must be outside the portable checkout") + } + result.Mirror = mirrorPath + if err := os.MkdirAll(filepath.Dir(mirrorPath), 0o700); err != nil { + return result, err + } + budget, err := newPortableBudget(ctx, []string{root, filepath.Dir(mirrorPath)}, options.Reserve, options.Growth) + if budget != nil { + defer func() { result.Capacity = budget.snapshot() }() + } + if err != nil { + return result, err + } + ctx, stopMonitor := budget.monitor(ctx) + defer stopMonitor() + defer func() { + if retErr != nil && ctx.Err() != nil { + retErr = context.Cause(ctx) + } + }() + staging, err := os.MkdirTemp(filepath.Dir(mirrorPath), ".gitcrawl-refresh-*") + if err != nil { + return result, err + } + stagingInfo, err := os.Lstat(staging) + if err != nil { + return result, err + } + defer func() { + if err := removeOwnedPortableStaging(staging, stagingInfo); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("remove owned staging: %w", err)) + } + }() + ctx = context.WithValue(ctx, portableValidationDirKey{}, staging) + checkBoundary := func(head, tracking string) error { + if err := budget.check(ctx); err != nil { + return err + } + stamp, err := portableFileSHA256(ctx, configPath) + if err != nil || stamp != configStamp { + return fmt.Errorf("configuration changed during refresh") + } + configuredSource, err := canonicalPortablePath(cfg.DBPath) + if err != nil || configuredSource != source { + return fmt.Errorf("configured portable database path changed during refresh") + } + return checkout.recheck(ctx, head, tracking) + } + if err := checkBoundary(checkout.head, checkout.tracking); err != nil { + return result, err + } + stage("fetch") + if err := runPortableGit(ctx, root, io.Discard, "fetch", "--no-auto-maintenance", "--no-prune", "--no-prune-tags", "--no-tags", "--no-recurse-submodules", "--refmap=", "--", "origin", "refs/heads/"+options.Branch); err != nil { + return result, fmt.Errorf("fetch failed: %w", err) + } + target, err := portableRef(ctx, root, "FETCH_HEAD") + if err != nil { + return result, err + } + result.TargetCommit = target + for _, ancestor := range []string{checkout.head, checkout.tracking} { + if err := runPortableGit(ctx, root, io.Discard, "merge-base", "--is-ancestor", ancestor, target); err != nil { + return result, fmt.Errorf("fetched branch is not a fast-forward of HEAD and the tracking ref") + } + } + stage("validate") + tree, err := portableCommitTree(ctx, root, target, options.Growth) + if err != nil { + return result, err + } + stagedDB, manifest, err := stagePortableCommit(ctx, root, staging, relative, tree, options.Growth) + if err != nil { + return result, err + } + result.ArtifactID, result.SHA256, result.ArtifactBytes = manifest.ArtifactID, manifest.SHA256, manifest.OutputBytes + if result.ArtifactID == "" { + result.ArtifactID = manifest.SHA256 + } + mirror, err := inspectPortableMirror(ctx, mirrorPath, source) + if err != nil { + return result, err + } + result.MirrorResult = "preserved-local" + if mirror.exists && strings.EqualFold(fmt.Sprintf("%x", mirror.digest), manifest.SHA256) { + mirror.preserve = true + result.MirrorResult = "unchanged" + } + if err := checkBoundary(checkout.head, checkout.tracking); err != nil { + return result, err + } + stage("advance") + if target != checkout.head { + advanced = true // Even a failed merge can have performed some writes. + if err := runPortableGit(ctx, root, io.Discard, "merge", "--ff-only", "--no-autostash", "--no-overwrite-ignore", "--", target); err != nil { + result.AfterCommit = "" // Unknown, not a claimed rollback. + return result, fmt.Errorf("fast-forward failed; inspect checkout before retry: %w", err) + } + result.AfterCommit = target + } + if err := checkBoundary(target, checkout.tracking); err != nil { + return result, err + } + if target != checkout.tracking { + advanced = true + if err := runPortableGit(ctx, root, io.Discard, "update-ref", "refs/remotes/origin/"+options.Branch, target, checkout.tracking); err != nil { + return result, fmt.Errorf("tracking ref compare-and-swap failed: %w", err) + } + } + if err := checkBoundary(target, target); err != nil { + return result, err + } + stage("promote") + if info, err := os.Lstat(staging); err != nil || !os.SameFile(info, stagingInfo) { + return result, fmt.Errorf("owned staging directory changed during refresh") + } + if err := mirror.recheck(ctx); err != nil { + return result, err + } + if !mirror.preserve { + digest, err := portableFileSHA256(ctx, stagedDB) + if err != nil || !strings.EqualFold(fmt.Sprintf("%x", digest), manifest.SHA256) { + return result, fmt.Errorf("validated runtime generation changed before promotion") + } + if err := checkBoundary(target, target); err != nil { + return result, err + } + if err := os.Rename(stagedDB, mirrorPath); err != nil { + return result, fmt.Errorf("promote portable mirror: %w", err) + } + advanced = true + result.MirrorResult = "promoted" + if err := markPortableMirrorHealthVerified(mirrorPath, portableStoreRefreshStatePath(mirrorPath), source, manifest.SHA256); err != nil { + return result, err + } + } else if result.MirrorResult == "preserved-local" { + // Preserve the decision across ordinary reads, including older mirrors + // without a recorded source digest. Do not relabel their generation. + statePath := portableStoreRefreshStatePath(mirrorPath) + state := readPortableStoreRefreshState(statePath) + if !state.MirrorWritable { + state.MirrorWritable = true + if err := writePortableStoreRefreshState(statePath, state); err != nil { + return result, err + } + } + } + if err := budget.check(ctx); err != nil { + return result, err + } + result.Stage, result.Result = "complete", "updated" + if checkout.head == target && checkout.tracking == target { + result.Result = "no-op" + } + return result, nil +} + +func pathWithin(root, path string) bool { + relative, err := filepath.Rel(root, path) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(os.PathSeparator)) && !filepath.IsAbs(relative) +} + +func removeOwnedPortableStaging(path string, original os.FileInfo) error { + current, err := os.Lstat(path) + if err != nil || !current.IsDir() || !os.SameFile(original, current) { + return fmt.Errorf("owned staging directory changed; refusing cleanup") + } + return os.RemoveAll(path) +} + +func stagePortableCommit(ctx context.Context, root, staging, relative string, tree map[string]portableTreeEntry, growth int64) (string, portableDBManifest, error) { + logical := filepath.Join(staging, filepath.FromSlash(relative)) + manifestEntry, ok := tree[relative+".manifest.json"] + if !ok || manifestEntry.size > 1<<20 { + return "", portableDBManifest{}, fmt.Errorf("strict refresh requires a manifest no larger than 1 MiB") + } + if err := extractPortableBlob(ctx, root, manifestEntry, portableDBManifestPath(logical)); err != nil { + return "", portableDBManifest{}, err + } + artifact, manifest, compressed, err := portableSourceArtifact(logical) + if err != nil { + return "", manifest, err + } + if manifest.OutputPath != relative && manifest.OutputPath != filepath.Base(relative) { + return "", manifest, fmt.Errorf("manifest outputPath does not match the configured logical database") + } + artifactRelative, err := filepath.Rel(staging, artifact) + if err != nil || !pathWithin(staging, artifact) { + return "", manifest, fmt.Errorf("artifact path escapes staging") + } + entry, ok := tree[filepath.ToSlash(artifactRelative)] + if !ok { + return "", manifest, fmt.Errorf("manifest artifact is absent from fetched commit") + } + // Allow the extracted blob, inflated DB and both semantic-identity copies, + // plus another artifact-sized checkout allocation. The monitor also counts + // fetched objects and all other positive growth throughout the operation. + if manifest.OutputBytes <= 0 || manifest.OutputBytes > growth/4 || entry.size > (growth-4*manifest.OutputBytes)/2 { + return "", manifest, fmt.Errorf("artifact validation would exceed growth budget") + } + if compressed && entry.size != manifest.ArchiveBytes || !compressed && entry.size != manifest.OutputBytes { + return "", manifest, fmt.Errorf("manifest artifact size does not match fetched blob") + } + if err := extractPortableBlob(ctx, root, entry, artifact); err != nil { + return "", manifest, err + } + stagedDB, err := stagePortableSQLiteSourceTempContext(ctx, logical, filepath.Join(staging, "runtime.db"), 0o600) + if err != nil { + return "", manifest, err + } + if err := sqliteStoreImmutableHealth(ctx, stagedDB); err != nil { + return "", manifest, err + } + if err := validatePortableDBManifest(ctx, stagedDB, portableDBManifestPath(logical)); err != nil { + return "", manifest, err + } + file, err := os.OpenFile(stagedDB, os.O_RDWR, 0) + if err != nil { + return "", manifest, err + } + err = file.Sync() + err = errors.Join(err, file.Close()) + return stagedDB, manifest, err +} + +func extractPortableBlob(ctx context.Context, root string, entry portableTreeEntry, destination string) error { + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return err + } + file, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + writer := &portableLimitedWriter{writer: file, left: entry.size} + err = runPortableGit(ctx, root, writer, "cat-file", "blob", entry.oid) + err = errors.Join(err, file.Close()) + if err == nil && writer.left != 0 { + return fmt.Errorf("incomplete fetched artifact") + } + return err +} + +func portableRefreshReason(err error) string { + if strings.Contains(err.Error(), "portable manifest mismatch") { + return "portable artifact manifest verification failed" + } + // Our own errors carry stage context. File-system and process errors can + // contain local paths; line breaks are escaped and diagnostics are bounded. + reason := strings.ReplaceAll(strings.ReplaceAll(err.Error(), "\n", " "), "\r", " ") + if len(reason) > 512 { + reason = reason[:512] + } + return reason +} diff --git a/internal/cli/portable_refresh_test.go b/internal/cli/portable_refresh_test.go new file mode 100644 index 00000000..da3ded13 --- /dev/null +++ b/internal/cli/portable_refresh_test.go @@ -0,0 +1,519 @@ +package cli + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openclaw/gitcrawl/internal/config" +) + +type portableRefreshFixture struct { + remote string + checkout string + configPath string + relative string + mirror string +} + +func portableTestGit(t *testing.T, root string, args ...string) string { + t.Helper() + output, err := gitOutput(context.Background(), root, args...) + if err != nil { + t.Fatal(err) + } + return output +} + +func portableTestCommit(t *testing.T, root string) { + t.Helper() + portableTestGit(t, root, "add", "--all") + portableTestGit(t, root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", "commit", "-m", "fixture generation") +} + +func portableTestManifest(t *testing.T, path, relative string, compressed bool) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + digest, err := fileSHA256(path) + if err != nil { + t.Fatal(err) + } + manifest := portableDBManifest{Schema: "gitcrawl-portable-sync-v2", OutputPath: relative, OutputBytes: int64(len(data)), SHA256: fmt.Sprintf("%x", digest), QuickCheck: "ok"} + if compressed { + var archive bytes.Buffer + writer := gzip.NewWriter(&archive) + if _, err := writer.Write(data); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path+".gz", archive.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + archiveSHA, err := fileSHA256(path + ".gz") + if err != nil { + t.Fatal(err) + } + manifest.Compression, manifest.ArchivePath = "gzip", filepath.Base(path)+".gz" + manifest.ArchiveBytes, manifest.ArchiveSHA256 = int64(archive.Len()), fmt.Sprintf("%x", archiveSHA) + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + } + encoded, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(portableDBManifestPath(path), encoded, 0o644); err != nil { + t.Fatal(err) + } +} + +func newPortableRefreshFixture(t *testing.T, compressed bool) portableRefreshFixture { + t.Helper() + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + dir := t.TempDir() + fixture := portableRefreshFixture{remote: filepath.Join(dir, "publisher"), checkout: filepath.Join(dir, "subscriber"), configPath: filepath.Join(dir, "config.toml"), relative: "data/archive.db"} + if err := os.MkdirAll(filepath.Join(fixture.remote, "data"), 0o755); err != nil { + t.Fatal(err) + } + portableTestGit(t, fixture.remote, "init", "-b", "main") + seedPortableThread(t, filepath.Join(fixture.remote, fixture.relative), 1, "first generation") + portableTestManifest(t, filepath.Join(fixture.remote, fixture.relative), fixture.relative, compressed) + portableTestCommit(t, fixture.remote) + fixture.init(t) + app := New() + app.configPath = fixture.configPath + mirror, err := app.portableRuntimeDBPath(context.Background(), filepath.Join(fixture.checkout, fixture.relative)) + if err != nil { + t.Fatal(err) + } + fixture.mirror, err = canonicalPortablePath(mirror) + if err != nil { + t.Fatal(err) + } + return fixture +} + +func (fixture portableRefreshFixture) init(t *testing.T) { + t.Helper() + app := New() + app.Stdout, app.Stderr = &bytes.Buffer{}, &bytes.Buffer{} + if err := app.Run(context.Background(), []string{"--config", fixture.configPath, "init", "--portable-store", fixture.remote, "--store-dir", fixture.checkout, "--portable-db", fixture.relative, "--json"}); err != nil { + t.Fatal(err) + } +} + +func (fixture portableRefreshFixture) refresh(t *testing.T, extra ...string) (portableRefreshResult, error) { + t.Helper() + app := New() + var stdout, stderr bytes.Buffer + app.Stdout, app.Stderr = &stdout, &stderr + args := []string{"--config", fixture.configPath, "portable", "refresh", "--expected-remote", fixture.remote, "--min-free-bytes", "1", "--max-growth-bytes", "67108864", "--json"} + err := app.Run(context.Background(), append(args, extra...)) + var result portableRefreshResult + if decodeErr := json.Unmarshal(stdout.Bytes(), &result); decodeErr != nil { + t.Fatalf("result: %v; command error: %v; stdout=%s stderr=%s", decodeErr, err, stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "stage=") { + t.Fatalf("missing progress: %s", stderr.String()) + } + return result, err +} + +func (fixture portableRefreshFixture) advance(t *testing.T, compressed bool) { + t.Helper() + path := filepath.Join(fixture.remote, fixture.relative) + if _, err := os.Stat(path); os.IsNotExist(err) { + temp, err := stagePortableSQLiteSourceTemp(path, path, 0o644) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(temp, path); err != nil { + t.Fatal(err) + } + } + seedPortableThread(t, path, 2, "second generation") + portableTestManifest(t, path, fixture.relative, compressed) + portableTestCommit(t, fixture.remote) +} + +func TestPortableInitGzipRepeatedAndTransition(t *testing.T) { + for _, compressed := range []bool{false, true} { + t.Run(fmt.Sprint(compressed), func(t *testing.T) { + fixture := newPortableRefreshFixture(t, compressed) + fixture.init(t) + fixture.advance(t, true) + fixture.init(t) + fixture.init(t) + cfg, err := config.Load(fixture.configPath) + if err != nil { + t.Fatal(err) + } + logical := filepath.Join(fixture.checkout, fixture.relative) + if cfg.DBPath != logical { + t.Fatalf("logical DB changed: %s", cfg.DBPath) + } + if _, err := os.Stat(logical); !os.IsNotExist(err) { + t.Fatalf("gzip init required a raw DB: %v", err) + } + }) + } +} + +func TestPortableInitValidatesBeforeGitAndPreservesConfig(t *testing.T) { + fixture := newPortableRefreshFixture(t, true) + before, _ := os.ReadFile(fixture.configPath) + head := portableTestGit(t, fixture.checkout, "rev-parse", "HEAD") + fixture.advance(t, true) + for _, path := range []string{"../escape.db", "/absolute.db", "data/../archive.db", "C:\\archive.db", ".git/config", "data//archive.db"} { + app := New() + app.Stdout, app.Stderr = &bytes.Buffer{}, &bytes.Buffer{} + err := app.Run(context.Background(), []string{"--config", fixture.configPath, "init", "--portable-store", fixture.remote, "--store-dir", fixture.checkout, "--portable-db", path}) + if err == nil { + t.Fatalf("accepted %q", path) + } + if got := portableTestGit(t, fixture.checkout, "rev-parse", "HEAD"); got != head { + t.Fatal("invalid syntax mutated Git") + } + } + if err := os.WriteFile(filepath.Join(fixture.remote, fixture.relative)+".gz", []byte("invalid artifact"), 0o644); err != nil { + t.Fatal(err) + } + portableTestCommit(t, fixture.remote) + app := New() + app.Stdout, app.Stderr = &bytes.Buffer{}, &bytes.Buffer{} + if err := app.Run(context.Background(), []string{"--config", fixture.configPath, "init", "--portable-store", fixture.remote, "--store-dir", fixture.checkout, "--portable-db", fixture.relative}); err == nil { + t.Fatal("invalid artifact accepted") + } + after, _ := os.ReadFile(fixture.configPath) + if !bytes.Equal(before, after) { + t.Fatal("failed init replaced config") + } +} + +func TestPortableRefreshSuccessNoopAndLocalMirror(t *testing.T) { + fixture := newPortableRefreshFixture(t, false) + initial, err := fixture.refresh(t) + if err != nil || initial.Result != "no-op" || initial.MirrorResult != "promoted" { + t.Fatalf("initial: %+v %v", initial, err) + } + beforeConfig, _ := os.ReadFile(fixture.configPath) + fixture.advance(t, true) + updated, err := fixture.refresh(t, "--store-dir", fixture.checkout, "--portable-db", fixture.relative) + if err != nil || updated.Result != "updated" || updated.BeforeCommit == updated.AfterCommit || updated.SHA256 == "" { + t.Fatalf("update: %+v %v", updated, err) + } + if updated.Capacity.PeakGrowth <= 0 || updated.Mirror != fixture.mirror { + t.Fatalf("observations: %+v", updated) + } + if err := sqliteStoreImmutableHealth(context.Background(), fixture.mirror); err != nil { + t.Fatal(err) + } + seedPortableThread(t, fixture.mirror, 3, "local writable runtime state") + localSHA, _ := fileSHA256(fixture.mirror) + result, err := fixture.refresh(t) + if err != nil || result.Result != "no-op" || result.MirrorResult != "preserved-local" { + t.Fatalf("local: %+v %v", result, err) + } + afterSHA, _ := fileSHA256(fixture.mirror) + if localSHA != afterSHA { + t.Fatal("local runtime changed") + } + afterConfig, _ := os.ReadFile(fixture.configPath) + if !bytes.Equal(beforeConfig, afterConfig) { + t.Fatal("refresh rewrote config") + } +} + +func TestPortableRefreshRefusalsPreserveLastGood(t *testing.T) { + cases := []string{"invalid", "dirty", "index", "ignored", "divergent", "ahead", "origin", "hook", "filter", "lock", "runtime-lock", "orphan", "low-space", "growth", "timeout", "submodule"} + for _, name := range cases { + t.Run(name, func(t *testing.T) { + fixture := newPortableRefreshFixture(t, false) + if _, err := fixture.refresh(t); err != nil { + t.Fatal(err) + } + fixture.advance(t, true) + extra := []string{} + switch name { + case "invalid": + if err := os.WriteFile(filepath.Join(fixture.remote, fixture.relative)+".gz", []byte("bad gzip"), 0o644); err != nil { + t.Fatal(err) + } + portableTestCommit(t, fixture.remote) + case "dirty", "index": + if err := os.WriteFile(filepath.Join(fixture.checkout, fixture.relative), []byte("local edit"), 0o644); err != nil { + t.Fatal(err) + } + if name == "index" { + portableTestGit(t, fixture.checkout, "add", fixture.relative) + } + case "ignored": + if err := os.WriteFile(filepath.Join(fixture.checkout, ".git", "info", "exclude"), []byte("collision\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixture.checkout, "collision"), []byte("keep ignored data"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixture.remote, "collision"), []byte("incoming"), 0o644); err != nil { + t.Fatal(err) + } + portableTestCommit(t, fixture.remote) + case "divergent", "ahead": + if err := os.WriteFile(filepath.Join(fixture.checkout, "local"), []byte("local commit"), 0o644); err != nil { + t.Fatal(err) + } + portableTestCommit(t, fixture.checkout) + if name == "ahead" { + portableTestGit(t, fixture.remote, "update-ref", "refs/heads/main", portableTestGit(t, fixture.remote, "rev-parse", "HEAD^")) + } + case "origin": + portableTestGit(t, fixture.checkout, "remote", "set-url", "origin", filepath.Join(t.TempDir(), "unrelated")) + case "hook": + if err := os.WriteFile(filepath.Join(fixture.checkout, ".git", "hooks", "post-merge"), []byte("#!/bin/sh\nexit 99\n"), 0o755); err != nil { + t.Fatal(err) + } + case "filter": + portableTestGit(t, fixture.checkout, "config", "filter.test.smudge", "false") + case "lock": + if err := os.WriteFile(filepath.Join(fixture.checkout, ".git", "index.lock"), []byte("live"), 0o600); err != nil { + t.Fatal(err) + } + case "orphan": + if err := os.WriteFile(filepath.Join(fixture.checkout, ".git", "objects", "pack", "tmp_pack_unknown"), []byte("preserve"), 0o600); err != nil { + t.Fatal(err) + } + case "runtime-lock": + if err := os.WriteFile(portableStoreRefreshStatePath(fixture.mirror)+".lock", []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + case "low-space": + extra = []string{"--min-free-bytes", "18446744073709551615"} + case "growth": + extra = []string{"--max-growth-bytes", "1"} + case "timeout": + extra = []string{"--timeout", "1ns"} + case "submodule": + portableTestGit(t, fixture.remote, "update-index", "--add", "--cacheinfo", "160000,"+portableTestGit(t, fixture.remote, "rev-parse", "HEAD")+",module") + portableTestGit(t, fixture.remote, "-c", "user.name=Test", "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", "commit", "-m", "gitlink") + } + head := portableTestGit(t, fixture.checkout, "rev-parse", "HEAD") + tracking := portableTestGit(t, fixture.checkout, "rev-parse", "refs/remotes/origin/main") + mirrorSHA, _ := fileSHA256(fixture.mirror) + configSHA, _ := fileSHA256(fixture.configPath) + if name == "timeout" { + // A deadline can expire during read-only flag validation, before + // the structured operation starts. + app := New() + app.Stdout, app.Stderr = &bytes.Buffer{}, &bytes.Buffer{} + if err := app.Run(context.Background(), []string{"--config", fixture.configPath, "portable", "refresh", "--expected-remote", fixture.remote, "--timeout", "1ns", "--json"}); err == nil { + t.Fatal("timeout accepted") + } + } else { + result, err := fixture.refresh(t, extra...) + if err == nil || result.Result != "refused" { + t.Fatalf("refusal: %+v %v", result, err) + } + } + if portableTestGit(t, fixture.checkout, "rev-parse", "HEAD") != head || portableTestGit(t, fixture.checkout, "rev-parse", "refs/remotes/origin/main") != tracking { + t.Fatal("refusal changed refs") + } + if sha, _ := fileSHA256(fixture.mirror); sha != mirrorSHA { + t.Fatal("refusal changed mirror") + } + if sha, _ := fileSHA256(fixture.configPath); sha != configSHA { + t.Fatal("refusal changed config") + } + staging, _ := filepath.Glob(filepath.Join(filepath.Dir(fixture.mirror), ".gitcrawl-refresh-*")) + if len(staging) != 0 { + t.Fatalf("owned staging leaked: %v", staging) + } + }) + } +} + +func TestPortableOwnershipCrossProcessAndSymlink(t *testing.T) { + if root := os.Getenv("GITCRAWL_TEST_LOCK_ROOT"); root != "" { + _, release, err := acquirePortableOwner(context.Background(), root) + if err == nil { + release() + os.Exit(19) + } + os.Exit(0) + } + root := filepath.Join(t.TempDir(), "store") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + ctx, release, err := acquirePortableOwner(context.Background(), root) + if err != nil { + t.Fatal(err) + } + defer release() + owner := ctx.Value(portableOwnerKey{}).(*portableOwner) + old := time.Now().Add(-24 * time.Hour) + if err := os.Chtimes(owner.file.Name(), old, old); err != nil { + t.Fatal(err) + } + command := exec.Command(os.Args[0], "-test.run=^TestPortableOwnershipCrossProcessAndSymlink$") + command.Env = append(os.Environ(), "GITCRAWL_TEST_LOCK_ROOT="+root) + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("competing child stole live lock: %v %s", err, out) + } + alias := filepath.Join(filepath.Dir(root), "alias") + if err := os.Symlink(root, alias); err == nil { + if _, closeAlias, err := acquirePortableOwner(context.Background(), alias); err == nil { + closeAlias() + t.Fatal("symlink bypassed lock") + } + } + if err := owner.check(); err != nil { + t.Fatal(err) + } + _, nestedRelease, err := acquirePortableOwner(ctx, root) + if err != nil { + t.Fatal(err) + } + nestedRelease() + if _, err := os.Stat(owner.file.Name()); err != nil { + t.Fatal("live lock removed", err) + } +} + +func TestPortableBudgetCountsGrowthWithoutDeletionCredit(t *testing.T) { + root := t.TempDir() + old := filepath.Join(root, "old") + if err := os.WriteFile(old, bytes.Repeat([]byte("x"), 100), 0o600); err != nil { + t.Fatal(err) + } + budget, err := newPortableBudget(context.Background(), []string{root}, 1, 32) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(old); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "new"), bytes.Repeat([]byte("x"), 33), 0o600); err != nil { + t.Fatal(err) + } + if err := budget.check(context.Background()); err == nil { + t.Fatal("deletion credited against growth") + } +} + +func TestPortableStagingCleanupPreservesReplacement(t *testing.T) { + root := t.TempDir() + staging := filepath.Join(root, "owned") + if err := os.Mkdir(staging, 0o700); err != nil { + t.Fatal(err) + } + info, err := os.Stat(staging) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(staging, staging+"-moved"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(staging, 0o700); err != nil { + t.Fatal(err) + } + unknown := filepath.Join(staging, "unknown") + if err := os.WriteFile(unknown, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := removeOwnedPortableStaging(staging, info); err == nil { + t.Fatal("cleanup accepted a replacement directory") + } + if data, err := os.ReadFile(unknown); err != nil || string(data) != "keep" { + t.Fatal("cleanup touched unrelated contents") + } +} + +func TestPortableOwnershipBeforeCloneConvergesThroughLink(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "future-store") + alias := filepath.Join(parent, "alias") + if err := os.Symlink(root, alias); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + _, release, err := acquirePortableOwner(context.Background(), alias) + if err != nil { + t.Fatal(err) + } + defer release() + if _, otherRelease, err := acquirePortableOwner(context.Background(), root); err == nil { + otherRelease() + t.Fatal("first clone could acquire competing locks through a dangling alias") + } +} + +func TestPortableCommandOwnershipRetainsOnlyWritableSessions(t *testing.T) { + for _, retain := range []bool{false, true} { + t.Run(fmt.Sprint(retain), func(t *testing.T) { + root := t.TempDir() + session := &portableCommandSession{retain: retain} + defer session.close() + ctx := context.WithValue(context.Background(), portableCommandKey{}, session) + _, release, err := acquirePortableOwner(ctx, root) + if err != nil { + t.Fatal(err) + } + release() + _, otherRelease, err := acquirePortableOwner(context.Background(), root) + if err == nil { + otherRelease() + } + if retain && err == nil { + t.Fatal("writable session released ownership before close") + } + if !retain && err != nil { + t.Fatal("reader retained ownership after preparation", err) + } + }) + } +} + +func TestPortableRefreshUsesExistingRuntimeForStoreAlias(t *testing.T) { + fixture := newPortableRefreshFixture(t, true) + alias := filepath.Join(filepath.Dir(fixture.checkout), "linked-store") + if err := os.Symlink(fixture.checkout, alias); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + cfg, err := config.Load(fixture.configPath) + if err != nil { + t.Fatal(err) + } + cfg.DBPath = filepath.Join(alias, fixture.relative) + if err := config.Save(fixture.configPath, cfg); err != nil { + t.Fatal(err) + } + app := New() + app.configPath = fixture.configPath + expected, err := app.portableRuntimeDBPath(context.Background(), cfg.DBPath) + if err != nil { + t.Fatal(err) + } + expected, err = canonicalPortablePath(expected) + if err != nil { + t.Fatal(err) + } + result, err := fixture.refresh(t, "--store-dir", alias) + if err != nil || result.Mirror != expected { + t.Fatalf("alias runtime moved: %+v %v; expected %s", result, err, expected) + } +} diff --git a/internal/cli/portable_safety_unix_test.go b/internal/cli/portable_safety_unix_test.go new file mode 100644 index 00000000..e4af9c4d --- /dev/null +++ b/internal/cli/portable_safety_unix_test.go @@ -0,0 +1,350 @@ +//go:build !windows + +package cli + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestPortableConfigScopeIsolationAndRefusal(t *testing.T) { + fixture := newPortableRefreshFixture(t, true) + if _, err := fixture.refresh(t); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + realGit, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + marker := filepath.Join(dir, "helper-ran") + helper := filepath.Join(dir, "helper") + if err := os.WriteFile(helper, []byte("#!/bin/sh\ntouch \"$GITCRAWL_TEST_MARKER\"\nexit 99\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_TEST_MARKER", marker) + t.Setenv("GITCRAWL_TEST_REAL_GIT", realGit) + // An operator-selected wrapper supplies a synthetic system scope without + // modifying the machine's Git configuration or reading its values. + wrapper := filepath.Join(dir, "git") + script := "#!/bin/sh\nexport GIT_CONFIG_SYSTEM=\"${GIT_CONFIG_SYSTEM:-$GITCRAWL_TEST_SYSTEM_CONFIG}\"\nexec \"$GITCRAWL_TEST_REAL_GIT\" \"$@\"\n" + if err := os.WriteFile(wrapper, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_PORTABLE_GIT", wrapper) + t.Setenv("HOME", dir) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "xdg")) + for _, scope := range []string{"global", "system"} { + t.Run(scope, func(t *testing.T) { + configPath := filepath.Join(dir, ".gitconfig") + if scope == "system" { + configPath = filepath.Join(dir, "system.config") + t.Setenv("GITCRAWL_TEST_SYSTEM_CONFIG", configPath) + t.Setenv("GIT_CONFIG_SYSTEM", "") + t.Setenv("GIT_CONFIG_NOSYSTEM", "") + } else { + t.Setenv("GIT_CONFIG_GLOBAL", "") + } + portableTestGit(t, fixture.checkout, "config", "--file", configPath, "filter.fixture.smudge", helper) + before := portableTestSnapshot(t, filepath.Dir(fixture.configPath)) + result, err := fixture.refresh(t) + if err == nil || result.Stage != "admission" || result.Reason != "unsupported portable Git configuration ("+scope+" scope: filters)" { + t.Fatalf("synthetic %s scope not diagnosed: %+v %v", scope, result, err) + } + if !bytes.Equal(before, portableTestSnapshot(t, filepath.Dir(fixture.configPath))) { + t.Fatal("config refusal mutated subscriber") + } + t.Setenv("GIT_CONFIG_"+strings.ToUpper(scope), os.DevNull) + if result, err := fixture.refresh(t); err != nil || result.MirrorResult != "unchanged" { + t.Fatalf("scope isolation was discarded: %+v %v", result, err) + } + }) + } + // These settings are overridden for every portable operation. Actual hook + // files, attributes, filters and redirection remain separate refusals. + for _, key := range []string{"core.hooksPath", "core.fsmonitor", "core.attributesFile", "core.sshCommand"} { + portableTestGit(t, fixture.checkout, "config", key, helper) + } + if _, err := fixture.refresh(t); err != nil { + t.Fatalf("neutralized config refused: %v", err) + } + for _, key := range []string{"filter.fixture.smudge", "url.https://example.invalid/private-subsection.insteadOf", "remote.origin.uploadpack"} { + t.Run(key, func(t *testing.T) { + portableTestGit(t, fixture.checkout, "config", key, helper) + before := portableTestSnapshot(t, filepath.Dir(fixture.configPath)) + result, err := fixture.refresh(t) + if err == nil || result.Stage != "admission" || !strings.Contains(result.Reason, "local scope") || strings.Contains(result.Reason, "private-subsection") || strings.Contains(result.Reason, helper) { + t.Fatalf("unsafe/unsanitized local config admission: %+v %v", result, err) + } + if !bytes.Equal(before, portableTestSnapshot(t, filepath.Dir(fixture.configPath))) { + t.Fatal("local config refusal mutated subscriber") + } + portableTestGit(t, fixture.checkout, "config", "--unset", key) + }) + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatalf("Git executed a neutralized or rejected helper: %v", err) + } +} + +func TestPortableGitFailureDiagnostics(t *testing.T) { + dir := t.TempDir() + git := filepath.Join(dir, "git") + if err := os.WriteFile(git, []byte("#!/bin/sh\nprintf '%s' \"$GITCRAWL_TEST_DIAGNOSTIC\" >&2\nexit 128\n"), 0o700); err != nil { + t.Fatal(err) + } + ctx, err := portableGitContext(context.Background(), git) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, message, want string }{ + {"missing-remote", "fatal: synthetic-private-path does not appear to be a git repository", "remote repository unavailable; verify the remote path and read access"}, + {"missing-local", "fatal: repository 'synthetic-private-path' does not exist", "remote repository unavailable"}, + {"not-found", "remote: Repository not found. synthetic-private-path", "remote repository unavailable"}, + {"authentication", "fatal: Authentication failed for synthetic-private-path", "check the credential helper or SSH identity"}, + {"ssh", "synthetic-private-path: Permission denied (publickey).", "Git authentication failed"}, + {"prompt", "fatal: could not read Username for synthetic-private-path: terminal prompts disabled", "Git authentication failed"}, + {"disk", "fatal: cannot write synthetic-private-path: No space left on device", "restore free-space headroom before retrying"}, + {"dns", "fatal: unable to access synthetic-private-path: Could not resolve host", "check network connectivity and remote availability"}, + {"connection", "Failed to connect to synthetic-private-path port 443", "Git connection failed"}, + {"index-lock", "fatal: Unable to create synthetic-private-path/index.lock: File exists", "index.lock file exists"}, + {"dirty", "Your local changes to synthetic-private-path would be overwritten by merge", "Your local changes would be overwritten by merge"}, + {"unknown", "unrecognized synthetic-private-path diagnostic", "verify Git version, repository state and remote access"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GITCRAWL_TEST_DIAGNOSTIC", tc.message) + err := runPortableGit(ctx, dir, io.Discard, "fetch", "origin") + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 128 || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected safe classification %q with exit 128, got %v", tc.want, err) + } + if strings.Contains(err.Error(), "synthetic-private-path") { + t.Fatal("Git failure exposed raw diagnostics") + } + }) + } +} + +func TestPortableGitCancellationAllowsOwnedCleanup(t *testing.T) { + dir := t.TempDir() + owned := filepath.Join(dir, "tmp_pack_owned") + unknown := filepath.Join(dir, "tmp_pack_preexisting") + ready := filepath.Join(dir, "ready") + childPID := filepath.Join(dir, "child.pid") + if err := os.WriteFile(unknown, []byte("preserve"), 0o600); err != nil { + t.Fatal(err) + } + script := `#!/bin/sh +trap 'wait; exit 143' TERM +sh -c 'trap '\''rm -f "$GITCRAWL_TEST_OWNED"; exit 0'\'' TERM + echo $$ > "$GITCRAWL_TEST_CHILD_PID" + echo owned > "$GITCRAWL_TEST_OWNED" + echo ready > "$GITCRAWL_TEST_READY" + sleep 30 & wait' & +wait +` + git := filepath.Join(dir, "git") + if err := os.WriteFile(git, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_TEST_OWNED", owned) + t.Setenv("GITCRAWL_TEST_CHILD_PID", childPID) + t.Setenv("GITCRAWL_TEST_READY", ready) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx, err := portableGitContext(ctx, git) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- runPortableGit(ctx, dir, io.Discard, "fetch") }() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } + if time.Now().After(deadline) { + cancel() + <-done + t.Fatal("child not ready") + } + time.Sleep(10 * time.Millisecond) + } + started := time.Now() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation: %v", err) + } + if time.Since(started) > 2*time.Second { + t.Fatal("cleanup exceeded bound") + } + if _, err := os.Stat(owned); !os.IsNotExist(err) { + t.Fatalf("owned child did not clean its pack: %v", err) + } + if data, err := os.ReadFile(unknown); err != nil || string(data) != "preserve" { + t.Fatal("unknown file changed") + } + data, err := os.ReadFile(childPID) + if err != nil { + t.Fatal(err) + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + t.Fatal(err) + } + if err := syscall.Kill(pid, 0); err == nil { + t.Fatal("owned child remains alive") + } +} + +func TestPortableGitGrowthCancelsOnlyOwnedProcess(t *testing.T) { + dir := t.TempDir() + git := filepath.Join(t.TempDir(), "git") + if err := os.WriteFile(git, []byte("#!/bin/sh\ntrap 'exit 143' TERM\ndd if=/dev/zero of=owned-growth bs=1024 count=64 2>/dev/null\nsleep 30 & wait\n"), 0o755); err != nil { + t.Fatal(err) + } + ctx, err := portableGitContext(context.Background(), git) + if err != nil { + t.Fatal(err) + } + budget, err := newPortableBudget(ctx, []string{dir}, 1, 32<<10) + if err != nil { + t.Fatal(err) + } + ctx, stop := budget.monitor(ctx) + defer stop() + started := time.Now() + err = runPortableGit(ctx, dir, io.Discard, "fetch") + if err == nil || !strings.Contains(err.Error(), "growth budget") { + t.Fatalf("growth: %v", err) + } + if time.Since(started) > 3*time.Second { + t.Fatal("growth cancellation exceeded bound") + } + if budget.snapshot().PeakGrowth < 64<<10 { + t.Fatal("missing growth observation") + } +} + +func TestPortableGitTimeoutAndForcedCleanup(t *testing.T) { + dir := t.TempDir() + git := filepath.Join(dir, "git") + if err := os.WriteFile(git, []byte("#!/bin/sh\ntrap '' TERM\nsleep 30 & wait\n"), 0o755); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + ctx, err := portableGitContext(ctx, git) + if err != nil { + t.Fatal(err) + } + started := time.Now() + err = runPortableGit(ctx, dir, io.Discard, "fetch") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("timeout: %v", err) + } + if time.Since(started) > 2*time.Second { + t.Fatal("forced cleanup exceeded bound") + } +} + +func TestPortableMaintenanceSuppressedAcrossEntryPoints(t *testing.T) { + fixture := newPortableRefreshFixture(t, false) + git, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + log := filepath.Join(t.TempDir(), "argv") + wrapper := filepath.Join(t.TempDir(), "git") + script := `#!/bin/sh +maintenance=0 +gc=0 +for arg do + [ "$arg" = 'maintenance.auto=false' ] && maintenance=1 + [ "$arg" = 'gc.auto=0' ] && gc=1 +done +[ "$maintenance$gc" = 11 ] || exit 91 +printf '%s\n' "$@" >> "$GITCRAWL_TEST_GIT_LOG" +printf 'END\n' >> "$GITCRAWL_TEST_GIT_LOG" +exec "$GITCRAWL_TEST_REAL_GIT" "$@" +` + if err := os.WriteFile(wrapper, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_TEST_REAL_GIT", git) + t.Setenv("GITCRAWL_TEST_GIT_LOG", log) + t.Setenv("GITCRAWL_PORTABLE_GIT", wrapper) + fixture.advance(t, true) + if _, err := fixture.refresh(t, "--git", wrapper); err != nil { + t.Fatal(err) + } + fixture.init(t) + if err := os.WriteFile(filepath.Join(fixture.checkout, fixture.relative)+".gz", []byte("dirty"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := syncPortableStore(context.Background(), fixture.remote, fixture.checkout); err != nil { + t.Fatal(err) + } + if err := refreshPortableStoreForDB(context.Background(), filepath.Join(fixture.checkout, fixture.relative)); err != nil { + t.Fatal(err) + } + if _, err := recloneMalformedPortableStoreForDB(context.Background(), filepath.Join(fixture.checkout, fixture.relative), fixture.configPath); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(log) + if err != nil { + t.Fatal(err) + } + for _, command := range []string{"fetch", "merge", "clone", "reset"} { + if !bytes.Contains(data, []byte("\n"+command+"\n")) { + t.Fatalf("entry point not exercised: %s", command) + } + } + for _, call := range strings.Split(string(data), "END\n") { + if strings.Contains(call, "\nfetch\n") && !strings.Contains(call, "\n--no-auto-maintenance\n") { + t.Fatal("fetch omitted maintenance suppression") + } + } +} + +func TestPortableRefreshReportsPartialRefFailure(t *testing.T) { + fixture := newPortableRefreshFixture(t, false) + initial, err := fixture.refresh(t) + if err != nil { + t.Fatal(err) + } + beforeMirror, _ := fileSHA256(fixture.mirror) + fixture.advance(t, true) + git, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + wrapper := filepath.Join(t.TempDir(), "git") + if err := os.WriteFile(wrapper, []byte("#!/bin/sh\nfor arg do\n[ \"$arg\" = update-ref ] && exit 42\ndone\nexec \"$GITCRAWL_TEST_REAL_GIT\" \"$@\"\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GITCRAWL_TEST_REAL_GIT", git) + result, err := fixture.refresh(t, "--git", wrapper) + if err == nil || result.Result != "partial" || result.AfterCommit != result.TargetCommit || result.AfterCommit == result.BeforeCommit { + t.Fatalf("partial advancement: %+v %v", result, err) + } + if portableTestGit(t, fixture.checkout, "rev-parse", "HEAD") != result.TargetCommit || portableTestGit(t, fixture.checkout, "rev-parse", "refs/remotes/origin/main") != initial.AfterCommit { + t.Fatal("partial result does not match actual refs") + } + if after, _ := fileSHA256(fixture.mirror); beforeMirror != after { + t.Fatal("partial ref failure replaced last-good mirror") + } + if _, err := fixture.refresh(t); err != nil { + t.Fatal("safe retry after partial result", err) + } +} diff --git a/internal/cli/portable_transition_test.go b/internal/cli/portable_transition_test.go new file mode 100644 index 00000000..32359229 --- /dev/null +++ b/internal/cli/portable_transition_test.go @@ -0,0 +1,308 @@ +package cli + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/openclaw/crawlkit/control" + "github.com/openclaw/gitcrawl/internal/store" +) + +func newPortableTransitionFixture(t *testing.T, compressed bool) (portableRefreshFixture, string) { + t.Helper() + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + dir := t.TempDir() + publisher := filepath.Join(dir, "publisher") + if err := os.Mkdir(publisher, 0o755); err != nil { + t.Fatal(err) + } + portableTestGit(t, publisher, "init", "-b", "main") + seedPortableThread(t, filepath.Join(publisher, "archive.db"), 1, "original raw generation") + if compressed { + portableTestManifest(t, filepath.Join(publisher, "archive.db"), "archive.db", true) + } + portableTestCommit(t, publisher) + remotePath := filepath.ToSlash(publisher) + if filepath.VolumeName(publisher) != "" { + remotePath = "/" + remotePath + } + fixture := portableRefreshFixture{ + remote: (&url.URL{Scheme: "file", Path: remotePath}).String(), + checkout: filepath.Join(dir, "subscriber"), configPath: filepath.Join(dir, "config.toml"), relative: "archive.db", + mirror: filepath.Join(dir, "runtime", "subscriber", "archive.db"), + } + fixture.init(t) + return fixture, publisher +} + +func portableTransitionThreads(t *testing.T, fixture portableRefreshFixture, want string) { + t.Helper() + var output struct { + Threads []store.Thread `json:"threads"` + } + if err := json.Unmarshal(fixture.command(t, "threads", "openclaw/openclaw", "--json"), &output); err != nil { + t.Fatal(err) + } + if len(output.Threads) != 1 || output.Threads[0].Title != want { + t.Fatalf("runtime threads = %+v, want %q", output.Threads, want) + } +} + +func publishPortableTransition(t *testing.T, publisher, format string) [32]byte { + t.Helper() + path := filepath.Join(publisher, "archive.db") + if _, err := os.Stat(path); os.IsNotExist(err) { + temp, err := stagePortableSQLiteSourceTemp(path, path, 0o600) + if err != nil { + t.Fatal(err) + } + if err := os.Rename(temp, path); err != nil { + t.Fatal(err) + } + } + seedPortableThread(t, path, 1, "replacement generation") + digest := portableTestDigest(t, path) + if format != "raw" { + portableTestManifest(t, path, "archive.db", format == "gzip") + } + portableTestCommit(t, publisher) + return digest +} + +func assertPortableReplica(t *testing.T, fixture portableRefreshFixture, digest [32]byte) { + t.Helper() + if portableTestDigest(t, fixture.mirror) != digest { + t.Fatal("runtime bytes differ from the published generation") + } + state := readPortableStoreRefreshState(portableStoreRefreshStatePath(fixture.mirror)) + if state.MirrorWritable || state.MirrorHealthSourceSHA256 != fmt.Sprintf("%x", digest) { + t.Fatalf("replica lost its source identity: %+v", state) + } + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + if _, err := os.Lstat(fixture.mirror + suffix); !os.IsNotExist(err) { + t.Fatalf("replica read created %s: %v", suffix, err) + } + } +} + +func TestPortableReplicaTransitions(t *testing.T) { + cases := []struct { + name, format, update string + compressed bool + }{ + {"raw-to-gzip-init", "gzip", "init", false}, + {"raw-to-raw-init", "raw", "init", false}, + {"raw-to-manifest-init", "manifest", "init", false}, + {"gzip-to-gzip-init", "gzip", "init", true}, + {"gzip-to-manifest-init", "manifest", "init", true}, + {"raw-to-gzip-refresh", "gzip", "refresh", false}, + {"raw-to-manifest-refresh", "manifest", "refresh", false}, + {"raw-to-gzip-read", "gzip", "read", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, prepared := range []bool{true, false} { + t.Run(fmt.Sprintf("prepared=%t", prepared), func(t *testing.T) { + fixture, publisher := newPortableTransitionFixture(t, tc.compressed) + if prepared { + for range 2 { + portableTransitionThreads(t, fixture, "original raw generation") + } + if !tc.compressed { + assertPortableReplica(t, fixture, portableTestDigest(t, filepath.Join(publisher, fixture.relative))) + } + } else if _, err := os.Stat(fixture.mirror); !os.IsNotExist(err) { + t.Fatalf("init unexpectedly prepared runtime: %v", err) + } + before := portableTestGit(t, fixture.checkout, "rev-parse", "HEAD") + digest := publishPortableTransition(t, publisher, tc.format) + switch tc.update { + case "init": + var result struct { + Action string `json:"portable_store"` + } + data := fixture.command(t, "init", "--portable-store", fixture.remote, "--store-dir", fixture.checkout, "--portable-db", fixture.relative, "--json") + if err := json.Unmarshal(data, &result); err != nil || result.Action != "pulled" { + t.Fatalf("re-init: %s %v", data, err) + } + case "refresh": + result, err := fixture.refresh(t) + if err != nil || result.Result != "updated" || result.MirrorResult != "promoted" { + t.Fatalf("refresh: %+v %v", result, err) + } + case "read": + t.Setenv("GITCRAWL_PORTABLE_REFRESH_TTL", "0") + } + for range 2 { + portableTransitionThreads(t, fixture, "replacement generation") + assertPortableReplica(t, fixture, digest) + } + if got := portableTestGit(t, fixture.checkout, "rev-parse", "HEAD"); got == before || got != portableTestGit(t, publisher, "rev-parse", "HEAD") { + t.Fatal("subscriber did not advance to publisher HEAD") + } + if tc.format == "gzip" { + if _, err := os.Stat(filepath.Join(fixture.checkout, fixture.relative)); !os.IsNotExist(err) { + t.Fatalf("gzip transition retained raw artifact: %v", err) + } + } + }) + } + }) + } +} + +func TestPortableRawLocalWorkTransitions(t *testing.T) { + for _, format := range []string{"raw", "gzip"} { + t.Run(format, func(t *testing.T) { + for _, ownership := range []string{"close-thread", "external-write"} { + t.Run(ownership, func(t *testing.T) { + fixture, publisher := newPortableTransitionFixture(t, false) + portableTransitionThreads(t, fixture, "original raw generation") + baseline := readPortableStoreRefreshState(portableStoreRefreshStatePath(fixture.mirror)) + if ownership == "close-thread" { + fixture.command(t, "close-thread", "openclaw/openclaw", "--number", "1", "--reason", "preserve raw closure", "--json") + seedPortableThread(t, fixture.mirror, 3, "local extra thread") + } else { + info, err := os.Stat(fixture.mirror) + if err != nil { + t.Fatal(err) + } + seedPortableThread(t, fixture.mirror, 1, "external raw generation") + if fileSize(fixture.mirror) != info.Size() { + t.Fatal("external rewrite must preserve file size") + } + if err := os.Chtimes(fixture.mirror, info.ModTime(), info.ModTime()); err != nil { + t.Fatal(err) + } + } + readClosure := func() (string, string) { + t.Helper() + st, err := store.OpenReadOnly(context.Background(), fixture.mirror) + if err != nil { + t.Fatal(err) + } + defer st.Close() + var at, reason sql.NullString + if err := st.DB().QueryRow(`select closed_at_local, close_reason_local from threads where number=1`).Scan(&at, &reason); err != nil { + t.Fatal(err) + } + return at.String, reason.String + } + var closedAt, reason string + if ownership == "close-thread" { + closedAt, reason = readClosure() + if closedAt == "" || reason != "preserve raw closure" { + t.Fatal("close-thread did not create a closure") + } + } + local := portableTestDigest(t, fixture.mirror) + publishPortableTransition(t, publisher, format) + fixture.init(t) + for range 2 { + want := "external raw generation" + if ownership == "close-thread" { + want = "local extra thread" + } + portableTransitionThreads(t, fixture, want) + if portableTestDigest(t, fixture.mirror) != local { + t.Fatal("source transition or ordinary read changed local runtime bytes") + } + if ownership == "close-thread" { + if at, why := readClosure(); at != closedAt || why != reason { + t.Fatal("source transition changed raw local closure") + } + } + } + state := readPortableStoreRefreshState(portableStoreRefreshStatePath(fixture.mirror)) + if !state.MirrorWritable || state.MirrorHealthSourceSHA256 != baseline.MirrorHealthSourceSHA256 { + t.Fatalf("local work lost ownership or original source identity: %+v", state) + } + }) + } + }) + } +} + +func TestPortableRawReplicaStatusDoesNotClaimOwnership(t *testing.T) { + fixture, publisher := newPortableTransitionFixture(t, false) + portableTransitionThreads(t, fixture, "original raw generation") + digest := portableTestDigest(t, filepath.Join(publisher, fixture.relative)) + before := portableTestSnapshot(t, filepath.Dir(fixture.configPath)) + var result control.Status + if err := json.Unmarshal(fixture.command(t, "status", "--json"), &result); err != nil { + t.Fatal(err) + } + if result.State == "stale" || countValue(result.Counts, "threads") != 1 || !sameExistingPath(result.DatabasePath, fixture.mirror) || result.DatabaseBytes != fileSize(fixture.mirror) { + t.Fatalf("incorrect raw runtime status: %+v", result) + } + if string(before) != string(portableTestSnapshot(t, filepath.Dir(fixture.configPath))) { + t.Fatal("status changed raw subscriber state") + } + assertPortableReplica(t, fixture, digest) +} + +func TestPortableRawReplicaHealthDoesNotInventIdentity(t *testing.T) { + fixture, publisher := newPortableTransitionFixture(t, false) + portableTransitionThreads(t, fixture, "original raw generation") + statePath := portableStoreRefreshStatePath(fixture.mirror) + state := readPortableStoreRefreshState(statePath) + state.MirrorHealthSourceSHA256 = "" + if err := writePortableStoreRefreshState(statePath, state); err != nil { + t.Fatal(err) + } + if err := portableMirrorCachedHealth(context.Background(), fixture.mirror, filepath.Join(fixture.checkout, fixture.relative), statePath); err != nil { + t.Fatal(err) + } + if got := readPortableStoreRefreshState(statePath).MirrorHealthSourceSHA256; got != "" { + t.Fatalf("health-only check invented raw replica identity: %s", got) + } + publishPortableTransition(t, publisher, "gzip") + result, err := fixture.refresh(t) + if err != nil || result.MirrorResult != "preserved-local" { + t.Fatalf("unproven legacy runtime was not preserved: %+v %v", result, err) + } + for range 2 { + portableTransitionThreads(t, fixture, "original raw generation") + } + if got := readPortableStoreRefreshState(statePath).MirrorHealthSourceSHA256; got != "" { + t.Fatalf("preservation invented a legacy source identity: %s", got) + } +} + +func TestPortableRawWALSurvivesTransition(t *testing.T) { + fixture, publisher := newPortableTransitionFixture(t, false) + portableTransitionThreads(t, fixture, "original raw generation") + st, err := store.Open(context.Background(), fixture.mirror) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, err := st.DB().Exec(`update threads set title='uncheckpointed local work' where number=1`); err != nil { + t.Fatal(err) + } + wal := fixture.mirror + "-wal" + if fileSize(wal) == 0 { + t.Fatal("fixture did not create a live WAL") + } + beforeDB, beforeWAL := portableTestDigest(t, fixture.mirror), portableTestDigest(t, wal) + publishPortableTransition(t, publisher, "gzip") + fixture.init(t) + result, err := fixture.refresh(t) + if err != nil || result.MirrorResult != "preserved-local" { + t.Fatalf("live WAL was not preserved: %+v %v", result, err) + } + for range 2 { + portableTransitionThreads(t, fixture, "uncheckpointed local work") + if portableTestDigest(t, fixture.mirror) != beforeDB || portableTestDigest(t, wal) != beforeWAL { + t.Fatal("source transition changed the local database or live WAL") + } + } +} diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 8272186b..8526a209 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -56,6 +56,11 @@ const staleGitIndexLockAge = 2 * time.Second var errPortableStoreDirty = errors.New("portable store checkout has local changes") func (a *App) openLocalRuntime(ctx context.Context) (localRuntime, error) { + if session, ok := ctx.Value(portableCommandKey{}).(*portableCommandSession); ok { + session.mu.Lock() + session.retain = true + session.mu.Unlock() + } cfg, err := config.LoadRuntime(a.configPath) if err != nil { return localRuntime{}, err @@ -74,6 +79,14 @@ func (a *App) openLocalRuntime(ctx context.Context) (localRuntime, error) { } cfg.DBPath = mirrorPath remoteSource = true + // Writable opens can migrate schema as well as change user data. Record + // ownership before either can happen, without changing source identity. + statePath := portableStoreRefreshStatePath(mirrorPath) + state := readPortableStoreRefreshState(statePath) + state.MirrorWritable = true + if err := writePortableStoreRefreshState(statePath, state); err != nil { + return localRuntime{}, err + } a.dbTargetNoticeOnce.Do(func() { fmt.Fprintf(a.Stderr, "gitcrawl: portable store checkout detected; writes go to the runtime mirror at %s, not the checkout database %s. Run 'gitcrawl portable prune' to publish.\n", mirrorPath, sourceDBPath) }) @@ -99,9 +112,15 @@ func (a *App) openLocalRuntimeReadOnlyWithConfig(ctx context.Context, cfg config } sourceDBPath := cfg.DBPath remoteSource := false - if _, ok, err := portableStoreRoot(ctx, cfg.DBPath); err != nil { + if root, ok, err := portableStoreRoot(ctx, cfg.DBPath); err != nil { return localRuntime{}, err } else if ok { + var release func() + ctx, release, err = acquirePortableOwner(ctx, root) + if err != nil { + return localRuntime{}, err + } + defer release() mirrorPath, _, err := a.ensurePortableRuntimeDB(ctx, cfg.DBPath, true) if err != nil { return localRuntime{}, err @@ -109,7 +128,13 @@ func (a *App) openLocalRuntimeReadOnlyWithConfig(ctx context.Context, cfg config cfg.DBPath = mirrorPath remoteSource = true } - st, err := store.OpenReadOnly(ctx, cfg.DBPath) + open := store.OpenReadOnly + if remoteSource { + open = func(ctx context.Context, path string) (*store.Store, error) { + return openPortableMirrorReadOnly(ctx, path, sourceDBPath) + } + } + st, err := open(ctx, cfg.DBPath) if err != nil { return localRuntime{}, err } @@ -139,6 +164,12 @@ func refreshPortableStoreForDB(ctx context.Context, dbPath string) error { if !ok { return nil } + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return err + } + defer release() + root = ctx.Value(portableOwnerKey{}).(*portableOwner).root clean := gitWorktreeClean(ctx, root) if !clean { removed, _ := removeStaleGitIndexLock(ctx, root, staleGitIndexLockAge) @@ -173,6 +204,12 @@ func repairMalformedPortableStoreForDB(ctx context.Context, dbPath, configPath s if !ok { return result, nil } + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return result, err + } + defer release() + root = ctx.Value(portableOwnerKey{}).(*portableOwner).root if !portableStoreRepairAllowed(root, configPath) { return result, fmt.Errorf("refuse destructive repair for unmarked portable store checkout %s", root) } @@ -207,6 +244,12 @@ func recloneMalformedPortableStoreForDB(ctx context.Context, dbPath, configPath if !ok { return result, nil } + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return result, err + } + defer release() + root = ctx.Value(portableOwnerKey{}).(*portableOwner).root if !portableStoreRepairAllowed(root, configPath) { return result, fmt.Errorf("refuse reclone for unmarked portable store checkout %s", root) } @@ -230,7 +273,7 @@ func recloneMalformedPortableStoreForDB(ctx context.Context, dbPath, configPath if strings.TrimSpace(branch) != "" { cloneArgs = append(cloneArgs, "--branch", branch) } - cloneArgs = append(cloneArgs, remote, root) + cloneArgs = append(cloneArgs, "--", remote, root) if err := runGit(cloneCtx, "", cloneArgs...); err != nil { _ = os.RemoveAll(root) _ = os.Rename(backupPath, root) @@ -275,17 +318,54 @@ func (a *App) portableRuntimeDBPath(ctx context.Context, sourceDBPath string) (s func refreshPortableRuntimeDB(ctx context.Context, sourceDBPath, mirrorPath string, refresh bool, configPath string) (bool, error) { portableRuntimeMu.Lock() defer portableRuntimeMu.Unlock() - sweepOrphanPortableRuntimeTempFiles(mirrorPath, portableRuntimeTempMaxAge) - _, isPortableSource, err := portableStoreRoot(ctx, sourceDBPath) + root, isPortableSource, err := portableStoreRoot(ctx, sourceDBPath) if err != nil { return false, err } + if isPortableSource { + var release func() + ctx, release, err = acquirePortableOwner(ctx, root) + if err != nil { + return false, err + } + defer release() + } + statePath := portableStoreRefreshStatePath(mirrorPath) + state := readPortableStoreRefreshState(statePath) + local, err := portableRuntimeHasLocalChanges(ctx, sourceDBPath, mirrorPath, state) + if err != nil { + return false, err + } + if local { + if err := sqliteStoreHealth(ctx, mirrorPath); err == nil { + // Keep the original source digest/stamp: healthy local work does + // not become a copy of the publisher's latest generation. + changed := !state.MirrorWritable + if state.MirrorHealthSourceSHA256 == "" { + modTime, size, sha, stampErr := portableDBManifestStamp(sourceDBPath) + if stampErr == nil && modTime != "" && portableManifestGenerationUnchanged(state, modTime, size, sha) { + state.MirrorHealthSourceSHA256 = sha + changed = true + } + } + if changed { + state.MirrorWritable = true + if err := writePortableStoreRefreshState(statePath, state); err != nil { + return false, err + } + } + return false, nil + } else if state.MirrorWritable || !isSQLiteCorruption(err) { + return false, fmt.Errorf("check locally modified portable runtime (preserved): %w", err) + } + // A corrupt, never-writable replica still follows normal recovery. + } + sweepOrphanPortableRuntimeTempFiles(mirrorPath, portableRuntimeTempMaxAge) isRepairablePortableSource := isPortableSource if refresh { _ = refreshPortableStoreForDBIfDue(ctx, sourceDBPath, mirrorPath) } needsCopy, err := portableRuntimeNeedsCopy(sourceDBPath, mirrorPath) - statePath := portableStoreRefreshStatePath(mirrorPath) if err != nil { if !isRepairablePortableSource || !errors.Is(err, os.ErrNotExist) { return false, err @@ -366,11 +446,14 @@ func refreshPortableRuntimeDB(ctx context.Context, sourceDBPath, mirrorPath stri if !needsCopy { return false, nil } - if err := copySQLiteFileAtomicVerified(ctx, sourceDBPath, mirrorPath); err != nil { + digest, err := copySQLiteFileAtomicVerified(ctx, sourceDBPath, mirrorPath) + if err != nil { return false, err } if isRepairablePortableSource { - _ = markPortableMirrorHealthVerified(mirrorPath, statePath, sourceDBPath) + if err := markPortableMirrorHealthVerified(mirrorPath, statePath, sourceDBPath, fmt.Sprintf("%x", digest)); err != nil { + return false, err + } } return true, nil } @@ -385,6 +468,7 @@ type portableStoreRefreshState struct { MirrorHealthManifestModTime string `json:"mirror_health_manifest_mod_time,omitempty"` MirrorHealthManifestSize int64 `json:"mirror_health_manifest_size,omitempty"` MirrorHealthSourceSHA256 string `json:"mirror_health_source_sha256,omitempty"` + MirrorWritable bool `json:"mirror_writable,omitempty"` LastRepair string `json:"last_repair,omitempty"` LastRepairBackup string `json:"last_repair_backup,omitempty"` LastRepairAt string `json:"last_repair_at,omitempty"` @@ -392,6 +476,40 @@ type portableStoreRefreshState struct { LastRecloneAttempt string `json:"last_reclone_attempt,omitempty"` } +func portableRuntimeHasLocalChanges(ctx context.Context, source, path string, state portableStoreRefreshState) (bool, error) { + info, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if state.MirrorWritable { + return true, nil + } + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + if _, err := os.Stat(path + suffix); err == nil { + return true, nil + } else if !errors.Is(err, os.ErrNotExist) { + return false, err + } + } + if state.MirrorHealthSourceSHA256 == "" { + return false, nil + } + // A legacy health stamp can describe local bytes, not a pristine replica. + // Recheck the digest whenever the source or runtime may have changed. + if state.MirrorHealthSize == info.Size() && state.MirrorHealthModTime == info.ModTime().UTC().Format(time.RFC3339Nano) { + modTime, size, sha, err := portableDBManifestStamp(source) + needsCopy, copyErr := portableRuntimeNeedsCopy(source, path) + if err == nil && copyErr == nil && !needsCopy && portableManifestGenerationUnchanged(state, modTime, size, sha) { + return false, nil + } + } + digest, err := portableFileSHA256(ctx, path) + return !strings.EqualFold(fmt.Sprintf("%x", digest), state.MirrorHealthSourceSHA256), err +} + func recoverMissingPortableSource(ctx context.Context, sourceDBPath, configPath, statePath string) error { repair, err := repairMalformedPortableStoreForDB(ctx, sourceDBPath, configPath) recordPortableRepairState(statePath, repair, err) @@ -421,6 +539,15 @@ func recoverMissingPortableSource(ctx context.Context, sourceDBPath, configPath, } func refreshPortableStoreForDBIfDue(ctx context.Context, sourceDBPath, mirrorPath string) error { + root, ok, err := portableStoreRoot(ctx, sourceDBPath) + if err != nil || !ok { + return err + } + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return err + } + defer release() ttl := portableStoreRefreshInterval() statePath := portableStoreRefreshStatePath(mirrorPath) state := readPortableStoreRefreshState(statePath) @@ -431,26 +558,16 @@ func refreshPortableStoreForDBIfDue(ctx context.Context, sourceDBPath, mirrorPat if ttl > 0 && recentPortableRefresh(state.LastFailure, now, portableStoreRefreshFailureBackoff) { return nil } - lockPath := statePath + ".lock" if err := os.MkdirAll(filepath.Dir(statePath), 0o755); err != nil { return err } - removeStalePortableRefreshLock(lockPath, now) - lock, locked := tryGHCommandCacheLock(lockPath) - if !locked { - return nil - } - defer func() { - _ = lock.Close() - _ = os.Remove(lockPath) - }() state = readPortableStoreRefreshState(statePath) now = time.Now().UTC() if ttl > 0 && recentPortableRefresh(state.LastSuccess, now, ttl) { return nil } state.LastAttempt = now.Format(time.RFC3339Nano) - err := refreshPortableStoreForDB(ctx, sourceDBPath) + err = refreshPortableStoreForDB(ctx, sourceDBPath) if err != nil { state.LastFailure = time.Now().UTC().Format(time.RFC3339Nano) state.Error = err.Error() @@ -463,17 +580,6 @@ func refreshPortableStoreForDBIfDue(ctx context.Context, sourceDBPath, mirrorPat return writePortableStoreRefreshState(statePath, state) } -func removeStalePortableRefreshLock(path string, now time.Time) { - info, err := os.Stat(path) - if err != nil { - return - } - if now.Sub(info.ModTime()) <= 2*portableStoreRefreshTimeout { - return - } - _ = os.Remove(path) -} - func portableStoreRefreshInterval() time.Duration { if raw := strings.TrimSpace(os.Getenv("GITCRAWL_PORTABLE_REFRESH_TTL")); raw != "" { if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 { @@ -534,20 +640,6 @@ func recordPortableRepairState(path string, result portableRepairResult, repairE _ = writePortableStoreRefreshState(path, state) } -func sqliteStoreOpenHealth(ctx context.Context, path string) error { - if strings.TrimSpace(path) == "" { - return os.ErrNotExist - } - if _, err := os.Stat(path); err != nil { - return err - } - st, err := store.OpenReadOnly(ctx, path) - if err != nil { - return err - } - return st.Close() -} - func portableMirrorCachedHealth(ctx context.Context, mirrorPath, sourceDBPath, statePath string) error { manifestModTime, manifestSize, sourceSHA256, err := portableDBManifestStamp(sourceDBPath) if err != nil { @@ -560,6 +652,9 @@ func portableMirrorCachedHealth(ctx context.Context, mirrorPath, sourceDBPath, s } func sqliteStoreCachedHealthWithManifest(ctx context.Context, path, sourceDBPath, statePath, manifestModTime string, manifestSize int64, sourceSHA256 string) error { + open := func(ctx context.Context, path string) (*store.Store, error) { + return openPortableMirrorReadOnly(ctx, path, sourceDBPath) + } return sqliteStoreCachedHealthWithManifestChecks( ctx, path, @@ -568,8 +663,16 @@ func sqliteStoreCachedHealthWithManifest(ctx context.Context, path, sourceDBPath manifestModTime, manifestSize, sourceSHA256, - sqliteStoreOpenHealth, - sqliteStoreHealth, + func(ctx context.Context, path string) error { + st, err := open(ctx, path) + if err != nil { + return err + } + return st.Close() + }, + func(ctx context.Context, path string) error { + return sqliteStoreHealthWithOpen(ctx, path, open) + }, ) } @@ -599,7 +702,10 @@ func sqliteStoreCachedHealthWithManifestChecks(ctx context.Context, path, source } return markSQLiteStoreHealthVerifiedWithManifest(path, statePath, manifestModTime, manifestSize, sourceSHA256) } - if err := validatePortableSQLiteFile(ctx, path, sourceDBPath); err != nil { + if err := fullHealthCheck(ctx, path); err != nil { + return err + } + if err := validatePortableDBManifest(ctx, path, portableDBManifestPath(sourceDBPath)); err != nil { return err } return markSQLiteStoreHealthVerifiedWithManifest(path, statePath, manifestModTime, manifestSize, sourceSHA256) @@ -615,12 +721,24 @@ func portableManifestGenerationUnchanged(state portableStoreRefreshState, manife state.MirrorHealthManifestModTime == manifestModTime } -func markPortableMirrorHealthVerified(path, statePath, sourceDBPath string) error { +func markPortableMirrorHealthVerified(path, statePath, sourceDBPath, replicaSHA256 string) error { manifestModTime, manifestSize, sourceSHA256, err := portableDBManifestStamp(sourceDBPath) if err != nil { return err } - return markSQLiteStoreHealthVerifiedWithManifest(path, statePath, manifestModTime, manifestSize, sourceSHA256) + if sourceSHA256 != "" && !strings.EqualFold(sourceSHA256, replicaSHA256) { + return fmt.Errorf("portable manifest changed after runtime validation") + } + state := readPortableStoreRefreshState(statePath) + if state.MirrorWritable { + state.MirrorWritable = false + if err := writePortableStoreRefreshState(statePath, state); err != nil { + return err + } + } + // The caller just promoted validated bytes. A raw source without a + // manifest still has a replica identity; health-only checks cannot invent it. + return markSQLiteStoreHealthVerifiedWithManifest(path, statePath, manifestModTime, manifestSize, replicaSHA256) } func markSQLiteStoreHealthVerifiedWithManifest(path, statePath, manifestModTime string, manifestSize int64, sourceSHA256 string) error { @@ -633,7 +751,9 @@ func markSQLiteStoreHealthVerifiedWithManifest(path, statePath, manifestModTime state.MirrorHealthModTime = info.ModTime().UTC().Format(time.RFC3339Nano) state.MirrorHealthManifestSize = manifestSize state.MirrorHealthManifestModTime = manifestModTime - state.MirrorHealthSourceSHA256 = sourceSHA256 + if sourceSHA256 != "" { + state.MirrorHealthSourceSHA256 = sourceSHA256 + } return writePortableStoreRefreshState(statePath, state) } @@ -740,7 +860,7 @@ func validatePortableSQLiteSourceFile(ctx context.Context, dbPath, manifestDBPat return fmt.Errorf("create portable source validation dir: %w", err) } defer os.RemoveAll(tempDir) - tempPath, err := stagePortableSQLiteSourceTemp(dbPath, filepath.Join(tempDir, filepath.Base(dbPath)), 0o600) + tempPath, err := stagePortableSQLiteSourceTempContext(ctx, dbPath, filepath.Join(tempDir, filepath.Base(dbPath)), 0o600) if err != nil { return err } @@ -781,7 +901,7 @@ func validatePortableDBManifest(ctx context.Context, dbPath, manifestPath string if manifest.OutputBytes > 0 && info.Size() != manifest.OutputBytes { return fmt.Errorf("portable manifest mismatch: size %d != %d", info.Size(), manifest.OutputBytes) } - sum, err := fileSHA256(dbPath) + sum, err := portableFileSHA256(ctx, dbPath) if err != nil { return err } @@ -804,7 +924,8 @@ func validatePortableDBManifest(ctx context.Context, dbPath, manifestPath string if artifactIDProfile != portableexport.CurrentStateSemanticV1 { return fmt.Errorf("portable manifest mismatch: unsupported artifactIdProfile %q", manifest.ArtifactIDProfile) } - computedArtifactID, err := portableexport.ComputeArtifactID(ctx, dbPath, artifactIDProfile) + tempParent, _ := ctx.Value(portableValidationDirKey{}).(string) + computedArtifactID, err := portableexport.ComputeArtifactIDInDirectory(ctx, dbPath, artifactIDProfile, tempParent) if err != nil { return fmt.Errorf("portable manifest mismatch: recompute artifactId: %w", err) } @@ -880,6 +1001,10 @@ func portableSourceArtifact(dbPath string) (string, portableDBManifest, bool, er } func validatePortableArchive(path string, manifest portableDBManifest) error { + return validatePortableArchiveContext(context.Background(), path, manifest) +} + +func validatePortableArchiveContext(ctx context.Context, path string, manifest portableDBManifest) error { info, err := os.Stat(path) if err != nil { return err @@ -891,7 +1016,7 @@ func validatePortableArchive(path string, manifest portableDBManifest) error { manifest.ArchiveBytes, ) } - sum, err := fileSHA256(path) + sum, err := portableFileSHA256(ctx, path) if err != nil { return err } @@ -1006,6 +1131,10 @@ func copyFileAtomic(sourcePath, targetPath string) error { } func stageFileCopyTemp(sourcePath, targetPath string, mode os.FileMode) (string, error) { + return stageFileCopyTempContext(context.Background(), sourcePath, targetPath, mode) +} + +func stageFileCopyTempContext(ctx context.Context, sourcePath, targetPath string, mode os.FileMode) (string, error) { if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { return "", fmt.Errorf("create portable runtime dir: %w", err) } @@ -1026,7 +1155,7 @@ func stageFileCopyTemp(sourcePath, targetPath string, mode os.FileMode) (string, removeSQLiteTempSidecars(tempPath) } }() - if _, err := io.Copy(temp, source); err != nil { + if _, err := io.Copy(temp, portableContextReader{ctx: ctx, reader: source}); err != nil { _ = temp.Close() return "", fmt.Errorf("copy portable runtime db: %w", err) } @@ -1041,10 +1170,11 @@ func stageFileCopyTemp(sourcePath, targetPath string, mode os.FileMode) (string, return tempPath, nil } -func copySQLiteFileAtomicVerified(ctx context.Context, sourcePath, targetPath string) error { - tempPath, err := stagePortableSQLiteSourceTemp(sourcePath, targetPath, 0o600) +func copySQLiteFileAtomicVerified(ctx context.Context, sourcePath, targetPath string) ([32]byte, error) { + var digest [32]byte + tempPath, err := stagePortableSQLiteSourceTempContext(ctx, sourcePath, targetPath, 0o600) if err != nil { - return err + return digest, err } cleanup := true defer func() { @@ -1054,26 +1184,34 @@ func copySQLiteFileAtomicVerified(ctx context.Context, sourcePath, targetPath st } }() if err := validatePortableSQLiteFile(ctx, tempPath, sourcePath); err != nil { - return fmt.Errorf("validate portable runtime temp db: %w", err) + return digest, fmt.Errorf("validate portable runtime temp db: %w", err) + } + digest, err = portableFileSHA256(ctx, tempPath) + if err != nil { + return digest, err } if err := os.Rename(tempPath, targetPath); err != nil { - return fmt.Errorf("replace portable runtime db: %w", err) + return digest, fmt.Errorf("replace portable runtime db: %w", err) } cleanup = false removeSQLiteTempSidecars(tempPath) removeSQLiteTempSidecars(targetPath) - return nil + return digest, nil } func stagePortableSQLiteSourceTemp(sourceDBPath, targetPath string, mode os.FileMode) (string, error) { + return stagePortableSQLiteSourceTempContext(context.Background(), sourceDBPath, targetPath, mode) +} + +func stagePortableSQLiteSourceTempContext(ctx context.Context, sourceDBPath, targetPath string, mode os.FileMode) (string, error) { sourcePath, manifest, compressed, err := portableSourceArtifact(sourceDBPath) if err != nil { return "", err } if !compressed { - return stageFileCopyTemp(sourcePath, targetPath, mode) + return stageFileCopyTempContext(ctx, sourcePath, targetPath, mode) } - if err := validatePortableArchive(sourcePath, manifest); err != nil { + if err := validatePortableArchiveContext(ctx, sourcePath, manifest); err != nil { return "", err } if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { @@ -1101,7 +1239,11 @@ func stagePortableSQLiteSourceTemp(sourceDBPath, targetPath string, mode os.File removeSQLiteTempSidecars(tempPath) } }() - written, copyErr := io.Copy(temp, io.LimitReader(reader, manifest.OutputBytes+1)) + if manifest.OutputBytes <= 0 || manifest.OutputBytes == int64(^uint64(0)>>1) { + _ = temp.Close() + return "", fmt.Errorf("portable manifest mismatch: invalid outputBytes") + } + written, copyErr := io.Copy(temp, portableContextReader{ctx: ctx, reader: io.LimitReader(reader, manifest.OutputBytes+1)}) if copyErr != nil { _ = temp.Close() return "", fmt.Errorf("inflate portable runtime db: %w", copyErr) @@ -1130,6 +1272,18 @@ func stagePortableSQLiteSourceTemp(sourceDBPath, targetPath string, mode os.File // checkout before the first rename so a staging or validation failure leaves // the previously published pair untouched. func publishPortableCheckoutPair(ctx context.Context, mirrorDBPath, mirrorManifestPath, checkoutDBPath, checkoutManifestPath string) error { + root, ok, err := portableStoreRoot(ctx, checkoutDBPath) + if err != nil { + return err + } + if ok { + var release func() + ctx, release, err = acquirePortableOwner(ctx, root) + if err != nil { + return err + } + defer release() + } mode := os.FileMode(0o644) if info, err := os.Stat(checkoutDBPath); err == nil { mode = info.Mode().Perm() @@ -1294,21 +1448,25 @@ func probePortableStoreGitWorktree(ctx context.Context, dir string) (bool, error if !initialized { return false, nil } + ctx, err = portableGitContext(ctx, "") + if err != nil { + return false, err + } - topLevel, stderr, err := runGitCommandOutputWithEnvSeparate(ctx, "", portableStoreGitProbeEnv(), "-C", dir, "rev-parse", "--show-toplevel") + topLevel, err := portableGitOutput(ctx, dir, "rev-parse", "--show-toplevel") if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return false, ctxErr } - return false, fmt.Errorf("resolve portable store worktree: %w\n%s", err, strings.TrimSpace(stderr)) + return false, fmt.Errorf("resolve portable store worktree: %w", err) } if !sameExistingPath(strings.TrimSpace(topLevel), dir) { return false, fmt.Errorf("Git resolved portable store candidate %s to worktree %s", dir, strings.TrimSpace(topLevel)) } - gitDir, stderr, err := runGitCommandOutputWithEnvSeparate(ctx, "", portableStoreGitProbeEnv(), "-C", dir, "rev-parse", "--absolute-git-dir") + gitDir, err := portableGitOutput(ctx, dir, "rev-parse", "--absolute-git-dir") if err != nil { - return false, fmt.Errorf("resolve portable store Git directory: %w\n%s", err, strings.TrimSpace(stderr)) + return false, fmt.Errorf("resolve portable store Git directory: %w", err) } if !sameExistingPath(strings.TrimSpace(gitDir), filepath.Join(dir, ".git")) { return false, fmt.Errorf("Git resolved portable store candidate %s to Git directory %s", dir, strings.TrimSpace(gitDir)) @@ -1340,36 +1498,6 @@ func sameExistingPath(left, right string) bool { return err == nil && os.SameFile(leftInfo, rightInfo) } -func portableStoreGitProbeEnv() []string { - repositoryEnv := map[string]struct{}{ - "GIT_ALTERNATE_OBJECT_DIRECTORIES": {}, - "GIT_CEILING_DIRECTORIES": {}, - "GIT_COMMON_DIR": {}, - "GIT_DIR": {}, - "GIT_DISCOVERY_ACROSS_FILESYSTEM": {}, - "GIT_GRAFT_FILE": {}, - "GIT_INDEX_FILE": {}, - "GIT_NAMESPACE": {}, - "GIT_OBJECT_DIRECTORY": {}, - "GIT_PREFIX": {}, - "GIT_QUARANTINE_PATH": {}, - "GIT_SHALLOW_FILE": {}, - "GIT_WORK_TREE": {}, - } - env := make([]string, 0, len(os.Environ())) - for _, entry := range os.Environ() { - name, _, _ := strings.Cut(entry, "=") - upperName := strings.ToUpper(name) - _, excluded := repositoryEnv[upperName] - if excluded || upperName == "GIT_CONFIG_COUNT" || upperName == "GIT_CONFIG_PARAMETERS" || - strings.HasPrefix(upperName, "GIT_CONFIG_KEY_") || strings.HasPrefix(upperName, "GIT_CONFIG_VALUE_") { - continue - } - env = append(env, entry) - } - return env -} - func portableStoreRemoteURL(ctx context.Context, root string) string { branch := currentGitBranch(ctx, root) remoteName := gitBranchRemote(ctx, root, branch) @@ -1395,6 +1523,11 @@ func portableStoreRepairAllowed(root, configPath string) bool { } func gitWorktreeClean(ctx context.Context, dir string) bool { + ctx, release, err := acquirePortableOwner(ctx, dir) + if err != nil { + return false + } + defer release() if err := runGit(ctx, "", "-C", dir, "update-index", "-q", "--refresh"); err != nil { return false } @@ -1408,7 +1541,12 @@ func gitWorktreeClean(ctx context.Context, dir string) bool { } func fastForwardGitCheckoutWithStaleIndexLockRetry(ctx context.Context, root string, quiet bool) (bool, error) { - err := fastForwardGitCheckout(ctx, root, quiet) + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return false, err + } + defer release() + err = fastForwardGitCheckout(ctx, root, quiet) if err == nil { return false, nil } @@ -1426,7 +1564,12 @@ func fastForwardGitCheckoutWithStaleIndexLockRetry(ctx context.Context, root str } func runGitWithStaleIndexLockRetry(ctx context.Context, root string, args ...string) (bool, error) { - err := runGit(ctx, "", args...) + ctx, release, err := acquirePortableOwner(ctx, root) + if err != nil { + return false, err + } + defer release() + err = runGit(ctx, "", args...) if err == nil { return false, nil } diff --git a/internal/cli/runtime_extra_test.go b/internal/cli/runtime_extra_test.go index 5833932e..06c94ee6 100644 --- a/internal/cli/runtime_extra_test.go +++ b/internal/cli/runtime_extra_test.go @@ -239,7 +239,7 @@ func TestCopySQLiteFileAtomicVerifiedRemovesTempFiles(t *testing.T) { source := filepath.Join(dir, "source.db") target := filepath.Join(dir, "target.db") seedPortableThread(t, source, 1, "copy temp cleanup") - if err := copySQLiteFileAtomicVerified(context.Background(), source, target); err != nil { + if _, err := copySQLiteFileAtomicVerified(context.Background(), source, target); err != nil { t.Fatalf("copy verified sqlite file: %v", err) } matches, err := filepath.Glob(filepath.Join(dir, ".target.db.tmp-*")) @@ -254,7 +254,7 @@ func TestCopySQLiteFileAtomicVerifiedRemovesTempFiles(t *testing.T) { if err := os.WriteFile(invalidSource, []byte("not sqlite"), 0o600); err != nil { t.Fatalf("write invalid source: %v", err) } - if err := copySQLiteFileAtomicVerified(context.Background(), invalidSource, failedTarget); err == nil { + if _, err := copySQLiteFileAtomicVerified(context.Background(), invalidSource, failedTarget); err == nil { t.Fatal("invalid sqlite source should fail validation") } matches, err = filepath.Glob(filepath.Join(dir, ".failed.db.tmp-*")) @@ -1067,22 +1067,6 @@ func TestPortableRuntimeUtilityBranches(t *testing.T) { if recentPortableRefresh("", now, time.Minute) || recentPortableRefresh("bad", now, time.Minute) || !recentPortableRefresh(now.Format(time.RFC3339Nano), now, time.Minute) { t.Fatal("recent refresh classification mismatch") } - lockPath := filepath.Join(dir, "refresh.lock") - if err := os.WriteFile(lockPath, []byte("123\n"), 0o600); err != nil { - t.Fatalf("write lock: %v", err) - } - removeStalePortableRefreshLock(lockPath, now) - if _, err := os.Stat(lockPath); err != nil { - t.Fatalf("fresh lock should remain: %v", err) - } - old := now.Add(-3 * portableStoreRefreshTimeout) - if err := os.Chtimes(lockPath, old, old); err != nil { - t.Fatalf("age lock: %v", err) - } - removeStalePortableRefreshLock(lockPath, now) - if _, err := os.Stat(lockPath); !os.IsNotExist(err) { - t.Fatalf("stale lock should be removed, err=%v", err) - } t.Setenv("GITCRAWL_PORTABLE_REFRESH_TTL", "0") if got := portableStoreRefreshInterval(); got != 0 { t.Fatalf("zero ttl = %s", got) @@ -1127,7 +1111,7 @@ func TestPortableRuntimeUtilityBranches(t *testing.T) { t.Fatal("explicit-config default portable store should be repairable") } lockRoot := filepath.Join(dir, "locked-store") - lockPath = filepath.Join(lockRoot, ".git", "index.lock") + lockPath := filepath.Join(lockRoot, ".git", "index.lock") if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { t.Fatalf("mkdir lock dir: %v", err) } diff --git a/internal/cli/status.go b/internal/cli/status.go new file mode 100644 index 00000000..79fd1765 --- /dev/null +++ b/internal/cli/status.go @@ -0,0 +1,100 @@ +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + + "github.com/openclaw/crawlkit/control" + "github.com/openclaw/gitcrawl/internal/config" + "github.com/openclaw/gitcrawl/internal/store" +) + +// Status resolves the same artifact/runtime paths as readers, but never calls +// their refresh or repair path. A gzip-only source is inspected in private temp. +func (a *App) localArchiveStatus(ctx context.Context, cfg config.Config) (control.Status, error) { + path, reportedPath := cfg.DBPath, cfg.DBPath + var warnings []string + stale := false + immutable := false + runtimeMirror := false + _, portable, err := portableStoreRoot(ctx, cfg.DBPath) + if err != nil { + return control.Status{}, err + } + if portable { + mirror, err := a.portableRuntimeDBPath(ctx, cfg.DBPath) + if err != nil { + return control.Status{}, err + } + if _, err := os.Stat(mirror); err == nil { + path, reportedPath = mirror, mirror + runtimeMirror = true + state := readPortableStoreRefreshState(portableStoreRefreshStatePath(mirror)) + modTime, size, sha, stampErr := portableDBManifestStamp(cfg.DBPath) + if stampErr != nil || !portableManifestGenerationUnchanged(state, modTime, size, sha) { + stale = true + warnings = append(warnings, "Runtime source generation differs from the checkout or cannot be verified; status did not refresh it.") + } + if state.MirrorWritable { + warnings = append(warnings, "Runtime is writable local state; publisher updates do not replace it.") + } + } else if !errors.Is(err, os.ErrNotExist) { + return control.Status{}, err + } else { + artifact, _, compressed, err := portableSourceArtifact(cfg.DBPath) + if err != nil { + return control.Status{}, err + } + if compressed { + dir, err := os.MkdirTemp("", "gitcrawl-status-*") + if err != nil { + return control.Status{}, err + } + defer os.RemoveAll(dir) + path, err = stagePortableSQLiteSourceTempContext(ctx, cfg.DBPath, filepath.Join(dir, "archive.db"), 0o600) + if err != nil { + return control.Status{}, err + } + if err := validatePortableSQLiteFile(ctx, path, cfg.DBPath); err != nil { + return control.Status{}, err + } + reportedPath, immutable = artifact, true + warnings = append(warnings, "No runtime mirror exists; counts describe the validated gzip artifact and database bytes are its compressed on-disk size.") + } + } + } + status := store.Status{DBPath: reportedPath} + if _, err := os.Stat(path); err == nil { + open := store.OpenReadOnly + if immutable { + open = store.OpenReadOnlyImmutable + } else if runtimeMirror { + open = func(ctx context.Context, path string) (*store.Store, error) { + return openPortableMirrorReadOnly(ctx, path, cfg.DBPath) + } + } + st, err := open(ctx, path) + if err != nil { + return control.Status{}, err + } + defer st.Close() + status, err = st.Status(ctx) + if err != nil { + return control.Status{}, err + } + } else if !errors.Is(err, os.ErrNotExist) || portable { + return control.Status{}, err + } + status.DBPath = reportedPath + out := controlStatus(config.ResolvePath(a.configPath), cfg, status) + if stale { + out.State = "stale" + } + out.Warnings = append(out.Warnings, warnings...) + if immutable { + out.Databases[0].Kind = "sqlite-gzip" + } + return out, nil +} diff --git a/internal/portable/identity.go b/internal/portable/identity.go index ddceec72..09cc3f9f 100644 --- a/internal/portable/identity.go +++ b/internal/portable/identity.go @@ -2,10 +2,12 @@ package portable import ( "context" + "crypto/sha256" "database/sql" "encoding/binary" "errors" "fmt" + "io" "os" "path/filepath" "strings" @@ -166,10 +168,16 @@ type artifactIdentityOptions struct { // ComputeArtifactID returns the semantic identity for a finalized current-state // SQLite artifact. It never opens the artifact writable. func ComputeArtifactID(ctx context.Context, dbPath, profile string) (string, error) { + return ComputeArtifactIDInDirectory(ctx, dbPath, profile, "") +} + +// ComputeArtifactIDInDirectory confines disposable identity work to tempParent, +// allowing callers to account for its disk growth on a selected filesystem. +func ComputeArtifactIDInDirectory(ctx context.Context, dbPath, profile, tempParent string) (string, error) { if profile != CurrentStateSemanticV1 { return "", fmt.Errorf("unsupported artifact identity profile %q; supported profile: %s", profile, CurrentStateSemanticV1) } - return computeArtifactIDWithOptions(ctx, dbPath, currentStateSemanticPolicy, artifactIdentityOptions{}) + return computeArtifactIDWithOptions(ctx, dbPath, currentStateSemanticPolicy, artifactIdentityOptions{TempParent: tempParent}) } func computeArtifactIDWithOptions( @@ -255,14 +263,14 @@ func computeArtifactIDWithOptions( if err := ctx.Err(); err != nil { return "", fmt.Errorf("artifact identity canceled before hashing: %w", err) } - digest, err := hashArtifactIdentityFile(compactPath) + digest, err := hashArtifactIdentityFile(ctx, compactPath) if err != nil { return "", fmt.Errorf("hash artifact identity database: %w", err) } return digest, nil } -func hashArtifactIdentityFile(filePath string) (string, error) { +func hashArtifactIdentityFile(ctx context.Context, filePath string) (string, error) { file, err := os.OpenFile(filePath, os.O_RDWR, 0) if err != nil { return "", err @@ -291,7 +299,26 @@ func hashArtifactIdentityFile(filePath string) (string, error) { if err := file.Close(); err != nil { return "", err } - return fileSHA256(filePath) + input, err := os.Open(filePath) + if err != nil { + return "", err + } + defer input.Close() + hash := sha256.New() + buffer := make([]byte, 128<<10) + for { + if err := ctx.Err(); err != nil { + return "", err + } + count, err := input.Read(buffer) + _, _ = hash.Write(buffer[:count]) + if err == io.EOF { + return fmt.Sprintf("%x", hash.Sum(nil)), nil + } + if err != nil { + return "", err + } + } } func normalizeArtifactIdentity(ctx context.Context, db *sql.DB, policy artifactIdentityPolicy) error { diff --git a/internal/portable/identity_test.go b/internal/portable/identity_test.go index b4f38807..a6a5483b 100644 --- a/internal/portable/identity_test.go +++ b/internal/portable/identity_test.go @@ -242,11 +242,11 @@ func TestArtifactIdentityHashCanonicalizesSQLiteWriterHeader(t *testing.T) { if hashFile(t, firstPath) == hashFile(t, secondPath) { t.Fatal("fixture writer headers did not change exact file hashes") } - firstHash, err := hashArtifactIdentityFile(firstPath) + firstHash, err := hashArtifactIdentityFile(context.Background(), firstPath) if err != nil { t.Fatal(err) } - secondHash, err := hashArtifactIdentityFile(secondPath) + secondHash, err := hashArtifactIdentityFile(context.Background(), secondPath) if err != nil { t.Fatal(err) } @@ -267,7 +267,7 @@ func TestArtifactIdentityHashCanonicalizesSQLiteWriterHeader(t *testing.T) { if err := os.WriteFile(invalidPath, make([]byte, 100), 0o600); err != nil { t.Fatal(err) } - if _, err := hashArtifactIdentityFile(invalidPath); err == nil || !strings.Contains(err.Error(), "invalid SQLite header") { + if _, err := hashArtifactIdentityFile(context.Background(), invalidPath); err == nil || !strings.Contains(err.Error(), "invalid SQLite header") { t.Fatalf("invalid identity header error = %v", err) } }