From 53d3e44314f9384e8f9f0241d42501423df56884 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 14 Aug 2026 23:38:02 -0500 Subject: [PATCH 01/73] write-tree: preserve authenticated fsmonitor proofs A cache-tree miss makes write-tree rewrite the physical index. Unlike other proof-preserving writers, it never initialized the clean-status configuration digest or restored authenticated external history. Consequently, an otherwise valid FSCF extension was rewritten without its token and stat bindings while FSMN and FSUC remained valid. Subsequent read-only status calls reject the weakened proof and rebuild the entire attribute manifest. Snapshot commands copying that index repeat the same scan, and no proof-loss event is emitted because FSUC survived the rewrite. Initialize the configuration digest and authenticated history before write-tree reads the index, using the existing default configuration callback. This preserves complete proofs for both primary and linked worktrees without weakening alternate-index, filter, or attribute checks. Cover physical cache-tree rewrites, paired provider tokens, read-only follow-up status, and hostile staged attribute changes under both object formats. --- builtin/write-tree.c | 16 +++++- t/t7519-status-fsmonitor.sh | 99 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/builtin/write-tree.c b/builtin/write-tree.c index e3bd1a40dbf389..02f1ab48ac53c6 100644 --- a/builtin/write-tree.c +++ b/builtin/write-tree.c @@ -5,6 +5,8 @@ */ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -18,11 +20,19 @@ static const char * const write_tree_usage[] = { NULL }; +static int write_tree_config(const char *key, const char *value, + const struct config_context *ctx, void *data) +{ + clean_status_config_add(data, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + int cmd_write_tree(int argc, const char **argv, const char *cmd_prefix, struct repository *repo UNUSED) { + struct clean_status_config_digest clean_digest; int flags = 0, ret; const char *tree_prefix = NULL; struct object_id oid; @@ -44,7 +54,11 @@ int cmd_write_tree(int argc, OPT_END() }; - repo_config(the_repository, git_default_config, NULL); + clean_status_config_init(&clean_digest, the_repository->hash_algo); + repo_config(the_repository, write_tree_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); + clean_status_enable_external_history(the_repository); argc = parse_options(argc, argv, cmd_prefix, write_tree_options, write_tree_usage, 0); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 68ecc40f5b969b..c1feb4d29cb840 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -986,6 +986,105 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'write-tree preserves authenticated primary and linked index proofs' ' + test_when_finished "rm -rf write-tree-bound-proof write-tree-linked" && + test_create_repo write-tree-bound-proof && + ( + cd write-tree-bound-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + git worktree add --detach ../write-tree-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + cat >.git/check-write-tree-proof.pl <<-\EOF && + binmode STDIN; + local $/; + my $index = ; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + die "invalid $name token\n" if $end < 0; + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched provider tokens\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"} && + $tokens{"FSMN"} eq $tokens{"FSCF"}; + EOF + for worktree in "$PWD" "$PWD/../write-tree-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + test_write_lines changed >"$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git -C "$worktree" add tracked && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_grep "^1 M\\. .* tracked$" "$gitdir/prime" && + perl "$PWD/.git/check-write-tree-proof.pl" \ + <"$gitdir/index" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/write-tree.trace" \ + git -C "$worktree" write-tree \ + >"$gitdir/tree" && + test_file_not_empty "$gitdir/tree" && + test_region index do_write_index \ + "$gitdir/write-tree.trace" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/write-tree.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/write-tree.trace" && + perl "$PWD/.git/check-write-tree-proof.pl" \ + <"$gitdir/index" && + cp "$gitdir/index" "$gitdir/readonly.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/status" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + test_grep "^1 M\\. .* tracked$" "$gitdir/status" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/status.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/status.trace" && + test_write_lines "*.asset text" \ + >"$worktree/.gitattributes" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" add .gitattributes && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/attributes.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/attributes" && + test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/attributes.trace" && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/attributes.trace" && + test_grep "^1 A\\. .* \\.gitattributes$" \ + "$gitdir/attributes" || return 1 + done + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'expired add preserves untracked candidates until revalidation' ' test_when_finished "rm -rf pending-untracked-revalidation pending-untracked-hostile" && From 32095303dbc1ae590cebe3c8f5eee744c0e6c23a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 14 Aug 2026 23:53:36 -0500 Subject: [PATCH 02/73] write-tree: avoid clean-proof work for temporary indexes Temporary snapshot indexes can legitimately lose clean-proof bindings when they stage new paths. Do not initialize physical-index clean-status history for an explicit GIT_INDEX_FILE; preserve the previous configuration-only behavior and avoid scanning the complete attribute manifest. Handle write-tree help before inspecting repository hash settings so -h and --help-all continue to work outside a repository. Extend the physical-index regression with a newly staged temporary path, verify its intentionally weakened proof remains isolated, and require write-tree to avoid a full manifest scan. --- builtin/write-tree.c | 18 +++++++++++++----- t/t7519-status-fsmonitor.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/builtin/write-tree.c b/builtin/write-tree.c index 02f1ab48ac53c6..3a826add3459ce 100644 --- a/builtin/write-tree.c +++ b/builtin/write-tree.c @@ -54,11 +54,19 @@ int cmd_write_tree(int argc, OPT_END() }; - clean_status_config_init(&clean_digest, the_repository->hash_algo); - repo_config(the_repository, write_tree_config, &clean_digest); - clean_status_config_final(&clean_digest); - clean_status_set_config_digest(the_repository, &clean_digest); - clean_status_enable_external_history(the_repository); + show_usage_with_options_if_asked(argc, argv, + write_tree_usage, write_tree_options); + + if (!getenv(INDEX_ENVIRONMENT)) { + clean_status_config_init(&clean_digest, + the_repository->hash_algo); + repo_config(the_repository, write_tree_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); + clean_status_enable_external_history(the_repository); + } else { + repo_config(the_repository, git_default_config, NULL); + } argc = parse_options(argc, argv, cmd_prefix, write_tree_options, write_tree_usage, 0); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index c1feb4d29cb840..15beaf1b6e02c1 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1023,6 +1023,15 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ $tokens{"FSMN"} eq $tokens{"FSUC"} && $tokens{"FSMN"} eq $tokens{"FSCF"}; EOF + cat >.git/check-write-tree-unbound.pl <<-\EOF && + binmode STDIN; + local $/; + my $index = ; + my $offset = index($index, "FSCF"); + die "missing FSCF extension\n" if $offset < 0; + my $flags = unpack("N", substr($index, $offset + 16, 4)); + die "unexpected FSCF flags $flags\n" if $flags != 9; + EOF for worktree in "$PWD" "$PWD/../write-tree-linked" do gitdir=$(git -C "$worktree" \ @@ -1066,6 +1075,29 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <"$gitdir/status.trace" && ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/status.trace" && + cp "$gitdir/index" "$gitdir/snapshot.index" && + test_write_lines snapshot >"$worktree/snapshot-new" && + printf "%s\n" snapshot-new | + GIT_INDEX_FILE="$gitdir/snapshot.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/snapshot-add.trace" \ + git -C "$worktree" add --sparse \ + --pathspec-from-file=- && + perl "$PWD/.git/check-write-tree-unbound.pl" \ + <"$gitdir/snapshot.index" && + GIT_INDEX_FILE="$gitdir/snapshot.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/snapshot-tree.trace" \ + git -C "$worktree" write-tree \ + >"$gitdir/snapshot-tree" && + test_file_not_empty "$gitdir/snapshot-tree" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/snapshot-tree.trace" && + git -C "$worktree" ls-tree \ + "$(cat "$gitdir/snapshot-tree")" snapshot-new \ + >"$gitdir/snapshot-entry" && + test_grep "snapshot-new$" "$gitdir/snapshot-entry" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && test_write_lines "*.asset text" \ >"$worktree/.gitattributes" && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ From 8abeda59ec029ebb428d12ad00173362bee779f4 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 01:30:38 -0500 Subject: [PATCH 03/73] fsmonitor: preserve authenticated proofs across ordinary commands Ordinary index writers can discard authenticated FSUC/FSCF history even when their changes are semantically safe. This leaves status, snapshot generation, hooks, and pull/rebase chains rebuilding the complete attribute manifest on subsequent commands. Carry clean-status configuration and history through physical write-tree calls, rm/mv, mixed reset, fast-forward merge, rebase, and autostash. Authenticate every affected index entry, and do not preserve proofs for attributes, ignore rules, active filters, conflicts, sparse indexes, or genuine alternate indexes. Keep untracked snapshots explicitly pending when diff closes a restarted provider epoch, and refuse to certify stale untracked data from tracked- only status. Restore private extensions stripped by legacy writers and recognize the authenticated update-index invocation used by doctor. Treat truly distinct history-free snapshot indexes as a strongly invalidated stat-only fallback instead of rebuilding the whole manifest. Reject canonical primary-index, lockfile, symlink, and hardlink aliases. Exercise primary and linked worktrees under both object formats, native provider restarts, offline files and attributes, real pre-commit hooks, mixed writers, pull/rebase/autostash chains, doctor recovery, hostile filters, scratch indexes, and read-only index immutability. --- builtin/diff.c | 17 +- builtin/merge.c | 9 + builtin/mv.c | 47 +- builtin/pull.c | 19 +- builtin/rebase.c | 17 + builtin/reset.c | 3 +- builtin/rm.c | 35 +- builtin/stash.c | 25 +- builtin/update-index.c | 6 +- builtin/write-tree.c | 30 +- fsmonitor.c | 36 ++ merge-ort.c | 9 +- merge.c | 3 + reset.c | 6 + reset.h | 3 + sequencer.c | 3 +- t/t7519-status-fsmonitor.sh | 887 +++++++++++++++++++++++++++++++++++- wt-status.c | 1 + 18 files changed, 1136 insertions(+), 20 deletions(-) diff --git a/builtin/diff.c b/builtin/diff.c index 666e39497e3b2d..35f9a17f59f853 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -23,6 +23,7 @@ #include "diff.h" #include "diff-merges.h" #include "diffcore.h" +#include "dir.h" #include "preload-index.h" #include "read-cache-ll.h" #include "revision.h" @@ -245,6 +246,7 @@ static void refresh_index_quietly(void) struct lock_file lock_file = LOCK_INIT; struct index_state *istate = the_repository->index; int can_close_token; + int preserve_untracked; int fd; int refreshed; @@ -268,6 +270,8 @@ static void refresh_index_quietly(void) REFRESH_QUIET | REFRESH_UNMERGED | (can_close_token ? REFRESH_IN_PROOF_EPOCH : 0), NULL, NULL, NULL); + preserve_untracked = istate->untracked && + istate->untracked->fsmonitor_revalidation; /* A complete tracked refresh cannot also authenticate untracked files. */ if (!refreshed && can_close_token && fsmonitor_pending_token_from_provider(istate) && @@ -275,6 +279,17 @@ static void refresh_index_quietly(void) clean_status_mark_fsmonitor_config_valid( istate, istate->fsmonitor_last_update_pending); fsmonitor_accept_pending_token(istate, 0, 0); + if (preserve_untracked && + clean_status_revalidated_token_matches(istate)) { + /* + * Preserve directory snapshots only as candidates. Their + * pending proof requires a full untracked revalidation. + */ + istate->untracked->fsmonitor_revalidation = 1; + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + clean_status_begin_fsmonitor_semantic_baseline(istate); + } } repo_update_index_if_able(the_repository, &lock_file); } @@ -437,7 +452,7 @@ void prepare_diff_external_history(struct repository *repo) repo_config_values(repo)->apply_sparse_checkout) goto done; worktree = get_current_worktree(repo); - if (!worktree || !is_main_worktree(worktree) || + if (!worktree || clean_status_config_read_repository(repo, &digest) || digest.filter_configured) goto done; diff --git a/builtin/merge.c b/builtin/merge.c index 58d1b7bb07d90f..c7e8a31208a386 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -13,6 +13,8 @@ #include "abspath.h" #include "advice.h" +#include "clean-status-config.h" +#include "clean-status.h" #include "config.h" #include "editor.h" #include "environment.h" @@ -1373,6 +1375,7 @@ int cmd_merge(int argc, struct commit_list *common = NULL; const char *best_strategy = NULL, *wt_strategy = NULL; struct commit_list *remoteheads = NULL, *p; + struct clean_status_config_digest clean_digest; void *branch_to_free; int orig_argc = argc; int merge_log_config = -1; @@ -1469,6 +1472,12 @@ int cmd_merge(int argc, goto done; } + if (fast_forward != FF_NO && !getenv(INDEX_ENVIRONMENT) && + !clean_status_config_read_repository(the_repository, &clean_digest)) { + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, &clean_digest); + } + if (repo_read_index_unmerged(the_repository)) die_resolve_conflict("merge"); diff --git a/builtin/mv.c b/builtin/mv.c index a82fc97a19f6ee..150c77846a65c3 100644 --- a/builtin/mv.c +++ b/builtin/mv.c @@ -10,6 +10,8 @@ #include "builtin.h" #include "abspath.h" #include "advice.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -205,11 +207,20 @@ static int pathmap_cmp(const void *cmp_data UNUSED, return fspathcmp(e1->path, e2->path); } +static int mv_config(const char *key, const char *value, + const struct config_context *ctx, void *data) +{ + clean_status_config_add(data, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + int cmd_mv(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED) { + struct clean_status_config_digest clean_digest; + int preserve_clean_history = !getenv(INDEX_ENVIRONMENT); int i, flags, gitmodules_modified = 0; int verbose = 0, show_only = 0, force = 0, ignore_errors = 0, ignore_sparse = 0; struct option builtin_mv_options[] = { @@ -240,7 +251,19 @@ int cmd_mv(int argc, int ret; struct repo_config_values *cfg = repo_config_values(the_repository); - repo_config(the_repository, git_default_config, NULL); + show_usage_with_options_if_asked(argc, argv, + builtin_mv_usage, builtin_mv_options); + + if (preserve_clean_history) { + clean_status_config_init(&clean_digest, + the_repository->hash_algo); + repo_config(the_repository, mv_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); + clean_status_enable_external_history(the_repository); + } else { + repo_config(the_repository, git_default_config, NULL); + } argc = parse_options(argc, argv, prefix, builtin_mv_options, builtin_mv_usage, 0); @@ -571,6 +594,28 @@ int cmd_mv(int argc, the_repository->index->cache[pos], &st, 0); + if (preserve_clean_history) { + struct cache_entry *old_entry = + the_repository->index->cache[pos]; + struct cache_entry *new_entry; + size_t dstlen = strlen(dst); + int safe; + + new_entry = make_empty_cache_entry( + the_repository->index, dstlen); + copy_cache_entry(new_entry, old_entry); + new_entry->ce_namelen = dstlen; + new_entry->index = 0; + memcpy(new_entry->name, dst, dstlen + 1); + safe = clean_status_index_entry_is_semantically_safe( + the_repository->index, old_entry, NULL) && + clean_status_index_entry_is_semantically_safe( + the_repository->index, NULL, new_entry); + discard_cache_entry(new_entry); + if (!safe) + clean_status_invalidate_current_proof( + the_repository->index); + } rename_index_entry_at(the_repository->index, pos, dst); if (ignore_sparse && diff --git a/builtin/pull.c b/builtin/pull.c index db3ee0aab3ed91..daaf273f0da9a5 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -10,6 +10,8 @@ #include "builtin.h" #include "advice.h" +#include "clean-status-config.h" +#include "clean-status.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -226,6 +228,9 @@ static enum rebase_type config_get_rebase(int *rebase_unspecified) static int git_pull_config(const char *var, const char *value, const struct config_context *ctx, void *cb) { + if (cb) + clean_status_config_add(cb, var, value, ctx); + if (!strcmp(var, "rebase.autostash")) { /* * run_rebase() also reads this option. The reason we handle it here is @@ -248,7 +253,7 @@ static int git_pull_config(const char *var, const char *value, check_trust_level = 0; } - return git_default_config(var, value, ctx, cb); + return git_default_config(var, value, ctx, NULL); } /** @@ -862,6 +867,7 @@ int cmd_pull(int argc, struct oid_array merge_heads = OID_ARRAY_INIT; struct object_id orig_head, curr_head; struct object_id rebase_fork_point; + struct clean_status_config_digest clean_digest; int rebase_unspecified = 0; int can_ff; int divergent; @@ -1015,7 +1021,16 @@ int cmd_pull(int argc, if (!getenv("GIT_REFLOG_ACTION")) set_reflog_message(argc, argv); - repo_config(the_repository, git_pull_config, NULL); + if (the_repository->gitdir && !getenv(INDEX_ENVIRONMENT)) { + clean_status_config_init(&clean_digest, + the_repository->hash_algo); + repo_config(the_repository, git_pull_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, &clean_digest); + } else { + repo_config(the_repository, git_pull_config, NULL); + } if (the_repository->gitdir) { prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/rebase.c b/builtin/rebase.c index 10a306310cd439..5ca096f48a20c7 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -10,6 +10,8 @@ #include "builtin.h" #include "abspath.h" +#include "clean-status-config.h" +#include "clean-status.h" #include "environment.h" #include "gettext.h" #include "hex.h" @@ -135,6 +137,8 @@ struct rebase_options { int config_autosquash; int config_rebase_merges; int config_update_refs; + struct clean_status_config_digest clean_digest; + unsigned clean_history_enabled : 1; }; #define REBASE_OPTIONS_INIT { \ @@ -795,6 +799,9 @@ static int rebase_config(const char *var, const char *value, { struct rebase_options *opts = data; + if (opts->clean_history_enabled) + clean_status_config_add(&opts->clean_digest, var, value, ctx); + if (!strcmp(var, "rebase.stat")) { if (git_config_bool(var, value)) opts->flags |= REBASE_DIFFSTAT; @@ -1261,7 +1268,17 @@ int cmd_rebase(int argc, prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; + options.clean_history_enabled = !getenv(INDEX_ENVIRONMENT); + if (options.clean_history_enabled) + clean_status_config_init(&options.clean_digest, + the_repository->hash_algo); repo_config(the_repository, rebase_config, &options); + if (options.clean_history_enabled) { + clean_status_config_final(&options.clean_digest); + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, + &options.clean_digest); + } /* options.gpg_sign_opt will be either "-S" or NULL */ gpg_sign = options.gpg_sign_opt ? "" : NULL; FREE_AND_NULL(options.gpg_sign_opt); diff --git a/builtin/reset.c b/builtin/reset.c index d4129d64aa9fc9..c7c3570b3cb81b 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -543,8 +543,7 @@ int cmd_reset(int argc, (the_repository->index->split_index || the_repository->index->sparse_index || (the_repository->index->cache_changed & - (CE_ENTRY_CHANGED | CE_ENTRY_ADDED | - RESOLVE_UNDO_CHANGED)))) + RESOLVE_UNDO_CHANGED))) clean_status_invalidate_current_proof( the_repository->index); the_repository->index->updated_skipworktree = 1; diff --git a/builtin/rm.c b/builtin/rm.c index 081d0bc3754c52..39636abb93aedd 100644 --- a/builtin/rm.c +++ b/builtin/rm.c @@ -8,6 +8,8 @@ #include "builtin.h" #include "advice.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "lockfile.h" @@ -262,17 +264,38 @@ static struct option builtin_rm_options[] = { OPT_END(), }; +static int rm_config(const char *key, const char *value, + const struct config_context *ctx, void *data) +{ + clean_status_config_add(data, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + int cmd_rm(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED) { + struct clean_status_config_digest clean_digest; struct lock_file lock_file = LOCK_INIT; + int preserve_clean_history = !getenv(INDEX_ENVIRONMENT); int i, ret = 0; struct pathspec pathspec; char *seen; - repo_config(the_repository, git_default_config, NULL); + show_usage_with_options_if_asked(argc, argv, + builtin_rm_usage, builtin_rm_options); + + if (preserve_clean_history) { + clean_status_config_init(&clean_digest, + the_repository->hash_algo); + repo_config(the_repository, rm_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); + clean_status_enable_external_history(the_repository); + } else { + repo_config(the_repository, git_default_config, NULL); + } argc = parse_options(argc, argv, prefix, builtin_rm_options, builtin_rm_usage, 0); @@ -392,9 +415,19 @@ int cmd_rm(int argc, */ for (i = 0; i < list.nr; i++) { const char *path = list.entry[i].name; + int pos; if (!quiet) printf("rm '%s'\n", path); + pos = index_name_pos(the_repository->index, + path, strlen(path)); + if (preserve_clean_history && + (pos < 0 || + !clean_status_index_entry_is_semantically_safe( + the_repository->index, + the_repository->index->cache[pos], NULL))) + clean_status_invalidate_current_proof(the_repository->index); + if (remove_file_from_index(the_repository->index, path)) die(_("git rm: unable to remove %s"), path); } diff --git a/builtin/stash.c b/builtin/stash.c index 38712cafba13bc..f5bb8b37ac18c3 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -589,6 +589,7 @@ static void unstage_changes_unless_new(struct object_id *orig_tree) struct stat st; ce = the_repository->index->cache[pos]; + clean_status_invalidate_current_proof(the_repository->index); if (!lstat(ce->name, &st)) { /* Conflicting path present; relocate it */ struct strbuf new_path = STRBUF_INIT; @@ -629,6 +630,12 @@ static void unstage_changes_unless_new(struct object_id *orig_tree) &p->one->oid, p->one->path, 0, 0); + if (!clean_status_index_entry_is_semantically_safe( + the_repository->index, + pos >= 0 ? the_repository->index->cache[pos] : NULL, + ce)) + clean_status_invalidate_current_proof( + the_repository->index); add_index_entry(the_repository->index, ce, option); } } @@ -656,6 +663,12 @@ static int do_apply_stash(const char *prefix, struct stash_info *info, struct tree *head, *merge, *merge_base; struct lock_file lock = LOCK_INIT; + if (!getenv(INDEX_ENVIRONMENT)) { + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, + &stash_clean_digest); + } + repo_read_index_preload(the_repository, NULL, 0); if (repo_refresh_and_write_index(the_repository, REFRESH_QUIET, 0, 0, NULL, NULL, NULL)) @@ -761,10 +774,14 @@ static int do_apply_stash(const char *prefix, struct stash_info *info, */ cp.git_cmd = 1; cp.dir = prefix; - strvec_pushf(&cp.env, GIT_WORK_TREE_ENVIRONMENT"=%s", - absolute_path(repo_get_work_tree(the_repository))); - strvec_pushf(&cp.env, GIT_DIR_ENVIRONMENT"=%s", - absolute_path(repo_get_git_dir(the_repository))); + /* Keep discovered config origins stable unless discovery is overridden. */ + if (getenv(GIT_DIR_ENVIRONMENT) || + getenv(GIT_WORK_TREE_ENVIRONMENT)) { + strvec_pushf(&cp.env, GIT_WORK_TREE_ENVIRONMENT"=%s", + absolute_path(repo_get_work_tree(the_repository))); + strvec_pushf(&cp.env, GIT_DIR_ENVIRONMENT"=%s", + absolute_path(repo_get_git_dir(the_repository))); + } strvec_push(&cp.args, "status"); run_command(&cp); } diff --git a/builtin/update-index.c b/builtin/update-index.c index 66746d11352e67..1582d8ecc74b45 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -88,7 +88,11 @@ static int is_proof_preserving_rewrite(int argc, const char **argv) return (!strcmp(argv[1], "--refresh") && !strcmp(argv[2], "--force-write-index")) || (!strcmp(argv[1], "--force-write-index") && - !strcmp(argv[2], "--refresh")); + !strcmp(argv[2], "--refresh")) || + (!strcmp(argv[1], "--untracked-cache") && + !strcmp(argv[2], "--force-write-index")) || + (!strcmp(argv[1], "--force-write-index") && + !strcmp(argv[2], "--untracked-cache")); } static int is_fsmonitor_invalidation_rewrite(int argc, const char **argv) diff --git a/builtin/write-tree.c b/builtin/write-tree.c index 3a826add3459ce..338e4565c2fc29 100644 --- a/builtin/write-tree.c +++ b/builtin/write-tree.c @@ -5,12 +5,14 @@ */ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "abspath.h" #include "clean-status.h" #include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" #include "hex.h" +#include "strbuf.h" #include "tree.h" #include "cache-tree.h" #include "parse-options.h" @@ -27,6 +29,32 @@ static int write_tree_config(const char *key, const char *value, return git_default_config(key, value, ctx, NULL); } +static int write_tree_uses_worktree_index(void) +{ + const char *index_file = getenv(INDEX_ENVIRONMENT); + struct strbuf worktree_index = STRBUF_INIT; + struct stat st; + char *expected = NULL, *actual = NULL; + int matches = 0; + + if (!index_file) + return 1; + if (lstat(index_file, &st) || S_ISLNK(st.st_mode)) + return 0; + + strbuf_addf(&worktree_index, "%s/index", + repo_get_git_dir(the_repository)); + expected = real_pathdup(worktree_index.buf, 0); + actual = real_pathdup(index_file, 0); + if (expected && actual && !strcmp(expected, actual)) + matches = 1; + + free(expected); + free(actual); + strbuf_release(&worktree_index); + return matches; +} + int cmd_write_tree(int argc, const char **argv, const char *cmd_prefix, @@ -57,7 +85,7 @@ int cmd_write_tree(int argc, show_usage_with_options_if_asked(argc, argv, write_tree_usage, write_tree_options); - if (!getenv(INDEX_ENVIRONMENT)) { + if (write_tree_uses_worktree_index()) { clean_status_config_init(&clean_digest, the_repository->hash_algo); repo_config(the_repository, write_tree_config, &clean_digest); diff --git a/fsmonitor.c b/fsmonitor.c index 1d2420f20ba22c..7026d2148d3fb0 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -2,6 +2,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "abspath.h" #include "attr.h" #include "clean-status.h" #include "clean-status-manifest.h" @@ -1173,6 +1174,34 @@ static void invalidate_fsmonitor_for_bootstrap( invalidate_all_fsmonitor(istate); return; } + if (getenv(INDEX_ENVIRONMENT) && + !clean_status_has_persistent_fsmonitor_semantic_history(istate) && + !clean_status_has_worktree_manifest_history(istate)) { + char *physical = xstrfmt("%s/index", repo_get_git_dir(istate->repo)); + char *selected = real_pathdup(repo_get_index_file(istate->repo), 0); + char *canonical = real_pathdup(physical, 0); + char *physical_lock = canonical ? xstrfmt("%s.lock", canonical) : NULL; + struct stat selected_stat, physical_stat; + int temporary = selected && canonical && + fspathcmp(selected, canonical) && + fspathcmp(selected, physical_lock) && + !stat(selected, &selected_stat) && + !stat(canonical, &physical_stat) && + (selected_stat.st_dev != physical_stat.st_dev || + selected_stat.st_ino != physical_stat.st_ino); + + free(physical_lock); + free(canonical); + free(selected); + free(physical); + if (temporary) { + fsmonitor_invalidate_semantics(istate); + untracked_cache_invalidate_all(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/temporary-index-stat-fallback", 1); + return; + } + } if (physical_history_unavailable) { int authenticated_manifest = @@ -1664,6 +1693,13 @@ void fsmonitor_accept_pending_token(struct index_state *istate, trace2_data_intmax("fsmonitor", istate->repo, "untracked/provider-reset-revalidated", 1); } + if (untracked_cache_valid && + !istate->fsmonitor_untracked_extension_seen && + istate == istate->repo->index && + !getenv(INDEX_ENVIRONMENT) && !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(istate->repo) == FSMONITOR_MODE_IPC) + istate->fsmonitor_untracked_must_persist = 1; istate->untracked->fsmonitor_revalidation = 0; istate->untracked->use_fsmonitor = !!untracked_cache_valid; } diff --git a/merge-ort.c b/merge-ort.c index c410a5d353234c..a090759bc4b0ff 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -24,6 +24,7 @@ #include "advice.h" #include "attr.h" #include "cache-tree.h" +#include "clean-status.h" #include "commit.h" #include "commit-reach.h" #include "config.h" @@ -4603,7 +4604,8 @@ static int process_entries(struct merge_options *opt, static int checkout(struct merge_options *opt, struct tree *prev, - struct tree *next) + struct tree *next, + int preserve_semantic_history) { /* Switch the index/working copy from old to new */ int ret; @@ -4629,6 +4631,9 @@ static int checkout(struct merge_options *opt, /* 2-way merge to the new branch */ unpack_opts.update = 1; unpack_opts.merge = 1; + unpack_opts.preserve_semantic_history = + preserve_semantic_history && + clean_status_revalidated_token_matches(opt->repo->index); unpack_opts.quiet = 0; /* FIXME: sequencer might want quiet? */ unpack_opts.verbose_update = (opt->verbosity > 2); unpack_opts.fn = twoway_merge; @@ -4933,7 +4938,7 @@ void merge_switch_to_result(struct merge_options *opt, assert(opt->priv == NULL); if (result->clean >= 0 && update_worktree_and_index) { trace2_region_enter("merge", "checkout", opt->repo); - if (checkout(opt, head, result->tree)) { + if (checkout(opt, head, result->tree, result->clean > 0)) { /* failure to function */ result->clean = -1; merge_finalize(opt, result); diff --git a/merge.c b/merge.c index 0f5e823e63ed5f..ac37e84ad87465 100644 --- a/merge.c +++ b/merge.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "gettext.h" +#include "clean-status.h" #include "hash.h" #include "hex.h" #include "lockfile.h" @@ -96,6 +97,8 @@ int checkout_fast_forward(struct repository *r, opts.update = 1; opts.verbose_update = 1; opts.merge = 1; + opts.preserve_semantic_history = + clean_status_revalidated_token_matches(r->index); opts.fn = twoway_merge; init_checkout_metadata(&opts.meta, NULL, remote, NULL); setup_unpack_trees_porcelain(&opts, "merge"); diff --git a/reset.c b/reset.c index 71254bde93fc51..6d284f80c622ef 100644 --- a/reset.c +++ b/reset.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "cache-tree.h" +#include "clean-status.h" #include "gettext.h" #include "hex.h" #include "lockfile.h" @@ -166,6 +167,11 @@ int reset_working_tree(struct repository *r, unpack_tree_opts.update = !dry_run; unpack_tree_opts.dry_run = dry_run; unpack_tree_opts.merge = 1; + unpack_tree_opts.preserve_semantic_history = + !dry_run && + (!reset_hard || + (opts->flags & RESET_WORKING_TREE_PRESERVE_SEMANTIC_HISTORY)) && + clean_status_revalidated_token_matches(istate); unpack_tree_opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ init_checkout_metadata(&unpack_tree_opts.meta, switch_to_branch, oid, NULL); if (reset_hard) { diff --git a/reset.h b/reset.h index 4c992ba671c7f1..5d8a39b705ba25 100644 --- a/reset.h +++ b/reset.h @@ -30,6 +30,9 @@ enum reset_working_tree_flags { * any user-visible state. */ RESET_WORKING_TREE_DRY_RUN = (1 << 6), + + /* Preserve authenticated semantic history during an autostash reset. */ + RESET_WORKING_TREE_PRESERVE_SEMANTIC_HISTORY = (1 << 7), }; struct reset_working_tree_options { diff --git a/sequencer.c b/sequencer.c index 82ab3c536f94a2..6a8fcd1170bd98 100644 --- a/sequencer.c +++ b/sequencer.c @@ -4724,7 +4724,8 @@ static void create_autostash_internal(struct repository *r, struct child_process stash = CHILD_PROCESS_INIT; struct reset_working_tree_options ropts = { .flags = RESET_WORKING_TREE_HARD | - RESET_WORKING_TREE_UPDATE_HEAD, + RESET_WORKING_TREE_UPDATE_HEAD | + RESET_WORKING_TREE_PRESERVE_SEMANTIC_HISTORY, }; struct object_id oid; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 15beaf1b6e02c1..b384e03fe71e0d 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -598,6 +598,39 @@ prepare_builtin_closure_repo () { ) } +test_fsmonitor_pending_full_proof () { + perl - "$1" <<-\EOF + binmode STDIN; + open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; + binmode $input; + local $/; + my $index = <$input>; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + die "invalid $name token\n" if $end < 0; + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched tracked provider token\n" unless + $tokens{"FSMN"} eq $tokens{"FSCF"}; + my ($suffix) = $tokens{"FSMN"} =~ /\Abuiltin:(.+)\z/; + die "missing provider token\n" unless defined $suffix; + die "mismatched pending untracked token\n" unless + $tokens{"FSUC"} eq "pending:$suffix"; + EOF +} + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'bare status reuses a current tracked fsmonitor proof' ' test_when_finished "rm -rf builtin-tracked-clean" && @@ -1032,6 +1065,11 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ my $flags = unpack("N", substr($index, $offset + 16, 4)); die "unexpected FSCF flags $flags\n" if $flags != 9; EOF + write_script .git/hooks/pre-commit <<-\EOF && + test -n "$GIT_INDEX_FILE" || exit 1 + git write-tree >"$HOOK_PROOF_OUTPUT" || exit 1 + perl "$HOOK_PROOF_HELPER" <"$GIT_INDEX_FILE" + EOF for worktree in "$PWD" "$PWD/../write-tree-linked" do gitdir=$(git -C "$worktree" \ @@ -1075,6 +1113,44 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <"$gitdir/status.trace" && ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/status.trace" && + test_write_lines canonical >"$worktree/sibling" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=sibling \ + git -C "$worktree" add sibling && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/canonical-prime" && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/canonical-tree.trace" \ + git -C "$worktree" write-tree \ + >"$gitdir/canonical-tree" && + test_region index do_write_index \ + "$gitdir/canonical-tree.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/canonical-tree.trace" && + perl "$PWD/.git/check-write-tree-proof.pl" \ + <"$gitdir/index" && + test_write_lines hooked >"$worktree/sibling" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=sibling \ + git -C "$worktree" add sibling && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/hook-prime" && + HOOK_PROOF_HELPER="$PWD/.git/check-write-tree-proof.pl" \ + HOOK_PROOF_OUTPUT="$gitdir/hook-tree" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/hook.trace" \ + git -C "$worktree" commit -qm hooked && + test_file_not_empty "$gitdir/hook-tree" && + test_grep "\"name\":\"write-tree\"" \ + "$gitdir/hook.trace" && + perl "$PWD/.git/check-write-tree-proof.pl" \ + <"$gitdir/index" && + cp "$gitdir/index" "$gitdir/readonly.index" && cp "$gitdir/index" "$gitdir/snapshot.index" && test_write_lines snapshot >"$worktree/snapshot-new" && printf "%s\n" snapshot-new | @@ -1098,6 +1174,37 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ >"$gitdir/snapshot-entry" && test_grep "snapshot-new$" "$gitdir/snapshot-entry" && test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + GIT_INDEX_FILE="$gitdir/manifestless.index" \ + git -C "$worktree" read-tree HEAD && + test_grep ! FSCF "$gitdir/manifestless.index" && + test_write_lines manifestless \ + >"$worktree/manifestless-new" && + printf "%s\n" manifestless-new | + GIT_INDEX_FILE="$gitdir/manifestless.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/manifestless-add.trace" \ + git -C "$worktree" add --sparse \ + --pathspec-from-file=- && + test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/manifestless-add.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/manifestless-add.trace" && + GIT_INDEX_FILE="$gitdir/manifestless.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/manifestless-tree.trace" \ + git -C "$worktree" write-tree \ + >"$gitdir/manifestless-tree" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/manifestless-tree.trace" && + git -C "$worktree" ls-tree \ + "$(cat "$gitdir/manifestless-tree")" \ + manifestless-new >"$gitdir/manifestless-entry" && + test_grep "manifestless-new$" \ + "$gitdir/manifestless-entry" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && test_write_lines "*.asset text" \ >"$worktree/.gitattributes" && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ @@ -1117,6 +1224,466 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'fast-forward merges preserve authenticated worktree proofs' ' + test_when_finished "rm -rf fast-forward-bound-proof fast-forward-linked fast-forward-target" && + test_create_repo fast-forward-bound-proof && + ( + cd fast-forward-bound-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git worktree add --detach ../fast-forward-target HEAD && + test_write_lines incoming >../fast-forward-target/tracked && + git -C ../fast-forward-target add tracked && + git -C ../fast-forward-target commit -qm incoming && + target=$(git -C ../fast-forward-target rev-parse HEAD) && + git worktree add --detach ../fast-forward-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + cat >.git/check-merge-proof.pl <<-\EOF && + binmode STDIN; + local $/; + my $index = ; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + die "invalid $name token\n" if $end < 0; + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched provider tokens\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"} && + $tokens{"FSMN"} eq $tokens{"FSCF"}; + EOF + for worktree in "$PWD" "$PWD/../fast-forward-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + perl "$PWD/.git/check-merge-proof.pl" \ + <"$gitdir/index" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/merge.trace" \ + git -C "$worktree" merge --ff-only "$target" \ + >"$gitdir/merge" && + test_region index do_write_index "$gitdir/merge.trace" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/merge.trace" && + perl "$PWD/.git/check-merge-proof.pl" \ + <"$gitdir/index" && + cp "$gitdir/index" "$gitdir/readonly.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/status" && + test_must_be_empty "$gitdir/status" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/status.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/status.trace" && + test_write_lines "*.asset text" \ + >"$worktree/.gitattributes" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" add .gitattributes && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/attributes.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/attributes" && + test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/attributes.trace" && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/attributes.trace" && + test_grep "^1 A\\. .* \\.gitattributes$" \ + "$gitdir/attributes" || return 1 + done + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'full status durably repairs missing mixed-writer index proofs' ' + test_when_finished "rm -rf mixed-writer-missing-proofs" && + test_create_repo mixed-writer-missing-proofs && + ( + cd mixed-writer-missing-proofs && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSMN .git/index && + test_grep FSUC .git/index && + test_grep FSCF .git/index && + cat >.git/remove-mixed-writer-proofs.pl <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my $algorithm = $ARGV[0]; + my $rawsz = $algorithm eq "sha256" ? 32 : 20; + for my $name ("FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + substr($index, $offset, 8 + $size, ""); + } + my $payload = substr($index, 0, -$rawsz); + print $payload, + $algorithm eq "sha256" ? sha256($payload) : sha1($payload); + EOF + perl .git/remove-mixed-writer-proofs.pl "$(test_oid algo)" \ + <.git/index >.git/index.mixed && + mv .git/index.mixed .git/index && + test_grep FSMN .git/index && + test_grep UNTR .git/index && + test_grep ! FSUC .git/index && + test_grep ! FSCF .git/index && + cp .git/index .git/cold.before && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/cold.trace" \ + git status --porcelain=v2 >.git/cold && + test_must_be_empty .git/cold && + test_cmp_bin .git/cold.before .git/index && + ! test_region index do_write_index .git/cold.trace && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/repair.trace" \ + git status >.git/repair && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/repair.trace && + test_region index do_write_index .git/repair.trace && + test_grep FSMN .git/index && + test_grep FSUC .git/index && + test_grep FSCF .git/index && + cat >.git/check-mixed-writer-proof.pl <<-\EOF && + binmode STDIN; + local $/; + my $index = ; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched provider tokens\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"} && + $tokens{"FSMN"} eq $tokens{"FSCF"}; + EOF + perl .git/check-mixed-writer-proof.pl <.git/index && + for run in first second + do + cp .git/index ".git/readonly-$run.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/readonly-$run.trace" \ + git status --porcelain=v2 >".git/readonly-$run" && + test_must_be_empty ".git/readonly-$run" && + test_cmp_bin ".git/readonly-$run.index" .git/index && + test_trace2_data fsmonitor config/coherent 1 \ + <".git/readonly-$run.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <".git/readonly-$run.trace" && + ! test_region index do_write_index \ + ".git/readonly-$run.trace" || return 1 + done + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'ordinary removals and renames preserve safe worktree proofs' ' + test_when_finished "rm -rf rm-mv-bound-proof rm-mv-linked" && + test_create_repo rm-mv-bound-proof && + ( + cd rm-mv-bound-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines remove >remove-me && + test_write_lines move >move-me && + test_write_lines sibling >sibling && + test_write_lines "*.asset text" >.gitattributes && + test_write_lines "*.ignored" >.gitignore && + git add remove-me move-me sibling .gitattributes .gitignore && + git commit -qm base && + test_write_lines successor >sibling && + git add sibling && + git commit -qm successor && + git worktree add --detach ../rm-mv-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + cat >.git/check-rm-mv-proof.pl <<-\EOF && + binmode STDIN; + local $/; + my $index = ; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched provider tokens\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"} && + $tokens{"FSMN"} eq $tokens{"FSCF"}; + EOF + for worktree in "$PWD" "$PWD/../rm-mv-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + for operation in remove rename mixed-reset + do + case "$operation" in + remove) set -- rm --quiet remove-me ;; + rename) set -- mv move-me renamed ;; + mixed-reset) set -- reset --mixed HEAD~1 ;; + esac && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$operation.trace" \ + git -C "$worktree" "$@" && + perl "$PWD/.git/check-rm-mv-proof.pl" \ + <"$gitdir/index" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/$operation.trace" && + cp "$gitdir/index" "$gitdir/$operation.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$operation-status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$operation-status" && + test_cmp_bin "$gitdir/$operation.index" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$operation-status.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$operation-status.trace" || return 1 + done && + test_write_lines exposed >"$worktree/hidden.ignored" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" rm --quiet .gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/ignore-removed" && + test_grep "^? hidden\\.ignored$" \ + "$gitdir/ignore-removed" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" mv .gitattributes \ + moved.attributes && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/attributes.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/attributes" && + test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/attributes.trace" && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/attributes.trace" || return 1 + done && + test_write_lines "*.filtered filter=demo" >.gitattributes && + git config filter.demo.clean cat && + git config filter.demo.required true && + test_write_lines raw >active.filtered && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git add .gitattributes active.filtered && + git config filter.demo.clean false && + test_must_fail env GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 --untracked-files=no \ + >.git/filtered 2>.git/filter-error && + test_grep "clean filter .demo. failed" .git/filter-error + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'configured pulls preserve authenticated worktree proofs' ' + test_when_finished "rm -rf pull-proof-origin.git pull-proof-seed pull-proof-ff pull-proof-ff-linked pull-proof-rebase pull-proof-rebase-linked pull-proof-autostash pull-proof-autostash-linked" && + git init --bare pull-proof-origin.git && + test_create_repo pull-proof-seed && + ( + cd pull-proof-seed && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git branch -M main && + git remote add origin "$PWD/../pull-proof-origin.git" && + git push --quiet -u origin main && + git --git-dir="$PWD/../pull-proof-origin.git" \ + symbolic-ref HEAD refs/heads/main && + cat >.git/check-pull-proof.pl <<-\EOF && + binmode STDIN; + local $/; + my $index = ; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched provider tokens\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"} && + $tokens{"FSMN"} eq $tokens{"FSCF"}; + EOF + for mode in ff rebase autostash + do + git clone --quiet "$PWD/../pull-proof-origin.git" \ + "$PWD/../pull-proof-$mode" && + repo="$PWD/../pull-proof-$mode" && + linked="$PWD/../pull-proof-$mode-linked" && + git -C "$repo" worktree add --quiet \ + -b "linked-$mode" "$linked" origin/main && + git -C "$linked" branch --quiet \ + --set-upstream-to=origin/main && + if test "$mode" = ff + then + git -C "$repo" config pull.ff only + else + git -C "$repo" config pull.rebase true + fi && + git -C "$repo" config core.untrackedCache true && + git -C "$repo" config core.fsmonitor true && + for role in main linked + do + case "$role" in + main) worktree="$repo" ;; + linked) worktree="$linked" ;; + esac && + if test "$mode" != ff + then + test_write_lines "$mode-$role" \ + >"$worktree/local-$role" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" add "local-$role" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" commit \ + -qm "local-$mode-$role" || return 1 + fi && + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + perl "$PWD/.git/check-pull-proof.pl" \ + <"$gitdir/index" && + test_write_lines "$mode-$role" \ + >"upstream-$mode-$role" && + git add "upstream-$mode-$role" && + git commit -qm "upstream-$mode-$role" && + git push --quiet origin main && + if test "$mode" = autostash + then + test_write_lines dirty >"$worktree/tracked" && + set -- pull --quiet --rebase --autostash + else + set -- pull --quiet + fi && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/pull.trace" \ + git -C "$worktree" "$@" \ + >"$gitdir/pull" && + if test "$mode" != ff + then + test_grep "\"name\":\"rebase\"" \ + "$gitdir/pull.trace" && + test_path_is_file "$worktree/local-$role" || + return 1 + fi && + ! test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/pull.trace" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/pull.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/pull.trace" && + perl "$PWD/.git/check-pull-proof.pl" \ + <"$gitdir/index" && + cp "$gitdir/index" "$gitdir/readonly.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/status" && + if test "$mode" = autostash + then + test_grep "^1 \\.M .* tracked$" \ + "$gitdir/status" || return 1 + else + test_must_be_empty "$gitdir/status" || return 1 + fi && + test_cmp_bin "$gitdir/readonly.index" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/status.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/status.trace" || return 1 + done || return 1 + done + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'expired add preserves untracked candidates until revalidation' ' test_when_finished "rm -rf pending-untracked-revalidation pending-untracked-hostile" && @@ -1533,7 +2100,7 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ' test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ - 'diff closes reset fsmonitor tokens in main and linked worktrees' ' + 'diff preserves pending reset proofs in main and linked worktrees' ' test_when_finished "rm -rf builtin-diff-reset builtin-diff-reset-linked" && test_create_repo builtin-diff-reset && ( @@ -1578,12 +2145,29 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_must_be_empty "$gitdir/reset.actual" && test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/reset.trace" && - test_trace2_data fsmonitor token_closure/accepted 1 \ + test_trace2_data fsmonitor \ + untracked/provider-reset-pending 1 \ <"$gitdir/reset.trace" && test_region index do_write_index "$gitdir/reset.trace" && test_grep FSMN "$gitdir/index" && - test_grep ! FSUC "$gitdir/index" && - test_grep "builtin:test:[2-9]" "$gitdir/index" && + test_grep FSUC "$gitdir/index" && + test_grep "pending:test:[2-9]" "$gitdir/index" && + test_fsmonitor_pending_full_proof "$gitdir/index" && + cp "$gitdir/index" "$gitdir/pending.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/pending-status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/pending-status" && + test_must_be_empty "$gitdir/pending-status" && + test_cmp_bin "$gitdir/pending.index" "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/pending-status.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/pending-status.trace" && + ! test_region index do_write_index \ + "$gitdir/pending-status.trace" && GIT_TEST_PRELOAD_INDEX=1 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ GIT_TRACE2_EVENT="$gitdir/next.trace" \ @@ -3063,6 +3647,301 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'update-index doctor permutations retain authenticated proofs' ' + test_when_finished "rm -rf doctor-update-proof doctor-update-linked" && + test_create_repo doctor-update-proof && + ( + cd doctor-update-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git worktree add --detach ../doctor-update-linked HEAD && + test-tool chmtime -120 tracked \ + ../doctor-update-linked/tracked && + git update-index --refresh && + git -C ../doctor-update-linked update-index --refresh && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + cat >.git/remove-doctor-proofs.pl <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my $rawsz = $ARGV[0] eq "sha256" ? 32 : 20; + for my $name ("FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + substr($index, $offset, 8 + $size, ""); + } + my $payload = substr($index, 0, -$rawsz); + print $payload, $rawsz == 32 ? sha256($payload) : sha1($payload); + EOF + for worktree in "$PWD" "$PWD/../doctor-update-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/checkpoint.trace" \ + git -C "$worktree" status --short \ + >"$gitdir/checkpoint" && + test_must_be_empty "$gitdir/checkpoint" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/checkpoint.trace" && + find "$gitdir" -maxdepth 1 -type f \ + -name "index.csh1.*" >"$gitdir/checkpoints" && + test_line_count = 1 "$gitdir/checkpoints" && + if test_have_prereq MACOS + then + find "$gitdir" -maxdepth 1 -type f \ + -name "index.cswi.*" >"$gitdir/witnesses" && + test_line_count = 1 "$gitdir/witnesses" || return 1 + fi && + for mode in healthy history + do + for order in normal reverse + do + if test "$mode" = history + then + perl "$PWD/.git/remove-doctor-proofs.pl" \ + "$(test_oid algo)" <"$gitdir/index" \ + >"$gitdir/index.foreign" && + mv "$gitdir/index.foreign" \ + "$gitdir/index" && + test_grep ! FSUC "$gitdir/index" && + test_grep ! FSCF "$gitdir/index" || return 1 + fi && + if test "$order" = normal + then + set -- --untracked-cache --force-write-index + else + set -- --force-write-index --untracked-cache + fi && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode-$order.trace" \ + git -C "$worktree" update-index "$@" && + test_grep FSUC "$gitdir/index" && + test_grep FSCF "$gitdir/index" && + ! test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/$mode-$order.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$mode-$order.trace" || return 1 + done || return 1 + done || return 1 + done + ) +' + +test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'provider restarts keep diff and untracked status correct' ' + test_when_finished "rm -rf daemon-diff-reset daemon-diff-linked" && + test_when_finished \ + "git -C daemon-diff-reset fsmonitor--daemon stop 2>/dev/null || :" && + test_when_finished \ + "git -C daemon-diff-linked fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo daemon-diff-reset && + ( + cd daemon-diff-reset && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/deep && + test_commit base cached/deep/tracked && + git worktree add --detach ../daemon-diff-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../daemon-diff-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + git -C "$worktree" fsmonitor--daemon start \ + --start-timeout=10 && + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + test_grep FSMN "$gitdir/index" && + test_grep FSUC "$gitdir/index" && + test_grep FSCF "$gitdir/index" && + git -C "$worktree" fsmonitor--daemon stop && + test-tool chmtime =-60 \ + "$worktree/cached/deep/tracked" && + git -C "$worktree" fsmonitor--daemon start \ + --start-timeout=10 && + GIT_TRACE2_EVENT="$gitdir/diff.trace" \ + git -C "$worktree" diff >"$gitdir/diff" && + test_must_be_empty "$gitdir/diff" && + test_trace2_data fsm_client query/trivial-response 1 \ + <"$gitdir/diff.trace" && + test_trace2_data fsmonitor \ + untracked/provider-reset-pending 1 \ + <"$gitdir/diff.trace" && + test_fsmonitor_pending_full_proof "$gitdir/index" && + cp "$gitdir/index" "$gitdir/readonly.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$gitdir/readonly.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/readonly" && + test_must_be_empty "$gitdir/readonly" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/readonly.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/readonly.trace" && + ! test_region index do_write_index \ + "$gitdir/readonly.trace" && + git -C "$worktree" fsmonitor--daemon stop && + test_write_lines hidden \ + >"$worktree/cached/deep/hidden-during-restart" && + test_write_lines "tracked -text" \ + >"$worktree/cached/deep/.gitattributes" && + test-tool chmtime =-30 \ + "$worktree/cached/deep/tracked" && + git -C "$worktree" fsmonitor--daemon start \ + --start-timeout=10 && + git -C "$worktree" diff >"$gitdir/hidden-diff" && + test_must_be_empty "$gitdir/hidden-diff" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" -c core.untrackedCache=false \ + status --porcelain=v2 >"$gitdir/hidden.expect" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/hidden.actual" && + test_cmp "$gitdir/hidden.expect" \ + "$gitdir/hidden.actual" && + test_grep "^? cached/deep/hidden-during-restart$" \ + "$gitdir/hidden.actual" && + test_grep "^? cached/deep/\\.gitattributes$" \ + "$gitdir/hidden.actual" && + git -C "$worktree" fsmonitor--daemon stop || return 1 + done + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'tracked-only status never authenticates stale untracked history' ' + test_when_finished "rm -rf tracked-only-reset tracked-only-linked" && + test_create_repo tracked-only-reset && + ( + cd tracked-only-reset && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/deep && + test_commit base cached/deep/tracked && + git worktree add --detach ../tracked-only-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../tracked-only-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + test_grep FSUC "$gitdir/index" && + test_write_lines unexpected \ + >"$worktree/cached/deep/new-untracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/tracked-only.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no >"$gitdir/tracked-only" && + test_must_be_empty "$gitdir/tracked-only" && + test_trace2_data fsm_client query/trivial-response 1 \ + <"$gitdir/tracked-only.trace" && + test_grep ! FSUC "$gitdir/index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" -c core.untrackedCache=false \ + status --porcelain=v2 >"$gitdir/expect" && + test_grep "^? cached/deep/new-untracked$" \ + "$gitdir/expect" && + cp "$gitdir/index" "$gitdir/readonly.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/readonly" && + test_cmp "$gitdir/expect" "$gitdir/readonly" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/writable" && + test_cmp "$gitdir/expect" "$gitdir/writable" || return 1 + done + ) +' + +test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'tracked-only status preserves new files after daemon restart' ' + test_when_finished "rm -rf daemon-tracked-only daemon-tracked-only-linked" && + test_when_finished \ + "git -C daemon-tracked-only fsmonitor--daemon stop 2>/dev/null || :" && + test_when_finished \ + "git -C daemon-tracked-only-linked fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo daemon-tracked-only && + ( + cd daemon-tracked-only && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/deep && + test_commit base cached/deep/tracked && + git worktree add --detach ../daemon-tracked-only-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../daemon-tracked-only-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + git -C "$worktree" fsmonitor--daemon start \ + --start-timeout=10 && + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + test_grep FSUC "$gitdir/index" && + git -C "$worktree" fsmonitor--daemon stop && + test_write_lines unexpected \ + >"$worktree/cached/deep/new-untracked" && + git -C "$worktree" fsmonitor--daemon start \ + --start-timeout=10 && + GIT_TRACE2_EVENT="$gitdir/tracked-only.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no >"$gitdir/tracked-only" && + test_must_be_empty "$gitdir/tracked-only" && + test_trace2_data fsm_client query/trivial-response 1 \ + <"$gitdir/tracked-only.trace" && + test_grep ! FSUC "$gitdir/index" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" -c core.untrackedCache=false \ + status --porcelain=v2 >"$gitdir/expect" && + test_grep "^? cached/deep/new-untracked$" \ + "$gitdir/expect" && + cp "$gitdir/index" "$gitdir/readonly.index" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/readonly" && + test_cmp "$gitdir/expect" "$gitdir/readonly" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/writable" && + test_cmp "$gitdir/expect" "$gitdir/writable" && + git -C "$worktree" fsmonitor--daemon stop || return 1 + done + ) +' + test_expect_success PTHREADS,UNTRACKED_CACHE,SHA1 'load cache-tree and untracked-cache extensions in parallel' ' test_create_repo parallel-extensions && ( diff --git a/wt-status.c b/wt-status.c index 945693a3abbb2f..e7356d3b5e29e3 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2226,6 +2226,7 @@ static int wt_status_close_fsmonitor_token( (!require_untracked && (s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || s->show_ignored_mode) && + istate->fsmonitor_untracked_valid && istate->fsmonitor_untracked_token && istate->fsmonitor_last_update && !strcmp(istate->fsmonitor_untracked_token, From 1565e9d2751a4779abe38b48cc03e8b482e2b9c3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 01:35:22 -0500 Subject: [PATCH 04/73] t7527: expect safe mixed resets to preserve clean history A mixed reset now authenticates every changed index entry before keeping its clean-status history. Updating an ordinary tracked file is safe, so require a coherent proof and no full attribute-manifest scan while still verifying that the worktree modification is reported. --- t/t7527-builtin-fsmonitor.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 4aaf8da578136b..e90a726eb31d7a 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -5859,7 +5859,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ' test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ - 'mixed reset drops history after a logical index change' ' + 'mixed reset preserves history across safe tracked content changes' ' test_when_finished "rm -rf reset-mixed-changed" && test_create_repo reset-mixed-changed && ( @@ -5889,7 +5889,9 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ git status >.git/actual && test_grep "modified:.*tracked" .git/actual && - test_trace2_data fsmonitor config/coherent 0 \ + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <.git/status.trace ) ' From e3c9a5b818ccbcd65e61f9352ceee95990599739 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 02:08:20 -0500 Subject: [PATCH 05/73] fsmonitor: preserve proofs through filters, replay, and index locks A configured LFS filter is not necessarily active for any tracked path. Bind the normal clean-status configuration before diff, but retain the existing rule against loading sidecar history for configured filters. Treat Git's actual primary index.lock as a physical index when a pre-commit hook runs write-tree. Keep arbitrary temporary indexes, symlinks, and hardlink aliases on their existing isolated paths. Initialize authenticated history for native-fsmonitor cherry-pick and revert operations, while leaving hook providers and unsupported systems untouched. After a daemon reset, preserve tracked-only status directory snapshots only as pending candidates. Require an authenticated complete tracked proof, exclude scoped, sparse, and alternate cases, and validate every directory before its untracked entries can be trusted. Cover inactive LFS, active required filters, generating commit-a hooks, replay conflicts, daemon restarts, offline untracked files, and repeated immutable read-only statuses in primary and linked SHA-1/SHA-256 trees. --- builtin/diff.c | 6 +- builtin/revert.c | 10 ++ builtin/write-tree.c | 13 ++- t/t7519-status-fsmonitor.sh | 212 ++++++++++++++++++++++++++++++++---- wt-status.c | 32 +++++- 5 files changed, 242 insertions(+), 31 deletions(-) diff --git a/builtin/diff.c b/builtin/diff.c index 35f9a17f59f853..2d9d6de62ecce6 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -453,11 +453,11 @@ void prepare_diff_external_history(struct repository *repo) goto done; worktree = get_current_worktree(repo); if (!worktree || - clean_status_config_read_repository(repo, &digest) || - digest.filter_configured) + clean_status_config_read_repository(repo, &digest)) goto done; clean_status_set_config_digest(repo, &digest); - clean_status_enable_external_history(repo); + if (!digest.filter_configured) + clean_status_enable_external_history(repo); done: free_worktree(worktree); diff --git a/builtin/revert.c b/builtin/revert.c index bedc40f368eccc..ac8968f3558d8d 100644 --- a/builtin/revert.c +++ b/builtin/revert.c @@ -2,9 +2,12 @@ #include "git-compat-util.h" #include "builtin.h" +#include "clean-status-config.h" +#include "clean-status.h" #include "parse-options.h" #include "diff.h" #include "environment.h" +#include "fsmonitor-settings.h" #include "gettext.h" #include "revision.h" #include "rerere.h" @@ -115,6 +118,7 @@ static int run_sequencer(int argc, const char **argv, const char *prefix, const char sentinel_value = 0; /* value not important */ const char *strategy = &sentinel_value; const char *gpg_sign = &sentinel_value; + struct clean_status_config_digest clean_digest; enum empty_action empty_opt = EMPTY_COMMIT_UNSPECIFIED; int cmd = 0; struct option base_options[] = { @@ -172,6 +176,12 @@ static int run_sequencer(int argc, const char **argv, const char *prefix, argc = parse_options(argc, argv, prefix, options, usage_str, PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN_OPT); + if (fstat_is_reliable() && !getenv(INDEX_ENVIRONMENT) && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC && + !clean_status_config_read_repository(the_repository, &clean_digest)) { + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, &clean_digest); + } prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/write-tree.c b/builtin/write-tree.c index 338e4565c2fc29..bb04b01968d068 100644 --- a/builtin/write-tree.c +++ b/builtin/write-tree.c @@ -34,22 +34,27 @@ static int write_tree_uses_worktree_index(void) const char *index_file = getenv(INDEX_ENVIRONMENT); struct strbuf worktree_index = STRBUF_INIT; struct stat st; - char *expected = NULL, *actual = NULL; + char *expected = NULL, *expected_lock = NULL, *actual = NULL; int matches = 0; if (!index_file) return 1; - if (lstat(index_file, &st) || S_ISLNK(st.st_mode)) + if (lstat(index_file, &st) || !S_ISREG(st.st_mode)) return 0; strbuf_addf(&worktree_index, "%s/index", repo_get_git_dir(the_repository)); expected = real_pathdup(worktree_index.buf, 0); actual = real_pathdup(index_file, 0); - if (expected && actual && !strcmp(expected, actual)) - matches = 1; + if (expected && actual) { + expected_lock = xstrfmt("%s.lock", expected); + if (!strcmp(expected, actual) || + !strcmp(expected_lock, actual)) + matches = 1; + } free(expected); + free(expected_lock); free(actual); strbuf_release(&worktree_index); return matches; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index b384e03fe71e0d..8ae4262d296408 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -598,8 +598,8 @@ prepare_builtin_closure_repo () { ) } -test_fsmonitor_pending_full_proof () { - perl - "$1" <<-\EOF +test_fsmonitor_full_proof () { + perl - "$1" "$2" <<-\EOF binmode STDIN; open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; binmode $input; @@ -626,8 +626,10 @@ test_fsmonitor_pending_full_proof () { $tokens{"FSMN"} eq $tokens{"FSCF"}; my ($suffix) = $tokens{"FSMN"} =~ /\Abuiltin:(.+)\z/; die "missing provider token\n" unless defined $suffix; - die "mismatched pending untracked token\n" unless - $tokens{"FSUC"} eq "pending:$suffix"; + my $untracked = $ARGV[1] eq "pending" ? + "pending:$suffix" : $tokens{"FSMN"}; + die "mismatched untracked token\n" unless + $tokens{"FSUC"} eq $untracked; EOF } @@ -1067,6 +1069,12 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ EOF write_script .git/hooks/pre-commit <<-\EOF && test -n "$GIT_INDEX_FILE" || exit 1 + if test -n "${HOOK_GENERATE-}" + then + test "$GIT_INDEX_FILE" = "$HOOK_EXPECT_INDEX" || exit 1 + printf "%s\n" generated >"$HOOK_GENERATE" || exit 1 + git add -- "$HOOK_GENERATE" || exit 1 + fi git write-tree >"$HOOK_PROOF_OUTPUT" || exit 1 perl "$HOOK_PROOF_HELPER" <"$GIT_INDEX_FILE" EOF @@ -1150,6 +1158,36 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ "$gitdir/hook.trace" && perl "$PWD/.git/check-write-tree-proof.pl" \ <"$gitdir/index" && + test_write_lines commit-all >"$worktree/sibling" && + HOOK_GENERATE=hook-generated \ + HOOK_EXPECT_INDEX="$gitdir/index.lock" \ + HOOK_PROOF_HELPER="$PWD/.git/check-write-tree-proof.pl" \ + HOOK_PROOF_OUTPUT="$gitdir/hook-all-tree" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=sibling \ + GIT_TRACE2_EVENT="$gitdir/hook-all.trace" \ + git -C "$worktree" commit -aqm commit-all && + test_file_not_empty "$gitdir/hook-all-tree" && + git -C "$worktree" ls-tree HEAD hook-generated \ + >"$gitdir/hook-all-entry" && + test_grep "hook-generated$" "$gitdir/hook-all-entry" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/hook-all.trace" && + perl "$PWD/.git/check-write-tree-proof.pl" \ + <"$gitdir/index" && + cp "$gitdir/index" "$gitdir/hook-all-before-status" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/hook-all-status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/hook-all-status" && + test_must_be_empty "$gitdir/hook-all-status" && + test_cmp_bin "$gitdir/hook-all-before-status" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/hook-all-status.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/hook-all-status.trace" && cp "$gitdir/index" "$gitdir/readonly.index" && cp "$gitdir/index" "$gitdir/snapshot.index" && test_write_lines snapshot >"$worktree/snapshot-new" && @@ -1684,6 +1722,86 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'clean sequencer operations preserve authenticated worktree proofs' ' + test_when_finished "rm -rf sequencer-proof sequencer-linked" && + test_create_repo sequencer-proof && + ( + cd sequencer-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_write_lines picked >tracked && + git add tracked && + git commit -qm picked && + picked=$(git rev-parse HEAD) && + git reset --hard HEAD^ && + git worktree add --detach ../sequencer-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../sequencer-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + for operation in pick revert + do + case "$operation" in + pick) set -- cherry-pick --no-edit "$picked" ;; + revert) set -- revert --no-edit HEAD ;; + esac && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/$operation.trace" \ + git -C "$worktree" "$@" \ + >"$gitdir/$operation" && + test_fsmonitor_full_proof \ + "$gitdir/index" paired && + cp "$gitdir/index" \ + "$gitdir/$operation.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$operation-status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$operation-status" && + test_must_be_empty "$gitdir/$operation-status" && + test_cmp_bin "$gitdir/$operation.index" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$operation-status.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$operation-status.trace" || return 1 + done || return 1 + done && + test_write_lines conflicting >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git commit -qm local-conflict && + test_must_fail env \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git cherry-pick --no-edit "$picked" \ + >.git/conflict.out 2>.git/conflict.err && + if test_fsmonitor_full_proof .git/index paired \ + >/dev/null 2>&1 + then + return 1 + fi && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/conflict && + test_grep "^u UU .* tracked$" .git/conflict + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'expired add preserves untracked candidates until revalidation' ' test_when_finished "rm -rf pending-untracked-revalidation pending-untracked-hostile" && @@ -2112,6 +2230,10 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ git config core.fsmonitor true && git -c core.fsmonitor=false worktree add --detach \ ../builtin-diff-reset-linked HEAD && + git config filter.lfs.clean "git-lfs clean -- %f" && + git config filter.lfs.smudge "git-lfs smudge -- %f" && + git config filter.lfs.process "git-lfs filter-process" && + git config filter.lfs.required true && for worktree in "$PWD" "$PWD/../builtin-diff-reset-linked" do gitdir=$(git -C "$worktree" \ @@ -2152,7 +2274,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_grep FSMN "$gitdir/index" && test_grep FSUC "$gitdir/index" && test_grep "pending:test:[2-9]" "$gitdir/index" && - test_fsmonitor_pending_full_proof "$gitdir/index" && + test_fsmonitor_full_proof "$gitdir/index" pending && cp "$gitdir/index" "$gitdir/pending.index" && GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ @@ -2176,7 +2298,17 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_must_be_empty "$gitdir/next.actual" && test_trace2_data index preload/sum_lstat 0 \ <"$gitdir/next.trace" || return 1 - done + done && + git config filter.lfs.process "" && + git config filter.lfs.clean false && + test_write_lines "tracked filter=lfs" >.gitattributes && + cp .git/index .git/filtered.index && + test_must_fail env GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + git diff >.git/filtered.out 2>.git/filtered.err && + test_grep "clean filter .lfs. failed" .git/filtered.err && + test_cmp_bin .git/filtered.index .git/index ) ' @@ -3758,6 +3890,10 @@ test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OP git worktree add --detach ../daemon-diff-linked HEAD && git config core.untrackedCache true && git config core.fsmonitor true && + git config filter.lfs.clean "git-lfs clean -- %f" && + git config filter.lfs.smudge "git-lfs smudge -- %f" && + git config filter.lfs.process "git-lfs filter-process" && + git config filter.lfs.required true && for worktree in "$PWD" "$PWD/../daemon-diff-linked" do gitdir=$(git -C "$worktree" \ @@ -3785,7 +3921,7 @@ test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OP test_trace2_data fsmonitor \ untracked/provider-reset-pending 1 \ <"$gitdir/diff.trace" && - test_fsmonitor_pending_full_proof "$gitdir/index" && + test_fsmonitor_full_proof "$gitdir/index" pending && cp "$gitdir/index" "$gitdir/readonly.index" && GIT_OPTIONAL_LOCKS=0 \ GIT_TRACE2_EVENT="$gitdir/readonly.trace" \ @@ -3861,20 +3997,36 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_must_be_empty "$gitdir/tracked-only" && test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/tracked-only.trace" && - test_grep ! FSUC "$gitdir/index" && + test_trace2_data fsmonitor \ + untracked/provider-reset-pending 1 \ + <"$gitdir/tracked-only.trace" && + test_fsmonitor_full_proof "$gitdir/index" pending && GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ git -C "$worktree" -c core.untrackedCache=false \ status --porcelain=v2 >"$gitdir/expect" && test_grep "^? cached/deep/new-untracked$" \ "$gitdir/expect" && - cp "$gitdir/index" "$gitdir/readonly.index" && - GIT_OPTIONAL_LOCKS=0 \ - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ - git -C "$worktree" status --porcelain=v2 \ - >"$gitdir/readonly" && - test_cmp "$gitdir/expect" "$gitdir/readonly" && - test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + for pass in first second + do + cp "$gitdir/index" "$gitdir/readonly-$pass.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/readonly-$pass.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/readonly-$pass" && + test_cmp "$gitdir/expect" \ + "$gitdir/readonly-$pass" && + test_cmp_bin "$gitdir/readonly-$pass.index" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/readonly-$pass.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/readonly-$pass.trace" && + ! test_region index do_write_index \ + "$gitdir/readonly-$pass.trace" || return 1 + done && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ git -C "$worktree" status --porcelain=v2 \ >"$gitdir/writable" && @@ -3922,18 +4074,34 @@ test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OP test_must_be_empty "$gitdir/tracked-only" && test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/tracked-only.trace" && - test_grep ! FSUC "$gitdir/index" && + test_trace2_data fsmonitor \ + untracked/provider-reset-pending 1 \ + <"$gitdir/tracked-only.trace" && + test_fsmonitor_full_proof "$gitdir/index" pending && GIT_OPTIONAL_LOCKS=0 \ git -C "$worktree" -c core.untrackedCache=false \ status --porcelain=v2 >"$gitdir/expect" && test_grep "^? cached/deep/new-untracked$" \ "$gitdir/expect" && - cp "$gitdir/index" "$gitdir/readonly.index" && - GIT_OPTIONAL_LOCKS=0 \ - git -C "$worktree" status --porcelain=v2 \ - >"$gitdir/readonly" && - test_cmp "$gitdir/expect" "$gitdir/readonly" && - test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + for pass in first second + do + cp "$gitdir/index" "$gitdir/readonly-$pass.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$gitdir/readonly-$pass.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/readonly-$pass" && + test_cmp "$gitdir/expect" \ + "$gitdir/readonly-$pass" && + test_cmp_bin "$gitdir/readonly-$pass.index" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/readonly-$pass.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/readonly-$pass.trace" && + ! test_region index do_write_index \ + "$gitdir/readonly-$pass.trace" || return 1 + done && git -C "$worktree" status --porcelain=v2 \ >"$gitdir/writable" && test_cmp "$gitdir/expect" "$gitdir/writable" && diff --git a/wt-status.c b/wt-status.c index e7356d3b5e29e3..c24c42f71350c1 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2168,8 +2168,21 @@ static int wt_status_close_fsmonitor_token( .staged_ignored = STRING_LIST_INIT_DUP, }; enum wt_status_token_closure_result result; + int preserve_untracked, token_accepted = 0; refresh_fsmonitor(istate); + preserve_untracked = !require_untracked && + s->show_untracked_files == SHOW_NO_UNTRACKED_FILES && + !s->show_ignored_mode && !s->pathspec.nr && + fstat_is_reliable() && !getenv(INDEX_ENVIRONMENT) && + istate == istate->repo->index && !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_IPC && + istate->fsmonitor_untracked_extension_seen && + !istate->fsmonitor_untracked_extension_invalid && + istate->untracked && istate->untracked->root && + istate->untracked->root->valid && + istate->untracked->fsmonitor_revalidation; if (!fsmonitor_has_pending_token(istate) || fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC) { int attr_inputs_match = @@ -2246,8 +2259,10 @@ static int wt_status_close_fsmonitor_token( if (proof) { result = wt_status_close_semantic_fsmonitor_token( &closure, &proof); - if (result == WT_STATUS_TOKEN_CLOSURE_ACCEPTED) + if (result == WT_STATUS_TOKEN_CLOSURE_ACCEPTED) { + token_accepted = 1; goto accepted; + } if (result == WT_STATUS_TOKEN_CLOSURE_FALLBACK) goto fallback; wt_status_reset_attr_snapshot_if_changed(s); @@ -2256,8 +2271,10 @@ static int wt_status_close_fsmonitor_token( } if (wt_status_close_ordinary_fsmonitor_token( - &closure, refreshed_before_closure)) + &closure, refreshed_before_closure)) { + token_accepted = 1; goto accepted; + } /* Keep the last valid token and fall back to complete scans. */ fallback: @@ -2274,6 +2291,17 @@ static int wt_status_close_fsmonitor_token( closure.refresh_result |= refresh_index( istate, refresh_flags, &s->pathspec, NULL, NULL); accepted: + if (token_accepted && preserve_untracked && + !istate->fsmonitor_untracked_valid && + istate->untracked->root && istate->untracked->root->valid && + clean_status_revalidated_token_matches(istate) && + !clean_status_filter_scope_needs_validation(istate)) { + /* Tracked closure leaves directory snapshots unverified. */ + istate->untracked->fsmonitor_revalidation = 1; + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + clean_status_begin_fsmonitor_semantic_baseline(istate); + } wt_status_publish_staged_untracked(&closure); wt_status_discard_staged_untracked(&closure); trace2_region_leave("status", "fsmonitor_token_closure", s->repo); From 136d1c0f1e4dd9e460e353ad3eec4e0cad9c7d7f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 02:27:09 -0500 Subject: [PATCH 06/73] t7527: accept either authenticated legacy checkpoint restore An intervening status may refresh the external checkpoint after a legacy writer stages a new directory. A later legacy unstaging can then restore the same logical index directly or recover both its authenticated semantic and untracked history across the changed index. Accept either complete authenticated restore path while retaining the exact status oracle and the existing no-bulk-traversal assertions. --- t/t7527-builtin-fsmonitor.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index e90a726eb31d7a..6b4c7c11a8b913 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -4147,8 +4147,18 @@ test_expect_success MACOS,LEGACY_PREVIEW_FSMONITOR_GIT,UNTRACKED_CACHE,SEMANTIC_ GIT_TRACE2_EVENT="$PWD/.git/foreign-unstaged.trace" \ git status --porcelain=v2 >.git/unstaged.actual && test_cmp .git/unstaged.expect .git/unstaged.actual && - test_trace2_data fsmonitor history/external-restored 1 \ - <.git/foreign-unstaged.trace && + { + test_trace2_data fsmonitor history/external-restored 1 \ + <.git/foreign-unstaged.trace || + { + test_trace2_data fsmonitor \ + history/external-semantic-restored 1 \ + <.git/foreign-unstaged.trace && + test_trace2_data fsmonitor \ + history/external-untracked-restored 1 \ + <.git/foreign-unstaged.trace + } + } && ! test_trace2_data index preload/bulk_useful \ "[1-9][0-9]*" <.git/foreign-unstaged.trace && ! test_trace2_data index preload/bulk_dirs \ From 57b3c9021c3d11ac4598087cdf419d6f5fc8c3c3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 02:37:25 -0500 Subject: [PATCH 07/73] diff: restore authenticated history with configured filters A configured but inactive filter must not prevent diff commands from restoring authenticated external clean history. Otherwise linked indexes rewritten by older Git repeatedly rescan every tracked path, even when their matching checkpoint remains intact. Always enable external history after binding the repository configuration. Checkpoint restoration already validates configuration, attributes, provider tokens, worktree identity, and active filter scope. Exercise diff, diff-files, and diff-index against a linked worktree with an inactive required LFS filter and missing physical proof extensions. Also verify that activating the filter invalidates the checkpoint and fails safely. --- builtin/diff.c | 3 +- t/t7519-status-fsmonitor.sh | 63 ++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/builtin/diff.c b/builtin/diff.c index 2d9d6de62ecce6..d2e4e2297f3808 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -456,8 +456,7 @@ void prepare_diff_external_history(struct repository *repo) clean_status_config_read_repository(repo, &digest)) goto done; clean_status_set_config_digest(repo, &digest); - if (!digest.filter_configured) - clean_status_enable_external_history(repo); + clean_status_enable_external_history(repo); done: free_worktree(worktree); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 8ae4262d296408..40c5d8418733f9 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3689,6 +3689,13 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO git -C "$worktree" config core.autocrlf false && git -C "$worktree" config core.untrackedCache true && git -C "$worktree" config core.fsmonitor true && + git -C "$worktree" config filter.lfs.clean \ + "git-lfs clean -- %f" && + git -C "$worktree" config filter.lfs.smudge \ + "git-lfs smudge -- %f" && + git -C "$worktree" config filter.lfs.process \ + "git-lfs filter-process" && + git -C "$worktree" config filter.lfs.required true && git -C "$worktree" config index.recordEndOfIndexEntries false && test-tool chmtime -120 \ "$worktree/existing/tracked" "$worktree/existing/sibling" && @@ -3735,6 +3742,43 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO test_grep UNTR "$gitdir/index" && test_grep ! FSUC "$gitdir/index" && test_grep ! FSCF "$gitdir/index" && + for command in diff diff-files diff-index + do + case "$command" in + diff-index) set -- "$command" HEAD ;; + *) set -- "$command" ;; + esac && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" -c core.fsmonitor=false \ + -c core.untrackedCache=false "$@" \ + >"$gitdir/$command.expect" && + cp "$gitdir/index" "$gitdir/$command.before" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$gitdir/$command.trace" \ + git -C "$worktree" "$@" \ + >"$gitdir/$command.actual" && + test_cmp "$gitdir/$command.expect" \ + "$gitdir/$command.actual" && + test_cmp_bin "$gitdir/$command.before" "$gitdir/index" && + { + test_trace2_data fsmonitor history/external-restored 1 \ + <"$gitdir/$command.trace" || + test_trace2_data fsmonitor \ + history/external-semantic-restored 1 \ + <"$gitdir/$command.trace" + } && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$command.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$command.trace" && + ! test_trace2_data index preload/sum_lstat \ + "[2-9][0-9]*" <"$gitdir/$command.trace" && + ! test_trace2_data index preload/sum_lstat \ + "1[0-9][0-9]*" <"$gitdir/$command.trace" || + return 1 + done && test_write_lines dirty >"$worktree/existing/tracked" && for run in first second do @@ -3775,7 +3819,24 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO "$gitdir/status-$run" && ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/status-$run.trace" || return 1 - done + done && + git -C "$worktree" config filter.lfs.process "" && + git -C "$worktree" config filter.lfs.clean false && + test_write_lines "existing/tracked filter=lfs" \ + >"$worktree/.gitattributes" && + cp "$gitdir/index" "$gitdir/active-filter.before" && + test_must_fail env GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$gitdir/active-filter.trace" \ + git -C "$worktree" diff \ + >"$gitdir/active-filter.out" \ + 2>"$gitdir/active-filter.err" && + test_grep "clean filter .lfs. failed" \ + "$gitdir/active-filter.err" && + test_cmp_bin "$gitdir/active-filter.before" "$gitdir/index" && + ! test_trace2_data fsmonitor history/external-restored 1 \ + <"$gitdir/active-filter.trace" && + ! test_trace2_data fsmonitor history/external-semantic-restored 1 \ + <"$gitdir/active-filter.trace" ) ' From 8514240f26ae208d2c6e2802aa9b2e9e474ca794 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 02:57:49 -0500 Subject: [PATCH 08/73] fsmonitor: skip missing-checkpoint hashes for read-only commands External-history restoration currently computes a logical digest of every index entry before checking whether an authenticated checkpoint was actually loaded. Missing, malformed, or differently namespaced sidecars therefore add an unnecessary full-index pass to every read-only diff. Return before the digest when optional locks are disabled and no checkpoint exists. Preserve the writable path because its source digest is required to issue the first authenticated checkpoint, and retain missing-provider invalidation telemetry. Cover all three unusable checkpoint cases, subsequent authenticated restoration, writable checkpoint issuance, and active required filters in the existing linked-worktree regression. --- clean-status-history.c | 6 ++++++ t/t7519-status-fsmonitor.sh | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/clean-status-history.c b/clean-status-history.c index 53adf902044184..16f49951fb0812 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -1545,6 +1545,12 @@ int clean_status_restore_external_history(struct index_state *istate) goto have_index_hash; } } + if (!record_loaded && !use_optional_locks()) { + if (missing_fsmonitor_recovery) + trace2_data_intmax("fsmonitor", istate->repo, + "history/external-proof-invalidated", 1); + goto done; + } if (clean_status_index_logical_digest(istate, index_hash)) goto done; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 40c5d8418733f9..dbb1b8f50c676c 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3742,6 +3742,39 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO test_grep UNTR "$gitdir/index" && test_grep ! FSUC "$gitdir/index" && test_grep ! FSCF "$gitdir/index" && + checkpoint=$(cat "$gitdir/checkpoints") && + cp "$checkpoint" "$gitdir/checkpoint.valid" && + for corruption in missing malformed wrong-namespace + do + rm -f "$checkpoint" "$checkpoint.wrong" && + case "$corruption" in + missing) : ;; + malformed) printf "%s\n" corrupt >"$checkpoint" ;; + wrong-namespace) + cp "$gitdir/checkpoint.valid" "$checkpoint.wrong" ;; + esac && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" -c core.fsmonitor=false \ + -c core.untrackedCache=false diff \ + >"$gitdir/$corruption.expect" && + cp "$gitdir/index" "$gitdir/$corruption.before" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$gitdir/$corruption.trace" \ + git -C "$worktree" diff \ + >"$gitdir/$corruption.actual" && + test_cmp "$gitdir/$corruption.expect" \ + "$gitdir/$corruption.actual" && + test_cmp_bin "$gitdir/$corruption.before" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/$corruption.trace" && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/$corruption.trace" && + test_grep ! "\"label\":\"history_logical_digest\"" \ + "$gitdir/$corruption.trace" || return 1 + done && + rm -f "$checkpoint.wrong" && + cp "$gitdir/checkpoint.valid" "$checkpoint" && for command in diff diff-files diff-index do case "$command" in @@ -3820,6 +3853,14 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/status-$run.trace" || return 1 done && + rm -f "$checkpoint" && + GIT_TRACE2_EVENT="$gitdir/reissue-checkpoint.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/reissue-checkpoint" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/reissue-checkpoint.trace" && + test_path_is_file "$checkpoint" && + test_fsmonitor_full_proof "$gitdir/index" paired && git -C "$worktree" config filter.lfs.process "" && git -C "$worktree" config filter.lfs.clean false && test_write_lines "existing/tracked filter=lfs" \ From ae84bbabfbe502de070ba66bf2705406b43e30d4 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 03:12:08 -0500 Subject: [PATCH 09/73] fsmonitor: reserve checkpoint source hashes for writable status Optional locks describe what a command may do, not whether it will publish external clean history. Ordinary diff, diff-files, and diff-index can therefore still hash every index entry after failing to find a checkpoint, even though none of them can create one. Explicitly mark writable status as the sole command that requires an external-history source digest. Missing or unusable checkpoints then return immediately for both ordinary and read-only diff commands, while status retains the pre-refresh digest needed to publish its first authenticated checkpoint. Exercise missing, malformed, and wrongly namespaced checkpoints with optional locks enabled and disabled, preserving writable checkpoint issuance and active-filter failure coverage. --- builtin/commit.c | 2 ++ clean-status-history.c | 9 +++++++- clean-status.h | 1 + t/t7519-status-fsmonitor.sh | 46 +++++++++++++++++++++++++------------ 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 07c565fbff6fa2..04bfacdb08354e 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1831,6 +1831,8 @@ struct repository *repo UNUSED) if (isatty(2)) clean_status_enable_progress(the_repository); } + if (use_optional_locks()) + clean_status_require_external_history_source(the_repository); repo_read_index(the_repository); if (use_optional_locks()) clean_status_capture_external_history_source( diff --git a/clean-status-history.c b/clean-status-history.c index 16f49951fb0812..e02d8830ecb67d 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -23,6 +23,8 @@ #define CLEAN_STATUS_HISTORY_SCHEMA "builtin-fsmonitor-history-v2" +static struct repository *external_history_source_repo; + static void invalidate_disk_history(struct clean_status_state *state) { state->disk_config_seen = 1; @@ -516,6 +518,11 @@ static int external_history_namespace(struct index_state *istate, char *out) return ret; } +void clean_status_require_external_history_source(struct repository *repo) +{ + external_history_source_repo = repo; +} + void clean_status_capture_external_history_source( struct index_state *istate) { @@ -1545,7 +1552,7 @@ int clean_status_restore_external_history(struct index_state *istate) goto have_index_hash; } } - if (!record_loaded && !use_optional_locks()) { + if (!record_loaded && external_history_source_repo != istate->repo) { if (missing_fsmonitor_recovery) trace2_data_intmax("fsmonitor", istate->repo, "history/external-proof-invalidated", 1); diff --git a/clean-status.h b/clean-status.h index 47fbeacd430918..8602e66f6213c0 100644 --- a/clean-status.h +++ b/clean-status.h @@ -140,6 +140,7 @@ int clean_status_has_recovered_tracked_stat( const struct index_state *istate); int clean_status_external_history_owns_index( const struct index_state *istate); +void clean_status_require_external_history_source(struct repository *repo); void clean_status_capture_external_history_source( struct index_state *istate); int clean_status_save_external_history(struct index_state *istate); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index dbb1b8f50c676c..eac59d4792abcd 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3757,21 +3757,37 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO git -C "$worktree" -c core.fsmonitor=false \ -c core.untrackedCache=false diff \ >"$gitdir/$corruption.expect" && - cp "$gitdir/index" "$gitdir/$corruption.before" && - GIT_OPTIONAL_LOCKS=0 \ - GIT_TRACE2_EVENT="$gitdir/$corruption.trace" \ - git -C "$worktree" diff \ - >"$gitdir/$corruption.actual" && - test_cmp "$gitdir/$corruption.expect" \ - "$gitdir/$corruption.actual" && - test_cmp_bin "$gitdir/$corruption.before" \ - "$gitdir/index" && - test_trace2_data fsmonitor config/coherent 0 \ - <"$gitdir/$corruption.trace" && - test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ - <"$gitdir/$corruption.trace" && - test_grep ! "\"label\":\"history_logical_digest\"" \ - "$gitdir/$corruption.trace" || return 1 + for locking in readonly default + do + trace="$gitdir/$corruption-$locking.trace" && + cp "$gitdir/index" \ + "$gitdir/$corruption-$locking.before" && + if test "$locking" = readonly + then + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$trace" \ + git -C "$worktree" diff \ + >"$gitdir/$corruption.actual" + else + sane_unset GIT_OPTIONAL_LOCKS && + GIT_TRACE2_EVENT="$trace" \ + git -C "$worktree" diff \ + >"$gitdir/$corruption.actual" + fi && + test_cmp "$gitdir/$corruption.expect" \ + "$gitdir/$corruption.actual" && + test_cmp_bin \ + "$gitdir/$corruption-$locking.before" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 0 \ + <"$trace" && + test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 <"$trace" && + test_grep ! "\"label\":\"history_logical_digest\"" \ + "$trace" && + ! test_region index do_write_index "$trace" || + return 1 + done || return 1 done && rm -f "$checkpoint.wrong" && cp "$gitdir/checkpoint.valid" "$checkpoint" && From dfc55f903ca1dc355fd916fcae91974a37891017 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 03:45:13 -0500 Subject: [PATCH 10/73] diff: repair provider-reset history before publishing its index After the builtin fsmonitor daemon loses its history, diff previously advanced the tracked token while leaving the untracked proof pending. Every subsequent full or read-only status then had to revalidate every cached directory again. Perform one complete authenticated worktree and exclude validation before updating the index. Pin the original physical index, keep its lock unheld during directory traversal, and verify that the same index still exists after acquiring the lock. Publish the repaired index only after tracked, untracked, configuration, and provider-token proofs all agree. Preserve fail-closed behavior for concurrent writers, held locks, sparse or split indexes, conflicts, filters, unreadable directories, and lost provider events. Extend scripted and real-daemon regressions to require fully paired proofs, bounded follow-up directory work, and correct offline changes. --- builtin/diff.c | 134 +++++++++++++++++++++++++++++++++--- t/t7519-status-fsmonitor.sh | 36 ++++++---- 2 files changed, 150 insertions(+), 20 deletions(-) diff --git a/builtin/diff.c b/builtin/diff.c index d2e4e2297f3808..0838d8edd0e561 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -9,6 +9,7 @@ #include "builtin.h" #include "clean-status.h" +#include "clean-status-index.h" #include "clean-status-sidecar.h" #include "config.h" #include "ewah/ewok.h" @@ -29,9 +30,11 @@ #include "revision.h" #include "log-tree.h" #include "setup.h" +#include "thread-utils.h" #include "oid-array.h" #include "tree.h" #include "worktree.h" +#include "wt-status.h" #define DIFF_NO_INDEX_EXPLICIT 1 #define DIFF_NO_INDEX_IMPLICIT 2 @@ -241,6 +244,29 @@ static void builtin_diff_combined(struct rev_info *revs, oid_array_clear(&parents); } +static pthread_mutex_t diff_refresh_warning_mutex; +static int diff_refresh_warning_seen; + +static void capture_diff_refresh_warning(const char *message UNUSED, + va_list params UNUSED) +{ + pthread_mutex_lock(&diff_refresh_warning_mutex); + diff_refresh_warning_seen = 1; + pthread_mutex_unlock(&diff_refresh_warning_mutex); +} + +static int can_close_diff_fsmonitor_token(struct index_state *istate) +{ + return fsmonitor_pending_token_from_provider(istate) && + fstat_is_reliable() && + the_repository->config_values_private_.trust_ctime && + the_repository->config_values_private_.check_stat && + !getenv(INDEX_ENVIRONMENT) && + !istate->split_index && istate->sparse_index == INDEX_EXPANDED && + !unmerged_index(istate) && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC; +} + static void refresh_index_quietly(void) { struct lock_file lock_file = LOCK_INIT; @@ -253,19 +279,111 @@ static void refresh_index_quietly(void) if (!use_optional_locks()) return; + can_close_token = can_close_diff_fsmonitor_token(istate); + if (can_close_token && + !repo_config_values(the_repository)->apply_sparse_checkout && + istate->untracked && + istate->untracked->root && + istate->fsmonitor_untracked_extension_seen && + !istate->fsmonitor_untracked_extension_invalid && + !istate->fsmonitor_untracked_valid) { + struct clean_status_index_snapshot source = { .fd = -1 }; + struct clean_status_config_digest digest; + struct object_id exclude_digest; + struct stat scanned_worktree; + struct wt_status status; + report_fn original_warning; + int proof_complete; + int warning_seen; + + if (clean_status_index_snapshot_open_allow_null_checksum( + &source, repo_get_index_file(the_repository), + the_repository->hash_algo)) + return; + if (clean_status_config_read_repository(the_repository, + &digest)) { + clean_status_index_snapshot_release(&source); + return; + } + clean_status_set_config_digest(the_repository, &digest); + discard_index(istate); + repo_read_index(the_repository); + if (!clean_status_index_snapshot_still_matches_proof_epoch( + &source, istate)) { + clean_status_index_snapshot_release(&source); + return; + } + refresh_fsmonitor(istate); + if (!can_close_diff_fsmonitor_token(istate) || + repo_config_values(the_repository)->apply_sparse_checkout || + !istate->untracked || !istate->untracked->root || + !istate->fsmonitor_untracked_extension_seen || + istate->fsmonitor_untracked_extension_invalid || + istate->fsmonitor_untracked_valid) { + clean_status_index_snapshot_release(&source); + return; + } + if (pthread_mutex_init(&diff_refresh_warning_mutex, NULL)) { + clean_status_index_snapshot_release(&source); + return; + } + diff_refresh_warning_seen = 0; + original_warning = get_warn_routine(); + set_warn_routine(capture_diff_refresh_warning); + wt_status_prepare(the_repository, &status); + status.certify_clean_status = 1; + status.show_untracked_files = istate->untracked->dir_flags ? + SHOW_NORMAL_UNTRACKED_FILES : SHOW_ALL_UNTRACKED_FILES; + wt_status_start_untracked_cache_preload(&status); + wt_status_refresh_index(&status, + REFRESH_QUIET | REFRESH_UNMERGED | + REFRESH_DEFER_BULK_DIRTY, 1); + proof_complete = !status.certify_untracked_scan_failed && + !wt_status_certified_excludes_digest( + &status, &exclude_digest, &scanned_worktree) && + !fsmonitor_has_pending_token(istate) && + istate->fsmonitor_untracked_valid && + istate->fsmonitor_last_update && + istate->fsmonitor_untracked_token && + !strcmp(istate->fsmonitor_last_update, + istate->fsmonitor_untracked_token) && + (!istate->clean_status || + clean_status_revalidated_token_matches(istate)); + wt_status_collect_free_buffers(&status); + set_warn_routine(original_warning); + pthread_mutex_lock(&diff_refresh_warning_mutex); + warning_seen = diff_refresh_warning_seen; + pthread_mutex_unlock(&diff_refresh_warning_mutex); + pthread_mutex_destroy(&diff_refresh_warning_mutex); + string_list_clear(&status.untracked, 0); + string_list_clear(&status.ignored, 0); + free(status.branch); + if (!proof_complete || warning_seen) { + clean_status_index_snapshot_release(&source); + return; + } + fd = repo_hold_locked_index(the_repository, &lock_file, 0); + if (fd < 0) { + clean_status_index_snapshot_release(&source); + return; + } + if (!clean_status_index_snapshot_still_matches_path( + &source, repo_get_index_file(the_repository), + the_repository->hash_algo)) { + rollback_lock_file(&lock_file); + clean_status_index_snapshot_release(&source); + return; + } + repo_update_index_if_able(the_repository, &lock_file); + clean_status_index_snapshot_release(&source); + return; + } fd = repo_hold_locked_index(the_repository, &lock_file, 0); if (fd < 0) return; discard_index(istate); repo_read_index(the_repository); - can_close_token = fstat_is_reliable() && - the_repository->config_values_private_.trust_ctime && - the_repository->config_values_private_.check_stat && - !getenv(INDEX_ENVIRONMENT) && - !istate->split_index && istate->sparse_index == INDEX_EXPANDED && - !unmerged_index(istate) && - fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC && - fsmonitor_pending_token_from_provider(istate); + can_close_token = can_close_diff_fsmonitor_token(istate); refreshed = refresh_index(istate, REFRESH_QUIET | REFRESH_UNMERGED | (can_close_token ? REFRESH_IN_PROOF_EPOCH : 0), diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index eac59d4792abcd..cf8d1e8c249c05 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -2218,7 +2218,7 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ' test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ - 'diff preserves pending reset proofs in main and linked worktrees' ' + 'diff fully revalidates reset proofs in main and linked worktrees' ' test_when_finished "rm -rf builtin-diff-reset builtin-diff-reset-linked" && test_create_repo builtin-diff-reset && ( @@ -2267,14 +2267,8 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_must_be_empty "$gitdir/reset.actual" && test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/reset.trace" && - test_trace2_data fsmonitor \ - untracked/provider-reset-pending 1 \ - <"$gitdir/reset.trace" && test_region index do_write_index "$gitdir/reset.trace" && - test_grep FSMN "$gitdir/index" && - test_grep FSUC "$gitdir/index" && - test_grep "pending:test:[2-9]" "$gitdir/index" && - test_fsmonitor_full_proof "$gitdir/index" pending && + test_fsmonitor_full_proof "$gitdir/index" paired && cp "$gitdir/index" "$gitdir/pending.index" && GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ @@ -2288,6 +2282,12 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ! test_trace2_data fsmonitor \ semantic/manifest-scan-count 1 \ <"$gitdir/pending-status.trace" && + ! test_trace2_data dir preload_untracked_cache/dirs \ + "[2-9][0-9]*" \ + <"$gitdir/pending-status.trace" && + ! test_trace2_data dir preload_untracked_cache/dirs \ + "1[0-9][0-9]*" \ + <"$gitdir/pending-status.trace" && ! test_region index do_write_index \ "$gitdir/pending-status.trace" && GIT_TEST_PRELOAD_INDEX=1 \ @@ -4031,15 +4031,21 @@ test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OP "$worktree/cached/deep/tracked" && git -C "$worktree" fsmonitor--daemon start \ --start-timeout=10 && + cp "$gitdir/index" "$gitdir/locked.index" && + : >"$gitdir/index.lock" && + GIT_TRACE2_EVENT="$gitdir/locked.trace" \ + git -C "$worktree" diff >"$gitdir/locked" && + rm -f "$gitdir/index.lock" && + test_must_be_empty "$gitdir/locked" && + test_cmp_bin "$gitdir/locked.index" "$gitdir/index" && + ! test_region index do_write_index \ + "$gitdir/locked.trace" && GIT_TRACE2_EVENT="$gitdir/diff.trace" \ git -C "$worktree" diff >"$gitdir/diff" && test_must_be_empty "$gitdir/diff" && test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/diff.trace" && - test_trace2_data fsmonitor \ - untracked/provider-reset-pending 1 \ - <"$gitdir/diff.trace" && - test_fsmonitor_full_proof "$gitdir/index" pending && + test_fsmonitor_full_proof "$gitdir/index" paired && cp "$gitdir/index" "$gitdir/readonly.index" && GIT_OPTIONAL_LOCKS=0 \ GIT_TRACE2_EVENT="$gitdir/readonly.trace" \ @@ -4052,6 +4058,12 @@ test_expect_success FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OP ! test_trace2_data fsmonitor \ semantic/manifest-scan-count 1 \ <"$gitdir/readonly.trace" && + ! test_trace2_data dir preload_untracked_cache/dirs \ + "[2-9][0-9]*" \ + <"$gitdir/readonly.trace" && + ! test_trace2_data dir preload_untracked_cache/dirs \ + "1[0-9][0-9]*" \ + <"$gitdir/readonly.trace" && ! test_region index do_write_index \ "$gitdir/readonly.trace" && git -C "$worktree" fsmonitor--daemon stop && From 7c19bf467bbdcbf4080c195d86bf907fd1c78590 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 19:15:59 -0500 Subject: [PATCH 11/73] fsmonitor: retain checked manifests across scoped directory deltas 60077b92ac (status: preserve semantic history across scoped and index changes, 2026-08-11) lets a second closing query preserve verified sibling subtrees. A directory event cannot use that path while recovery is still establishing its semantic proof, so even a local change rebuilds the complete attribute manifest and rescans every directory. Permit a bounded directory check only around the second closing query, after the tracked semantic proof has closed. Require a continuous provider delta, the same pinned index and namespace, and an unchanged configuration. Reopen each affected attribute candidate beneath the worktree root and compare its contents, indexed fallback, or absence with the checked manifest. Keep the existing full fallback for lost or global events, hardlinks, unsupported indexes, and changed sources. The directory still dirties every tracked entry and its untracked cone. Directory timestamps never certify tracked contents. When this new shortcut is used, revalidate the completed scan's exclude-source proof before retaining sibling results: a directory event can also conceal a changed .gitignore. Do not add that extra work to ordinary file events. In a local 2,050-directory fixture, the second walk visits two directories instead of 2,051, and only one attribute manifest is built. Three alternating runs reduced median recovery time from 604 ms to 480 ms; the existing file-event path remained within 2% of baseline. Retain both paths in the regression test and synchronize attribute and ignore changes between the two scans to exercise the full fallback. --- clean-status-manifest.c | 224 ++++++++++++++++++++++++++++++++++++ clean-status-manifest.h | 6 + fsmonitor.c | 101 +++++++++++++--- t/t7519-status-fsmonitor.sh | 161 ++++++++++++++++++++++++-- wt-status.c | 13 ++- 5 files changed, 479 insertions(+), 26 deletions(-) diff --git a/clean-status-manifest.c b/clean-status-manifest.c index 4259385110f016..d668abf8da01b4 100644 --- a/clean-status-manifest.c +++ b/clean-status-manifest.c @@ -3,6 +3,7 @@ #include "attr.h" #include "attr-manifest.h" #include "bloom.h" +#include "clean-status.h" #include "clean-status-config.h" #include "clean-status-index.h" #include "clean-status-internal.h" @@ -22,8 +23,10 @@ #include "read-cache-ll.h" #include "replace-object.h" #include "repository.h" +#include "semantic-verify.h" #include "semantic-verify-internal.h" #include "sparse-index.h" +#include "string-list.h" #include "trace2.h" #include "tree.h" #include "tree-walk.h" @@ -38,6 +41,18 @@ struct invalidate_manifest_data { int invalidated; }; +/* + * Only the second provider query after a closed semantic proof may reuse its + * manifest. A directory delta must authenticate every affected attribute + * source, hash, and absence; its tracked and untracked entries remain dirty. + * Directory timestamps never establish tracked-file content. + */ +static struct { + struct index_state *index; + const struct semantic_verify_proof *proof; + unsigned reused : 1; +} manifest_directory_delta; + static int build_manifest(struct index_state *istate, struct strbuf *manifest, unsigned char *manifest_hash, @@ -139,6 +154,215 @@ static int find_manifest_entry( return -1; } +void clean_status_manifest_begin_directory_delta( + struct index_state *istate, const struct semantic_verify_proof *proof) +{ + if (manifest_directory_delta.index) + BUG("nested clean-status directory delta"); + if (!proof || !semantic_verify_proof_is_current(istate, proof)) + return; + manifest_directory_delta.index = istate; + manifest_directory_delta.proof = proof; + manifest_directory_delta.reused = 0; +} + +int clean_status_manifest_end_directory_delta(struct index_state *istate) +{ + int reused; + + if (manifest_directory_delta.index != istate) + return 0; + reused = manifest_directory_delta.reused; + manifest_directory_delta.index = NULL; + manifest_directory_delta.proof = NULL; + manifest_directory_delta.reused = 0; + return reused; +} + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int directory_attribute_source_matches( + struct index_state *istate, struct semantic_verify_path *path, + const char *name, size_t position) +{ + struct clean_status_state *state = istate->clean_status; + struct attr_manifest_entry entry; + const struct cache_entry *indexed; + const struct git_hash_algo *algo = istate->repo->hash_algo; + const char *basename; + unsigned char observed[GIT_MAX_RAWSZ]; + struct stat st; + int parent_fd, found, present, pos; + + if (semantic_verify_resolve_parent(path, name, position, + &parent_fd, &basename)) + return 0; + if (fstatat(parent_fd, basename, &st, AT_SYMLINK_NOFOLLOW)) { + if (errno != ENOENT) + return 0; + } else if (!S_ISREG(st.st_mode) || st.st_nlink != 1) { + return 0; + } + if (worktree_attr_source_read(path, name, position, algo, + observed, &found)) + return 0; + present = !find_manifest_entry(&state->manifest.current, + name, algo, &entry); + if (found) + return present && entry.source == ATTR_MANIFEST_WORKTREE && + !memcmp(entry.hash, observed, algo->rawsz); + if (!fstatat(parent_fd, basename, &st, AT_SYMLINK_NOFOLLOW) || + errno != ENOENT) + return 0; + pos = index_name_pos(istate, name, strlen(name)); + if (pos < 0) + return !present; + indexed = istate->cache[pos]; + if (!S_ISREG(indexed->ce_mode) || ce_stage(indexed) || + ce_skip_worktree(indexed) || ce_intent_to_add(indexed) || + (indexed->ce_flags & CE_VALID)) + return 0; + return present && entry.source == ATTR_MANIFEST_INDEX && + !memcmp(entry.hash, indexed->oid.hash, algo->rawsz); +} +#endif + +int clean_status_manifest_directory_unchanged( + struct index_state *istate, const char *directory) +{ +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + struct clean_status_state *state = istate->clean_status; + struct semantic_verify_root *root = NULL; + struct semantic_verify_path *path = NULL; + struct clean_status_index_snapshot snapshot; + struct clean_status_config_digest config; + struct attr_fingerprint attrs; + struct string_list candidates = STRING_LIST_INIT_DUP; + struct strbuf candidate = STRBUF_INIT; + const struct git_hash_algo *algo = istate->repo->hash_algo; + unsigned int first, count = 0, namespace_unstable = 0; + size_t len; + int pos, pinned = 0, safe = 0; + uint32_t required = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + + if (manifest_directory_delta.index != istate || + !manifest_directory_delta.proof || + !fsmonitor_pending_token_from_provider(istate) || + fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC || + !fstat_is_reliable() || getenv(INDEX_ENVIRONMENT) || + istate != istate->repo->index || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || + repo_config_values(istate->repo)->apply_sparse_checkout || !state || + !state->current_config_valid || !state->current_semantic_valid || + !state->current_tracked_policy_valid || !state->current_attr_valid || + !state->config_enforced || + (state->filter_configured && !state->filter_scope_valid) || + !state->manifest.scan_count || !state->manifest.current_valid || + !state->manifest.checked || state->manifest.current_invalidated || + state->manifest.global_fallback || + (state->manifest.current_flags & required) != required || + !semantic_verify_proof_is_current( + istate, manifest_directory_delta.proof) || + clean_status_config_read_repository(istate->repo, &config) || + !config.finalized || + config.filter_configured != state->filter_configured || + config.semantic_config_explicit != state->current_semantic_explicit || + memcmp(config.hash, state->current_config_hash, algo->rawsz) || + memcmp(config.semantic_hash, state->current_semantic_hash, + algo->rawsz) || + memcmp(config.tracked_policy_hash, + state->current_tracked_policy_hash, algo->rawsz) || + attr_fingerprint_repository(istate->repo, &attrs) || + attrs.sources_present != state->current_attr_sources_present || + memcmp(attrs.content_hash, state->current_attr_hash, algo->rawsz) || + memcmp(attrs.namespace_hash, + state->current_attr_namespace_hash, algo->rawsz) || + !attr_manifest_valid(state->manifest.current.buf, + state->manifest.current.len, algo)) + goto done; + + len = strlen(directory); + if (!len || directory[len - 1] != '/') + goto done; + pos = index_name_pos(istate, directory, len); + if (pos >= 0) + goto done; + first = -pos - 1; + if (first >= istate->cache_nr || + !starts_with(istate->cache[first]->name, directory)) + goto done; + if (clean_status_index_snapshot_pin_proof_epoch(&snapshot, istate)) + goto done; + pinned = 1; + if (semantic_verify_root_init(istate->repo, &root)) + goto done; + path = semantic_verify_path_new(root); + if (!path) + goto done; + + strbuf_addstr(&candidate, directory); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + string_list_insert(&candidates, candidate.buf); + for (unsigned int i = first; i < istate->cache_nr && + starts_with(istate->cache[i]->name, directory); i++) { + const struct cache_entry *ce = istate->cache[i]; + const char *slash = ce->name + len; + + if (++count > 64 || ce_stage(ce) || ce_skip_worktree(ce) || + ce_intent_to_add(ce) || (ce->ce_flags & CE_VALID) || + S_ISSPARSEDIR(ce->ce_mode)) + goto done; + while ((slash = strchr(slash, '/')) != NULL) { + strbuf_reset(&candidate); + strbuf_add(&candidate, ce->name, + slash - ce->name + 1); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + string_list_insert(&candidates, candidate.buf); + if (candidates.nr > 64) + goto done; + slash++; + } + } + for (size_t i = 0; i < candidates.nr; i++) + if (!directory_attribute_source_matches( + istate, path, candidates.items[i].string, + first + i)) + goto done; + + semantic_verify_path_free(path, &namespace_unstable, NULL); + path = NULL; + if (namespace_unstable || !semantic_verify_root_stable(root) || + !clean_status_index_snapshot_still_matches_proof_epoch( + &snapshot, istate) || + !semantic_verify_proof_is_current( + istate, manifest_directory_delta.proof) || + attr_fingerprint_repository(istate->repo, &attrs) || + attrs.sources_present != state->current_attr_sources_present || + memcmp(attrs.content_hash, state->current_attr_hash, algo->rawsz) || + memcmp(attrs.namespace_hash, + state->current_attr_namespace_hash, algo->rawsz)) + goto done; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-directory-reused", 1); + manifest_directory_delta.reused = 1; + safe = 1; + +done: + if (path) + semantic_verify_path_free(path, NULL, NULL); + semantic_verify_root_clear(root); + if (pinned) + clean_status_index_snapshot_release(&snapshot); + string_list_clear(&candidates, 0); + strbuf_release(&candidate); + return safe; +#else + (void)istate; + (void)directory; + return 0; +#endif +} + int clean_status_manifest_reconcile_deleted_attribute( struct index_state *istate, const char *name) { diff --git a/clean-status-manifest.h b/clean-status-manifest.h index ba90498d56c247..81cad1124e6af9 100644 --- a/clean-status-manifest.h +++ b/clean-status-manifest.h @@ -5,6 +5,7 @@ #include "strbuf.h" struct index_state; +struct semantic_verify_proof; struct clean_status_manifest_state { struct strbuf disk; @@ -31,6 +32,11 @@ void clean_status_manifest_adopt_disk( struct clean_status_manifest_state *state); int clean_status_manifest_refresh(struct index_state *istate, struct clean_status_manifest_state *state); +void clean_status_manifest_begin_directory_delta( + struct index_state *istate, const struct semantic_verify_proof *proof); +int clean_status_manifest_end_directory_delta(struct index_state *istate); +int clean_status_manifest_directory_unchanged( + struct index_state *istate, const char *directory); int clean_status_manifest_reconcile_deleted_attribute( struct index_state *istate, const char *path); int clean_status_manifest_reconcile_display_only_attribute( diff --git a/fsmonitor.c b/fsmonitor.c index 7026d2148d3fb0..1800ae7174f6a0 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -20,6 +20,7 @@ #include "run-command.h" #include "strbuf.h" #include "trace2.h" +#include "wrapper.h" #define INDEX_EXTENSION_VERSION1 (1) #define INDEX_EXTENSION_VERSION2 (2) @@ -410,7 +411,8 @@ void fsmonitor_invalidate_cache_entry(struct cache_entry *ce) } static size_t handle_path_with_trailing_slash( - struct index_state *istate, const char *name, int pos); + struct index_state *istate, const char *name, int pos, + int directory_is_semantically_safe); int fsmonitor_invalidate_attributes_path(struct index_state *istate, const char *name) @@ -547,7 +549,9 @@ static size_t handle_using_dir_name_hash_icase( pos = index_name_pos(istate, canonical_path.buf, canonical_path.len); nr_in_cone = handle_path_with_trailing_slash( - istate, canonical_path.buf, pos); + istate, canonical_path.buf, pos, + clean_status_directory_event_is_semantically_safe( + istate, canonical_path.buf)); strbuf_release(&canonical_path); return nr_in_cone; } @@ -603,7 +607,9 @@ static size_t handle_path_without_trailing_slash( strbuf_addch(&work_path, '/'); pos = index_name_pos(istate, work_path.buf, work_path.len); nr_in_cone = handle_path_with_trailing_slash( - istate, work_path.buf, pos); + istate, work_path.buf, pos, + clean_status_directory_event_is_semantically_safe( + istate, work_path.buf)); strbuf_release(&work_path); return nr_in_cone; } @@ -642,7 +648,8 @@ static size_t handle_path_without_trailing_slash( * untracked or case-incorrect. */ static size_t handle_path_with_trailing_slash( - struct index_state *istate, const char *name, int pos) + struct index_state *istate, const char *name, int pos, + int directory_is_semantically_safe) { int i; size_t nr_in_cone = 0; @@ -667,8 +674,7 @@ static size_t handle_path_with_trailing_slash( nr_in_cone++; } - if (nr_in_cone && - !clean_status_directory_event_is_semantically_safe(istate, name)) { + if (nr_in_cone && !directory_is_semantically_safe) { /* * A matched directory event may stand in for a nested * attribute-file change. @@ -681,7 +687,8 @@ static size_t handle_path_with_trailing_slash( return nr_in_cone; } -static void fsmonitor_refresh_callback(struct index_state *istate, char *name) +static void fsmonitor_refresh_callback(struct index_state *istate, char *name, + int closing_delta) { int len = strlen(name); int pos; @@ -724,10 +731,13 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) fsmonitor_invalidate_attributes_path(istate, name); } directory_is_semantically_safe = name[len - 1] == '/' && - clean_status_directory_event_is_semantically_safe(istate, name); + (clean_status_directory_event_is_semantically_safe(istate, name) || + (closing_delta && + clean_status_manifest_directory_unchanged(istate, name))); if (name[len - 1] == '/') - nr_in_cone = handle_path_with_trailing_slash(istate, name, pos); + nr_in_cone = handle_path_with_trailing_slash( + istate, name, pos, directory_is_semantically_safe); else nr_in_cone = handle_path_without_trailing_slash(istate, name, pos); if (pos < 0 && nr_in_cone && !directory_is_semantically_safe) @@ -907,6 +917,45 @@ enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( return FSMONITOR_QUERY_ERROR; } +static int fsmonitor_test_query_barrier(size_t query_nr) +{ + const char *at = getenv("GIT_TEST_FSMONITOR_QUERY_BARRIER_AT"); + const char *ready = getenv("GIT_TEST_FSMONITOR_QUERY_BARRIER_READY"); + const char *resume = getenv("GIT_TEST_FSMONITOR_QUERY_BARRIER_RESUME"); + struct stat st; + uintmax_t selected; + char *end; + char resumed; + int fd, ret; + + if (!at && !ready && !resume) + return 0; + if (!at || !ready || !resume || !*at || !*ready || !*resume || + !isdigit((unsigned char)*at)) + return -1; + errno = 0; + selected = strtoumax(at, &end, 10); + if (errno || *end || !selected) + return -1; + if (selected != (uintmax_t)query_nr) + return 0; + if (lstat(resume, &st) || !S_ISFIFO(st.st_mode)) + return -1; + fd = open(ready, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + if (fd < 0) + return -1; + ret = write_in_full(fd, "ready\n", 6) == 6 ? 0 : -1; + if (close(fd) || ret) + return -1; + fd = open(resume, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + ret = read_in_full(fd, &resumed, 1) == 1 ? 0 : -1; + if (close(fd)) + ret = -1; + return ret; +} + enum fsmonitor_query_outcome query_builtin_fsmonitor( const char *since_token, struct fsmonitor_query_result *result) { @@ -927,6 +976,8 @@ enum fsmonitor_query_outcome query_builtin_fsmonitor( if (query_nr >= strlen(test_sequence)) return FSMONITOR_QUERY_ERROR; outcome = test_sequence[query_nr++]; + if (fsmonitor_test_query_barrier(query_nr)) + return FSMONITOR_QUERY_ERROR; if (outcome == 'E') return FSMONITOR_QUERY_ERROR; @@ -981,7 +1032,7 @@ static int fsmonitor_hardlink_inode_cmp(const void *unused UNUSED, } static int apply_fsmonitor_paths(struct index_state *istate, - const struct strbuf *paths) + const struct strbuf *paths, int closing_delta) { const char *p = paths->buf; const char *end = paths->buf + paths->len; @@ -990,6 +1041,20 @@ static int apply_fsmonitor_paths(struct index_state *istate, unsigned int matches = 0; int count = 0; + if (closing_delta) { + for (const char *changed = p; changed < end; + changed += strlen(changed) + 1) { + size_t changed_len = strlen(changed); + + if (!strcmp(changed, FSMONITOR_PATH_GLOBAL_INVALIDATE) || + fsmonitor_parse_hardlink_inode( + changed, changed_len, NULL)) { + closing_delta = 0; + break; + } + } + } + while (p < end) { size_t len = strlen(p); uint32_t inode; @@ -997,12 +1062,13 @@ static int apply_fsmonitor_paths(struct index_state *istate, if (parsed < 0) { fsmonitor_refresh_callback( - istate, (char *)FSMONITOR_PATH_GLOBAL_INVALIDATE); + istate, (char *)FSMONITOR_PATH_GLOBAL_INVALIDATE, 0); count++; goto done; } if (!parsed) { - fsmonitor_refresh_callback(istate, (char *)p); + fsmonitor_refresh_callback( + istate, (char *)p, closing_delta); count++; } else if (!hashmap_get_entry_from_hash( &inodes, memhash(&inode, sizeof(inode)), &inode, @@ -1031,7 +1097,7 @@ static int apply_fsmonitor_paths(struct index_state *istate, &inodes, memhash(&inode, sizeof(inode)), &inode, struct fsmonitor_hardlink_inode, ent)) continue; - fsmonitor_refresh_callback(istate, ce->name); + fsmonitor_refresh_callback(istate, ce->name, 0); matches++; count++; } @@ -1453,20 +1519,21 @@ void refresh_fsmonitor(struct index_state *istate) int count = 0; if (fsm_mode == FSMONITOR_MODE_IPC) { - count = apply_fsmonitor_paths(istate, &query_result); + count = apply_fsmonitor_paths(istate, &query_result, 0); } else { buf = query_result.buf; for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') continue; if (i > bol) { - fsmonitor_refresh_callback(istate, buf + bol); + fsmonitor_refresh_callback( + istate, buf + bol, 0); count++; } bol = i + 1; } if (bol < query_result.len) { - fsmonitor_refresh_callback(istate, buf + bol); + fsmonitor_refresh_callback(istate, buf + bol, 0); count++; } } @@ -1659,7 +1726,7 @@ enum fsmonitor_token_result fsmonitor_query_pending_token( goto done; } - count = apply_fsmonitor_paths(istate, &result.paths); + count = apply_fsmonitor_paths(istate, &result.paths, 1); if (istate->untracked) istate->untracked->use_fsmonitor = !!untracked_ready; trace2_data_intmax("fsmonitor", istate->repo, diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index cf8d1e8c249c05..45914588c794ec 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -633,6 +633,28 @@ test_fsmonitor_full_proof () { EOF } +wait_for_fsmonitor_query_barrier () { + for attempt in $(test_seq 1 500) + do + if test "$(cat "$1" 2>/dev/null)" = ready + then + return 0 + fi + kill -0 "$2" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +cleanup_fsmonitor_query_barrier () { + if test -n "${fsmonitor_query_pid-}" + then + kill "$fsmonitor_query_pid" 2>/dev/null || : + wait "$fsmonitor_query_pid" 2>/dev/null || : + fsmonitor_query_pid= + fi +} + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'bare status reuses a current tracked fsmonitor proof' ' test_when_finished "rm -rf builtin-tracked-clean" && @@ -3100,10 +3122,13 @@ test_expect_success HARDLINKS,!MINGW,!CYGWIN \ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'second closing-query change preserves verified sibling subtrees' ' - test_when_finished "rm -rf second-query-changed" && - test_create_repo second-query-changed && - ( - cd second-query-changed && + test_when_finished \ + "rm -rf second-query-changed-file second-query-changed-directory" && + for event in file directory + do + test_create_repo "second-query-changed-$event" && + ( + cd "second-query-changed-$event" && sane_unset GIT_TEST_SPLIT_INDEX && mkdir cached && test_write_lines "*.ignored" >.gitignore && @@ -3144,8 +3169,14 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ -c core.trustctime=true -c core.checkStat=default \ status --porcelain=v2 >.git/expect && + if test "$event" = directory + then + changed_path=cached/ + else + changed_path=cached/tracked + fi && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ - GIT_TEST_FSMONITOR_QUERY_PATH=cached/tracked \ + GIT_TEST_FSMONITOR_QUERY_PATH="$changed_path" \ GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ GIT_TRACE2_PERF="$PWD/.git/status.perf" \ git status --porcelain=v2 >.git/actual && @@ -3160,6 +3191,19 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <.git/status.trace && test_trace2_data fsmonitor token_closure/apply_count 1 \ <.git/status.trace && + if test "$event" = directory + then + test_trace2_data fsmonitor \ + semantic/manifest-directory-reused 1 \ + <.git/status.trace + else + ! test_trace2_data fsmonitor \ + semantic/manifest-directory-reused 1 \ + <.git/status.trace + fi && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <.git/status.trace >.git/manifest-scans && + test_line_count = 1 .git/manifest-scans && test_trace2_data status \ fsmonitor_token/reused-semantic-subtrees 1 \ <.git/status.trace && @@ -3196,9 +3240,106 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <.git/status.trace && test_trace2_data fsmonitor token_closure/accepted 1 \ <.git/status.trace && - test_grep FSCF .git/index && - test_grep FSUC .git/index - ) + test_fsmonitor_full_proof .git/index paired + ) || return 1 + done +' + +test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'directory closure rejects raced attributes and rechecks raced excludes' ' + test_when_finished "rm -rf directory-race-attributes directory-race-ignore" && + for mutation in attributes ignore + do + test_create_repo "directory-race-$mutation" && + ( + cd "directory-race-$mutation" && + sane_unset GIT_TEST_SPLIT_INDEX && + fsmonitor_query_pid= && + trap cleanup_fsmonitor_query_barrier 0 && + mkdir cached && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + printf "aaaa\n" >cached/tracked && + test_write_lines ignored >cached/junk.ignored && + for sibling in $(test_seq 1 8) + do + mkdir "sibling-$sibling" && + test_write_lines "$sibling" \ + >"sibling-$sibling/tracked" || return 1 + done && + git add .gitignore cached/.gitignore cached/tracked sibling-* && + git commit -qm base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test_write_lines visible >sibling-1/visible && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_grep "^? sibling-1/visible$" .git/prime && + test-tool chmtime =-60 cached/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get cached/tracked) && + printf "bbbb\n" >cached/tracked && + test-tool chmtime =$mtime cached/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid cached/tracked && + ready="$PWD/.git/provider.ready" && + resume="$PWD/.git/provider.resume" && + mkfifo "$resume" && + { + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/ \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_AT=3 \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_READY="$ready" \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + >.git/actual 2>.git/error & + fsmonitor_query_pid=$! + } && + wait_for_fsmonitor_query_barrier \ + "$ready" "$fsmonitor_query_pid" && + if test "$mutation" = attributes + then + test_write_lines "tracked text eol=crlf" \ + >cached/.gitattributes + else + test_write_lines "!junk.ignored" >cached/.gitignore + fi && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 >.git/expect && + printf x >"$resume" && + wait "$fsmonitor_query_pid" && + fsmonitor_query_pid= && + test_cmp .git/expect .git/actual && + test_grep "^1 \\.M .* cached/tracked$" .git/actual && + test_grep "^? sibling-1/visible$" .git/actual && + if test "$mutation" = attributes + then + test_grep "^? cached/\\.gitattributes$" \ + .git/actual && + test_trace2_data fsmonitor \ + semantic/manifest-scan-count 2 \ + <.git/status.trace && + ! test_trace2_data fsmonitor \ + semantic/manifest-directory-reused 1 \ + <.git/status.trace + else + test_grep "^1 \\.M .* cached/\\.gitignore$" \ + .git/actual && + test_grep "^? cached/junk\\.ignored$" .git/actual + fi && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace + ) || return 1 + done ' test_expect_success MACOS,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ @@ -3337,6 +3478,10 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_grep "^1 \.M .* cached/tracked$" .git/actual && test_trace2_data fsmonitor apply/global-invalidation 1 \ <.git/status.trace && + test_trace2_data fsmonitor semantic/manifest-scan-count 2 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/manifest-directory-reused 1 \ + <.git/status.trace && test_trace2_data fsmonitor semantic/strong-invalidation 1 \ <.git/status.trace >.git/strong-invalidations && test_line_count = 2 .git/strong-invalidations && diff --git a/wt-status.c b/wt-status.c index c24c42f71350c1..603fcbd3a67704 100644 --- a/wt-status.c +++ b/wt-status.c @@ -13,6 +13,7 @@ #include "commit.h" #include "clean-status.h" #include "clean-status-index.h" +#include "clean-status-manifest.h" #include "diff.h" #include "environment.h" #include "exclude-source-proof.h" @@ -2067,6 +2068,8 @@ wt_status_close_semantic_fsmonitor_token( } if (defer_untracked) { + int directory_delta_reused; + closure->untracked_ready = wt_status_stage_untracked(closure); closure->untracked_proof_complete = @@ -2081,15 +2084,23 @@ wt_status_close_semantic_fsmonitor_token( /* A second query closes the subsequent untracked scan. */ closure->queries++; + clean_status_manifest_begin_directory_delta(istate, *proof); result = fsmonitor_query_pending_token( istate, wt_status_untracked_cache_valid(closure)); + directory_delta_reused = + clean_status_manifest_end_directory_delta(istate); if (result != FSMONITOR_TOKEN_CLEAN) { + /* Only directory reuse adds an unobserved exclude risk. */ int reuse_semantic_subtrees = result == FSMONITOR_TOKEN_CHANGED && !clean_status_filter_scope_needs_validation(istate) && !clean_status_worktree_manifest_needs_refresh(istate) && - semantic_verify_proof_is_current(istate, *proof); + semantic_verify_proof_is_current(istate, *proof) && + (!directory_delta_reused || + (s->certify_exclude_proof && + exclude_source_proof_validate( + s->certify_exclude_proof))); wt_status_discard_staged_untracked(closure); if (reuse_semantic_subtrees) { From ef6cdb50392925e077500c4c1ab203c3ee12cd1c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 19:16:13 -0500 Subject: [PATCH 12/73] diff: reuse pinned observations during provider-reset repair dfc55f903c (diff: repair provider-reset history before publishing its index, 2026-08-15) repairs tracked and untracked history together after the provider loses its token. The repair rereads the physical index, however, discarding the pending provider token and attribute manifest that diff has already obtained. The reread then repeats both operations. Keep the existing index state when its pinned physical source still matches, its logical changes are limited to acceleration metadata, and its configuration, attributes, and pending proof epoch remain valid. Clear every tracked entry's up-to-date and fsmonitor-valid bits, along with any bulk result: observations made before the repair epoch cannot certify tracked contents. The ordinary complete tracked and untracked validation still runs before the single index write. Retain the reread fallback, late optional index lock, and final pinned path check. A concurrent writer must win even with index.skipHash set. The regression test now requires one index read and manifest scan, and pauses a repair while another process stages a file. Across three local real-daemon runs with 2,050 sibling directories, index reads and complete manifest scans fell from two to one. Median repair time fell from 138 ms to 55 ms. Each run published a fully paired proof and kept read-only followers correct without another full directory scan. --- builtin/diff.c | 47 +++++++++++++++++++++++-- t/t7519-status-fsmonitor.sh | 68 ++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/builtin/diff.c b/builtin/diff.c index 0838d8edd0e561..7aa66f7fda581a 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -8,6 +8,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "attr-fingerprint.h" #include "clean-status.h" #include "clean-status-index.h" #include "clean-status-sidecar.h" @@ -21,6 +22,7 @@ #include "fsmonitor-ll.h" #include "fsmonitor-settings.h" #include "tag.h" +#include "trace2.h" #include "diff.h" #include "diff-merges.h" #include "diffcore.h" @@ -267,6 +269,45 @@ static int can_close_diff_fsmonitor_token(struct index_state *istate) fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC; } +static int reuse_diff_recovery_observations( + struct index_state *istate, + const struct clean_status_index_snapshot *source) +{ + struct attr_source_snapshot *attrs = NULL; + struct clean_status_proof_epoch *epoch = NULL; + int reused = 0; + unsigned int i; + + if (!clean_status_index_snapshot_still_matches_proof_epoch( + source, istate) || + !clean_status_index_can_reuse_source_logical_hash(istate) || + !clean_status_fsmonitor_semantic_baseline_pending(istate) || + clean_status_fsmonitor_strong_mismatch(istate) || + clean_status_filter_scope_needs_validation(istate) || + clean_status_manifest_global_fallback(istate) || + clean_status_worktree_manifest_needs_refresh(istate) || + clean_status_capture_attr_snapshot(istate, &attrs) || !attrs) + goto done; + + epoch = clean_status_capture_proof_epoch(istate, attrs, 0); + if (!epoch || !clean_status_proof_epoch_prime_matches(istate, epoch)) + goto done; + + /* Observations made before the proof epoch cannot certify tracked files. */ + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags &= + ~(CE_UPTODATE | CE_FSMONITOR_VALID); + preload_index_bulk_result_clear(istate); + reused = 1; + trace2_data_intmax("diff", istate->repo, + "recovery/reused-provider-observations", 1); + +done: + clean_status_release_proof_epoch(epoch); + attr_source_snapshot_free(attrs); + return reused; +} + static void refresh_index_quietly(void) { struct lock_file lock_file = LOCK_INIT; @@ -306,8 +347,10 @@ static void refresh_index_quietly(void) return; } clean_status_set_config_digest(the_repository, &digest); - discard_index(istate); - repo_read_index(the_repository); + if (!reuse_diff_recovery_observations(istate, &source)) { + discard_index(istate); + repo_read_index(the_repository); + } if (!clean_status_index_snapshot_still_matches_proof_epoch( &source, istate)) { clean_status_index_snapshot_release(&source); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 45914588c794ec..1b450d5228c1dc 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -2282,13 +2282,22 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/readonly.trace" && GIT_TEST_PRELOAD_INDEX=1 \ - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TTCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$gitdir/reset.trace" \ git -C "$worktree" diff \ >"$gitdir/reset.actual" && test_must_be_empty "$gitdir/reset.actual" && test_trace2_data fsm_client query/trivial-response 1 \ <"$gitdir/reset.trace" && + test_trace2_data diff recovery/reused-provider-observations 1 \ + <"$gitdir/reset.trace" && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/reset.trace" >"$gitdir/reset.manifests" && + test_line_count = 1 "$gitdir/reset.manifests" && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"index\",\"label\":\"do_read_index\"" \ + "$gitdir/reset.trace" >"$gitdir/reset.reads" && + test_line_count = 1 "$gitdir/reset.reads" && test_region index do_write_index "$gitdir/reset.trace" && test_fsmonitor_full_proof "$gitdir/index" paired && cp "$gitdir/index" "$gitdir/pending.index" && @@ -2334,6 +2343,63 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'provider-reset diff never overwrites a competing skipHash writer' ' + test_when_finished "rm -rf diff-recovery-competing-writer" && + test_create_repo diff-recovery-competing-writer && + ( + cd diff-recovery-competing-writer && + sane_unset GIT_TEST_SPLIT_INDEX && + fsmonitor_query_pid= && + trap cleanup_fsmonitor_query_barrier 0 && + test_commit base tracked && + test_commit sibling sibling && + git config index.skipHash true && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_trailing_hash .git/index >.git/index.hash && + test_oid zero >.git/zero && + test_cmp .git/zero .git/index.hash && + test-tool chmtime =-60 tracked && + ready="$PWD/.git/provider.ready" && + resume="$PWD/.git/provider.resume" && + mkfifo "$resume" && + { + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_AT=2 \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_READY="$ready" \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$PWD/.git/diff.trace" \ + git diff >.git/actual 2>.git/error & + fsmonitor_query_pid=$! + } && + wait_for_fsmonitor_query_barrier \ + "$ready" "$fsmonitor_query_pid" && + test_trace2_data diff recovery/reused-provider-observations 1 \ + <.git/diff.trace && + test_path_is_missing .git/index.lock && + test_write_lines competing >sibling && + git -c core.fsmonitor=false -c core.untrackedCache=false \ + add sibling && + cp .git/index .git/competing.index && + printf x >"$resume" && + wait "$fsmonitor_query_pid" && + fsmonitor_query_pid= && + test_must_be_empty .git/actual && + test_cmp_bin .git/competing.index .git/index && + ! test_region index do_write_index .git/diff.trace && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + diff --cached --name-only >.git/staged && + test_grep "^sibling$" .git/staged + ) +' + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'builtin trivial closure can rescan and accept' ' test_when_finished "rm -rf builtin-closure-trivial" && From b4bd792a518fa087cbf394c017ac5ca427b48be7 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 21:43:12 -0500 Subject: [PATCH 13/73] dir: share normalized exclude hashes across identical blobs cae38fd1a9 (status: close fsmonitor tokens around complete status scans, 2026-07-29) rechecks cached ignore sources before pruning an fsmonitor-valid untracked tree. Older UNTR writers hash the newline appended for parsing, however, so their cached OIDs do not match the indexed blobs. A read-only status cannot persist the canonical OIDs and repeats those content checks on every invocation. Group eligible repeated index OIDs before starting the existing workers. When a worker has read a source coherently and proved both its raw and newline-appended hashes, publish that content relation for the group. Other members still require their own complete, non-racy singleton-file stat match and the normal conversion check. Unique blobs, failed mutex initialization, hardlinks, weak stat settings, and conversions retain the existing content-verification path. No extra reads or object-store lookups are needed to populate the cache. Keep both historical newline forms in a repeated read-only regression. In a 1,024-directory fixture with identical 32 KiB ignore files, three alternating runs reduced median status time from 71.2 ms to 35.9 ms and avoided 1,020 content checks. An all-unique control stayed within 1.3% of baseline. The preserved large-worktree index avoided about 883 repeated reads, with unchanged status output and physical index bytes. --- dir.c | 127 +++++++++++++++++++++++++++++++-- t/t7519-status-fsmonitor.sh | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 6 deletions(-) diff --git a/dir.c b/dir.c index 4ef901ca2d66bf..6dd004d3c81fc8 100644 --- a/dir.c +++ b/dir.c @@ -19,6 +19,7 @@ #include "gettext.h" #include "name-hash.h" #include "object-file.h" +#include "oidmap.h" #include "path.h" #include "path-namespace.h" #include "refs.h" @@ -76,8 +77,22 @@ struct cached_dir { struct untracked_cache_dir *ucd; }; +/* + * Older UNTR writers hash the extra newline used by the ignore parser. + * A checked file read can establish that representation for every path + * with the same indexed blob. Workers share only this content relation; + * each path still needs its own strong stat and conversion checks. + */ +struct normalized_exclude_oid { + struct oidmap_entry ent; + struct object_id normalized; + size_t candidates; + unsigned int valid : 1; +}; + struct untracked_cache_preload_task { struct untracked_cache_dir *ucd; + struct normalized_exclude_oid *normalized_oid; char *path; struct stat_data stat_data; struct object_id exclude_oid; @@ -88,6 +103,7 @@ struct untracked_cache_preload_task { unsigned int exclude_matches : 1; unsigned int exclude_index_present : 1; unsigned int exclude_index_candidate : 1; + unsigned int exclude_index_normalized_equivalent : 1; unsigned int exclude_index_matches : 1; unsigned int exclude_index_content_matches : 1; unsigned int normalize_exclude_oid : 1; @@ -111,14 +127,18 @@ struct untracked_cache_preload { const struct pathspec *pathspec; struct untracked_cache_preload_task *tasks; struct object_id *exclude_index_oids; + struct oidmap normalized_excludes; + pthread_mutex_t normalized_mutex; struct untracked_cache_preload_data *data; struct cache_time index_timestamp; char *exclude_per_dir; size_t nr; + size_t normalized_objects; int threads; unsigned int dir_flags; uint64_t started_at; unsigned int fsmonitor_excludes_only : 1; + unsigned int normalized_mutex_initialized : 1; }; #define UNTRACKED_CACHE_MAX_OVERLAP_THREADS 6 @@ -399,9 +419,64 @@ static void preload_fsmonitor_excludes_from_index( */ task->stat_data = ce->ce_stat_data; task->exclude_index_candidate = 1; + if (preload->normalized_mutex_initialized && + fstat_is_reliable() && !ce_stage(ce) && + !oideq(&ce->oid, &task->exclude_oid)) { + struct normalized_exclude_oid *entry = + oidmap_get(&preload->normalized_excludes, &ce->oid); + + if (!entry) { + CALLOC_ARRAY(entry, 1); + oidcpy(&entry->ent.oid, &ce->oid); + oidmap_put(&preload->normalized_excludes, entry); + } + entry->candidates++; + task->normalized_oid = entry; + } next: strbuf_release(&exclude_path); } + /* A unique blob has no other observation to share. */ + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + + if (task->normalized_oid && task->normalized_oid->candidates < 2) + task->normalized_oid = NULL; + } +} + +static int preload_normalized_exclude_matches( + struct untracked_cache_preload *preload, + const struct untracked_cache_preload_task *task) +{ + const struct normalized_exclude_oid *entry = task->normalized_oid; + int matches; + + if (!entry) + return 0; + pthread_mutex_lock(&preload->normalized_mutex); + matches = entry->valid && oideq(&entry->normalized, &task->exclude_oid); + pthread_mutex_unlock(&preload->normalized_mutex); + return matches; +} + +static void preload_remember_normalized_exclude( + struct untracked_cache_preload *preload, + struct untracked_cache_preload_task *task, + const struct object_id *raw, const struct object_id *normalized) +{ + struct normalized_exclude_oid *entry = task->normalized_oid; + + if (!entry || !oideq(raw, &entry->ent.oid) || + !oideq(normalized, &task->exclude_oid)) + return; + pthread_mutex_lock(&preload->normalized_mutex); + if (!entry->valid) { + oidcpy(&entry->normalized, normalized); + entry->valid = 1; + preload->normalized_objects++; + } + pthread_mutex_unlock(&preload->normalized_mutex); } static struct untracked_cache_preload *untracked_cache_preload_start_1( @@ -437,6 +512,10 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( fsmonitor_excludes_only, preload->pathspec); strbuf_release(&path); if (fsmonitor_excludes_only) { + if (!pthread_mutex_init(&preload->normalized_mutex, NULL)) { + preload->normalized_mutex_initialized = 1; + oidmap_init(&preload->normalized_excludes, 0); + } CALLOC_ARRAY(preload->exclude_index_oids, preload->nr); preload_fsmonitor_excludes_from_index(preload); } @@ -533,7 +612,8 @@ static void *preload_untracked_cache_thread(void *_data) struct stat st; if (preload->fsmonitor_excludes_only) { - struct object_id raw_oid; + struct object_id raw_oid, normalized_oid; + int normalized_equivalent; if (!preload->exclude_per_dir) continue; @@ -543,9 +623,12 @@ static void *preload_untracked_cache_thread(void *_data) strbuf_addch(&exclude_path, '/'); strbuf_addstr(&exclude_path, preload->exclude_per_dir); + normalized_equivalent = + preload_normalized_exclude_matches(preload, task); if (task->exclude_index_candidate && - oideq(&preload->exclude_index_oids[i], - &task->exclude_oid) && + (normalized_equivalent || + oideq(&preload->exclude_index_oids[i], + &task->exclude_oid)) && !lstat(exclude_path.buf, &st) && cached_exclude_file_matches_index_stat( &task->stat_data, &st)) { @@ -553,6 +636,9 @@ static void *preload_untracked_cache_thread(void *_data) task->exclude_index_matches = 1; task->exclude_index_content_matches = 1; task->exclude_matches = 1; + task->exclude_index_normalized_equivalent = + normalized_equivalent; + task->normalize_exclude_oid = normalized_equivalent; strbuf_release(&exclude_path); continue; } @@ -560,7 +646,11 @@ static void *preload_untracked_cache_thread(void *_data) preload->repo->hash_algo, exclude_path.buf, &task->exclude_oid, &raw_oid, - NULL, &task->exclude_mode); + task->normalized_oid ? &normalized_oid : NULL, + &task->exclude_mode); + if (task->exclude_matches && task->normalized_oid) + preload_remember_normalized_exclude( + preload, task, &raw_oid, &normalized_oid); if (task->exclude_matches && task->exclude_index_present && oideq(&preload->exclude_index_oids[i], @@ -725,6 +815,14 @@ static void untracked_cache_preload_free( trace2_data_intmax("dir", preload->repo, "preload_untracked_cache/wall_us", (getnanotime() - preload->started_at) / 1000); + if (preload->fsmonitor_excludes_only) + trace2_data_intmax("dir", preload->repo, + "preload_untracked_cache/index-normalized-objects", + preload->normalized_objects); + if (preload->normalized_mutex_initialized) { + pthread_mutex_destroy(&preload->normalized_mutex); + oidmap_clear(&preload->normalized_excludes, 1); + } free(preload->data); for (i = 0; i < preload->nr; i++) free(preload->tasks[i].path); @@ -820,10 +918,16 @@ static int update_preloaded_exclude_index_uptodate( ce = preload->istate->cache[pos]; if (!ce_stage(ce) && S_ISREG(ce->ce_mode) && oideq(&ce->oid, &preload->exclude_index_oids[task_nr])) { - if (task->exclude_index_matches) { + if (task->exclude_index_matches && + !task->exclude_index_normalized_equivalent) { converts = 0; content_matches = 1; } else { + /* + * The normalized relation only describes raw blob bytes. + * Keep conversion checks on the main thread and revalidate + * the actual file when the current path converts. + */ converts = would_convert_to_git( preload->istate, path.buf); content_matches = converts ? @@ -908,6 +1012,7 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, if (preload->fsmonitor_excludes_only) { size_t index_matches = 0; + size_t normalized_matches = 0; size_t invalidated = 0; size_t index_uptodate = 0; size_t normalized = 0; @@ -921,6 +1026,7 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, task->exclude_matches; int exclude_invalidated = !exclude_matches; int exclude_revalidated; + int index_marked; if (!exclude_matches) invalidate_gitignore(uc, task->ucd); @@ -928,10 +1034,15 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, if (task->exclude_index_matches) index_matches++; } - index_uptodate += + index_marked = update_preloaded_exclude_index_uptodate( preload, task, i, &normalized, &invalidated, &exclude_revalidated); + index_uptodate += index_marked; + if (index_marked && task->exclude_index_matches && + task->exclude_index_normalized_equivalent && + exclude_revalidated < 0) + normalized_matches++; if (exclude_matches && exclude_revalidated == 0) { invalidate_gitignore(uc, task->ucd); exclude_invalidated = 1; @@ -945,6 +1056,10 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, "dir", istate->repo, "preload_untracked_cache/index-excludes", index_matches); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-normalized-excludes", + normalized_matches); trace2_data_intmax( "dir", istate->repo, "preload_untracked_cache/index-uptodate", diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 1b450d5228c1dc..3adb495d107543 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -430,6 +430,144 @@ test_expect_success UNTRACKED_CACHE \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'historical normalized excludes reuse authenticated index stats' ' + test_when_finished "rm -rf normalized-excludes-lf normalized-excludes-no-lf" && + for ending in lf no-lf + do + test_create_repo "normalized-excludes-$ending" && + ( + cd "normalized-excludes-$ending" && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir one two && + if test "$ending" = lf + then + printf "ignored\n" >one/.gitignore + else + printf ignored >one/.gitignore + fi && + cp one/.gitignore two/.gitignore && + test_write_lines hidden >one/ignored && + test_write_lines hidden >two/ignored && + git add one/.gitignore two/.gitignore && + git commit -qm base && + test-tool chmtime -120 one/.gitignore two/.gitignore && + git update-index --refresh && + git config core.untrackedCache true && + raw=$(git rev-parse :one/.gitignore) && + normalized=$( + { cat one/.gitignore && printf "\n"; } | + git hash-object --stdin + ) && + test "$raw" != "$normalized" && + + # Exercise the genuine historical add_patterns() encoding. + git update-index --assume-unchanged \ + one/.gitignore two/.gitignore && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/historical && + test_must_be_empty .git/historical && + test-tool dump-untracked-cache >.git/historical.dump && + test_grep "^/one/ $normalized .*valid" \ + .git/historical.dump && + test_grep "^/two/ $normalized .*valid" \ + .git/historical.dump && + git update-index --no-assume-unchanged \ + one/.gitignore two/.gitignore && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSMN .git/index && + test_grep FSUC .git/index && + test_grep FSCF .git/index && + cat >.git/restore-historical-excludes.pl <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my ($algorithm, $raw_hex, $normalized_hex) = @ARGV; + my $size = $algorithm eq "sha256" ? 32 : 20; + my $body = substr($index, 0, -$size); + my $offset = index($body, "UNTR"); + die "missing UNTR extension\n" if $offset < 0; + my $length = unpack("N", substr($body, $offset + 4, 4)); + die "invalid UNTR size\n" + if $offset + 8 + $length > length($body); + my $payload = substr($body, $offset + 8, $length); + my $raw = pack("H*", $raw_hex); + my $normalized = pack("H*", $normalized_hex); + my $cursor = 0; + my $replaced = 0; + while (($cursor = index($payload, $raw, $cursor)) >= 0) { + substr($payload, $cursor, $size, $normalized); + $cursor += $size; + $replaced++; + } + die "expected exactly two historical excludes\n" + unless $replaced == 2; + substr($body, $offset + 8, $length, $payload); + print $body, + $size == 32 ? sha256($body) : sha1($body); + EOF + perl .git/restore-historical-excludes.pl \ + "$(test_oid algo)" "$raw" "$normalized" \ + <.git/index >.git/index.historical && + mv .git/index.historical .git/index && + test-tool dump-untracked-cache >.git/restored.dump && + test_grep "^/one/ $normalized " .git/restored.dump && + test_grep "^/two/ $normalized " .git/restored.dump && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expect && + cp .git/index .git/readonly.index && + for run in first second + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/$run.trace" \ + git status --porcelain=v2 \ + >".git/$run.actual" && + test_cmp .git/expect ".git/$run.actual" && + test_cmp_bin .git/readonly.index .git/index && + test_trace2_data fsmonitor config/coherent 1 \ + <".git/$run.trace" && + test_trace2_data dir \ + preload_untracked_cache/index-excludes 1 \ + <".git/$run.trace" && + test_trace2_data dir \ + preload_untracked_cache/index-normalized-excludes 1 \ + <".git/$run.trace" && + test_trace2_data dir \ + preload_untracked_cache/index-normalized-objects 1 \ + <".git/$run.trace" && + test_trace2_data dir \ + preload_untracked_cache/index-uptodate 2 \ + <".git/$run.trace" && + test_trace2_data dir \ + preload_untracked_cache/index-invalidated 0 \ + <".git/$run.trace" && + test_trace2_data dir \ + preload_untracked_cache/normalized-excludes 2 \ + <".git/$run.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <".git/$run.trace" && + ! test_trace2_data index refresh/sum_lstat \ + "[1-9][0-9]*" <".git/$run.trace" && + ! test_region index do_write_index \ + ".git/$run.trace" || return 1 + done + ) || return 1 + done +' + test_expect_success UNTRACKED_CACHE,HARDLINKS,POSIXPERM,SANITY \ 'fsmonitor rechecks cached unreadable per-directory excludes' ' test_when_finished "rm -rf fsmonitor-unreadable-exclude" && From f5d69f4ebb5eaed0b7b95065a8d8d9b8721e6f35 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 22:02:25 -0500 Subject: [PATCH 14/73] status: admit clean sidecars for inactive configured filters e4f0e5486f (status: issue sidecars after a verified full scan, 2026-07-28) refuses to issue a clean-status sidecar when any clean filter is configured. The reader imposes the same restriction. Even after semantic verification proves that no tracked path uses a filter, a repository with a global Git LFS configuration must read its index on every clean status. Allow issuance in the existing configured-filter proof domain only when the current configuration and semantic hashes match the supplied digest, the filter scope is authenticated and inactive, and the usual complete history and closed-token checks pass. Readers retain the exact configuration, provider, attribute, exclude, index, and hardlink checks. No sidecar format change is needed: previous issuers could not write a sidecar in this proof domain. A complete command-line filter-disable override intentionally shares the underlying configuration digest. Record that normalization in the in-memory digest and reject such invocations for both issuance and reuse, so a temporarily disabled filter cannot prime a clean proof. Cover consecutive read-only hits without index reads, configuration changes, active filters, external attributes, and the normalized override boundary. On a 100,000-file repository with an unused required LFS filter, five alternating read-only pairs against the same physical index reduce the median from 24.4 ms to 20.8 ms. Each new reader avoids the 8.8 MB index entirely, without changing the index or sidecar. --- clean-status-config.c | 3 + clean-status-config.h | 1 + clean-status-fast.c | 7 +- clean-status-sidecar-issue.c | 24 ++++- t/t7530-status-clean-sidecar.sh | 126 +++++++++++++++++++++++++++ t/unit-tests/u-clean-status-config.c | 3 + 6 files changed, 159 insertions(+), 5 deletions(-) diff --git a/clean-status-config.c b/clean-status-config.c index 547359faee2130..99b8d60377d817 100644 --- a/clean-status-config.c +++ b/clean-status-config.c @@ -224,6 +224,9 @@ static void flush_pending_filter(struct clean_status_config_digest *digest) if (!pending) return; + /* Remember command overrides omitted from the authenticated digest. */ + if (pending->mask == CLEAN_STATUS_FILTER_COMPLETE) + digest->normalized_filter_disable = 1; for (unsigned i = 0; i < pending->nr; i++) { struct clean_status_pending_filter_entry *entry = &pending->entries[i]; diff --git a/clean-status-config.h b/clean-status-config.h index 7e6fe8189a61a3..779a45e720aed3 100644 --- a/clean-status-config.h +++ b/clean-status-config.h @@ -19,6 +19,7 @@ struct clean_status_config_digest { unsigned initialized : 1; unsigned finalized : 1; unsigned filter_configured : 1; + unsigned normalized_filter_disable : 1; unsigned semantic_config_explicit : 1; unsigned attribute_tree_configured : 1; unsigned fsmonitor_value_seen : 1; diff --git a/clean-status-fast.c b/clean-status-fast.c index b72f1e592374ac..e0e2661b2be726 100644 --- a/clean-status-fast.c +++ b/clean-status-fast.c @@ -216,7 +216,8 @@ int clean_status_try_sidecar( int ret = 0; *repository_inputs_changed = 0; - if (!config->finalized || config->filter_configured || + if (!config->finalized || + (config->filter_configured && config->normalized_filter_disable) || getenv(INDEX_ENVIRONMENT) || is_bare_repository(repo) || !repo_get_work_tree(repo) || !current_worktree_is_main(repo) || @@ -291,7 +292,9 @@ int clean_status_try_sidecar( } if (clean_status_config_read_repository(repo, &fresh_config) || - fresh_config.filter_configured || + fresh_config.filter_configured != config->filter_configured || + (fresh_config.filter_configured && + fresh_config.normalized_filter_disable) || memcmp(fresh_config.hash, config->hash, repo->hash_algo->rawsz)) { trace_miss(repo, "fast-config-raced"); diff --git a/clean-status-sidecar-issue.c b/clean-status-sidecar-issue.c index d046307b2a7d34..91cd4bc9960d25 100644 --- a/clean-status-sidecar-issue.c +++ b/clean-status-sidecar-issue.c @@ -58,11 +58,29 @@ static int output_is_certifiable(const struct wt_status *status, !status->ignored.nr; } -static int history_is_certifiable(const struct index_state *istate) +static int history_is_certifiable( + const struct index_state *istate, + const struct clean_status_config_digest *config) { const struct clean_status_state *state = istate->clean_status; + /* + * The configured-filter proof domain requires an authenticated, + * fully classified inactive scope. A normalized disabled-filter + * override shares that digest and is never certifiable. + */ return state && + state->filter_configured == config->filter_configured && + (!config->filter_configured || + (!config->normalized_filter_disable && + state->current_config_valid && + state->current_semantic_valid && + !memcmp(state->current_config_hash, config->hash, + istate->repo->hash_algo->rawsz) && + !memcmp(state->current_semantic_hash, config->semantic_hash, + istate->repo->hash_algo->rawsz) && + state->filter_scope_valid && + !clean_status_filter_scope_needs_validation(istate))) && clean_status_has_persistent_fsmonitor_semantic_history(istate) && clean_status_revalidated_token_matches(istate) && state->manifest.current_valid && @@ -211,12 +229,12 @@ int clean_status_issue_sidecar( int installed = 0; if (!is_lock_file_locked(index_lock) || - !config->finalized || config->filter_configured || + !config->finalized || !output_is_certifiable(status, normal_clean_query)) { trace_miss(repo, "issue-command-or-output"); goto done; } - if (!history_is_certifiable(istate)) { + if (!history_is_certifiable(istate, config)) { trace_miss(repo, "issue-coherent-history"); goto done; } diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index c6d9b2330df9ab..c417b5547d3ad9 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -290,6 +290,132 @@ test_expect_success DURABLE_FSMONITOR \ test_grep ! "\"label\":\"do_read_index\"" hit.trace ' +test_expect_success DURABLE_FSMONITOR \ + 'inactive configured filters can issue authenticated clean sidecars' ' + test_when_finished "stop_daemon sidecar-inactive-filter" && + setup_repo sidecar-inactive-filter && + git -C sidecar-inactive-filter config core.autocrlf false && + git -C sidecar-inactive-filter config core.untrackedCache true && + git -C sidecar-inactive-filter config filter.sidecar.clean cat && + git -C sidecar-inactive-filter config filter.sidecar.smudge cat && + git -C sidecar-inactive-filter config filter.sidecar.process \ + "missing-inactive-filter-process" && + git -C sidecar-inactive-filter config filter.sidecar.required true && + prime_semantic_history sidecar-inactive-filter && + cp sidecar-inactive-filter/.git/index inactive-filter.index && + + test_env GIT_TRACE2_EVENT="$PWD/inactive-filter.issue.trace" \ + bulk_status -C sidecar-inactive-filter \ + status --porcelain=v2 >inactive-filter.issue && + test_must_be_empty inactive-filter.issue && + test_cmp_bin inactive-filter.index \ + sidecar-inactive-filter/.git/index && + test_trace2_data status clean-proof/sidecar 1 \ + inactive-filter.config && + test_must_be_empty inactive-filter.config && + test_trace2_data status clean-proof/miss fast-config-changed \ + inactive-filter.disabled-read && + test_must_be_empty inactive-filter.disabled-read && + test_trace2_data status clean-proof/miss fast-repository-shape \ + inactive-filter.disabled-issue && + test_must_be_empty inactive-filter.disabled-issue && + test_path_is_missing sidecar-inactive-filter/.git/index.csts && + test_grep ! "\"key\":\"clean-proof/sidecar\"" \ + inactive-filter.disabled-issue.trace && + + bulk_status -C sidecar-inactive-filter \ + status --porcelain=v2 >inactive-filter.reissue && + test_must_be_empty inactive-filter.reissue && + test_path_is_file sidecar-inactive-filter/.git/index.csts && + test_write_lines "tracked -text" \ + >sidecar-inactive-filter/.git/info/attributes && + assert_fallback_matches_oracle sidecar-inactive-filter \ + inactive-filter.external-attrs.trace && + test_trace2_data status clean-proof/miss \ + fast-repository-unavailable \ + active-filter.issue && + test_must_be_empty active-filter.issue && + test_path_is_file sidecar-active-filter/.git/index.csts && + + test_write_lines "tracked filter=sidecar" \ + >sidecar-active-filter/.gitattributes && + assert_fallback_matches_oracle sidecar-active-filter \ + active-filter.activation.trace && + test_grep "^1 \\.M .* tracked$" actual && + test_grep "^? \\.gitattributes$" actual && + + git -c filter.sidecar.clean= \ + -c filter.sidecar.smudge= \ + -c filter.sidecar.process= \ + -c filter.sidecar.required=false \ + -C sidecar-active-filter add .gitattributes && + git -c filter.sidecar.clean= \ + -c filter.sidecar.smudge= \ + -c filter.sidecar.process= \ + -c filter.sidecar.required=false \ + -C sidecar-active-filter commit -qm "activate disabled filter" && + rm -f sidecar-active-filter/.git/index.csts && + test_env GIT_TRACE2_EVENT="$PWD/active-filter.disabled.trace" \ + bulk_status -c filter.sidecar.clean= \ + -c filter.sidecar.smudge= \ + -c filter.sidecar.process= \ + -c filter.sidecar.required=false \ + -C sidecar-active-filter status --porcelain=v2 \ + >active-filter.disabled && + test_must_be_empty active-filter.disabled && + test_path_is_missing sidecar-active-filter/.git/index.csts && + test_grep ! "\"key\":\"clean-proof/sidecar\"" \ + active-filter.disabled.trace && + assert_fallback_matches_oracle sidecar-active-filter \ + active-filter.restored.trace && + test_grep "^1 \\.M .* tracked$" actual +' + test_expect_success DURABLE_FSMONITOR \ 'ordinary clean status installs its first missing sidecar' ' test_when_finished "stop_daemon sidecar-plain-first" && diff --git a/t/unit-tests/u-clean-status-config.c b/t/unit-tests/u-clean-status-config.c index 07065b3fbe6bac..2df1ac6fad30e0 100644 --- a/t/unit-tests/u-clean-status-config.c +++ b/t/unit-tests/u-clean-status-config.c @@ -207,6 +207,7 @@ void test_clean_status_config__only_complete_disabled_filters_are_normalized(voi clean_status_config_init(&baseline, algo); clean_status_config_add(&baseline, keys[0], "configured", &ctx); clean_status_config_final(&baseline); + cl_assert(!baseline.normalized_filter_disable); for (unsigned mask = 0; mask < (1U << ARRAY_SIZE(keys)); mask++) { clean_status_config_init(&digest, algo); @@ -221,6 +222,8 @@ void test_clean_status_config__only_complete_disabled_filters_are_normalized(voi clean_status_config_final(&digest); cl_assert_equal_i(hasheq(digest.hash, baseline.hash, algo), !mask || mask == 15); + cl_assert_equal_i(digest.normalized_filter_disable, + mask == 15); if (mask == 15) { cl_assert(hasheq(digest.semantic_hash, baseline.semantic_hash, algo)); From 44b6b8b7a148507deabdb1b23493830d8176c155 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 22:10:43 -0500 Subject: [PATCH 15/73] exclude: avoid an intermediate reopen of regular sources Scoped directory-delta recovery must revalidate its exclude sources before retaining already scanned siblings. Removing that check would miss a changed ignore rule, but reopening each tiny source repeatedly is expensive. A 2,050-directory fixture spends about 155 ms validating only 20 KiB of ignore-file contents. After reading a regular source, use anchored fstatat() for the first pathname identity check. Preserve the follow policy and compare the same complete identity. The final check still reopens the source for reading, and the complete content hash and parent checks are unchanged. Nonregular sources and unsupported platforms retain their old path. Five alternating runs against the same initial index reduce the median exclude-validation phase from 155 ms to 129 ms and complete recovery from 476 ms to 447 ms. Both builds produce identical repaired indexes. Extend the unit tests to validate followed regular symlinks and reject same-content symlink replacement under a no-follow policy. --- exclude-source-proof.c | 19 +++++++++++- t/unit-tests/u-exclude-source-proof.c | 44 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/exclude-source-proof.c b/exclude-source-proof.c index 83ee1626d93746..8dba2273540a4a 100644 --- a/exclude-source-proof.c +++ b/exclude-source-proof.c @@ -259,6 +259,23 @@ static int source_matches(struct exclude_source_capture *capture, return ret; } +static int source_matches_after_read(struct exclude_source_capture *capture, + const struct stat *expected) +{ +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + if (S_ISREG(expected->st_mode)) { + struct stat named; + int flags = capture->nofollow ? AT_SYMLINK_NOFOLLOW : 0; + + /* The final reopened descriptor still proves readability. */ + return !fstatat(capture->parent_fd, capture->relative, + &named, flags) && + path_namespace_stat_equal(expected, &named); + } +#endif + return source_matches(capture, expected); +} + static int same_observation( const struct exclude_source_proof_entry *entry, int exists, size_t size, const struct object_id *oid) @@ -384,7 +401,7 @@ static int proof_entry_matches( if ((size_t)read_in_full(fd, buf, size) != size || fstat(fd, &after) || !path_namespace_stat_equal(&before, &after) || - !source_matches(capture, &after)) + !source_matches_after_read(capture, &after)) goto done; hash_object_file(proof->istate->repo->hash_algo, buf, size, OBJ_BLOB, &oid); diff --git a/t/unit-tests/u-exclude-source-proof.c b/t/unit-tests/u-exclude-source-proof.c index 4ac7690f2bf286..21e7cc33c5870c 100644 --- a/t/unit-tests/u-exclude-source-proof.c +++ b/t/unit-tests/u-exclude-source-proof.c @@ -263,6 +263,7 @@ void test_exclude_source_proof__honors_nofollow(void) char *parent = make_path("parent"); char *source = make_path("parent/source"); char *target = make_path("parent/target"); + char *replacement = make_path("parent/replacement"); int fd; cl_must_pass(mkdir(parent, 0700)); @@ -275,12 +276,54 @@ void test_exclude_source_proof__honors_nofollow(void) cl_must_pass(fd); cl_must_pass(close(fd)); exclude_source_capture_release(capture); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + write_file_buf(replacement, "content", 7); + cl_must_pass(unlink(source)); + cl_must_pass(symlink("replacement", source)); + cl_assert(exclude_source_proof_validate(proof)); capture = exclude_source_capture_begin(proof, source, 1); cl_assert(capture != NULL); fd = exclude_source_capture_open(capture); cl_assert(fd < 0 && errno == ELOOP); exclude_source_capture_release(capture); + write_file_buf(replacement, "changed", 7); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(replacement); + free(target); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_nofollow_symlink_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + char *target = make_path("parent/target"); + struct stat st; + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + write_file_buf(target, "content", 7); + capture = exclude_source_capture_begin(proof, source, 1); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + exclude_source_capture_record(capture, fd, &st, "content", 7); + exclude_source_capture_release(capture); + cl_must_pass(close(fd)); + cl_assert(exclude_source_proof_validate(proof)); + + cl_must_pass(unlink(source)); + cl_must_pass(symlink("target", source)); + cl_assert(!exclude_source_proof_validate(proof)); exclude_source_proof_release(proof); free(target); @@ -450,6 +493,7 @@ SKIP_TEST(test_exclude_source_proof__digest_deduplicates_and_ignores_identity) SKIP_TEST(test_exclude_source_proof__rejects_open_failure) SKIP_TEST(test_exclude_source_proof__fails_closed_without_parent_opener) SKIP_TEST(test_exclude_source_proof__honors_nofollow) +SKIP_TEST(test_exclude_source_proof__rejects_nofollow_symlink_replacement) SKIP_TEST(test_exclude_source_proof__opens_directory_sources) SKIP_TEST(test_exclude_source_proof__accepts_same_content_parent_replacement) SKIP_TEST(test_exclude_source_proof__reresolves_absent_source_parent) From cdcd1c4d8d8bc38769fe9e5035d374233ca21f0a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 22:18:47 -0500 Subject: [PATCH 16/73] status: preserve legacy proof epochs across preload tuning 4bc13b15c9 (status: ignore command-scoped preload tuning in proofs, 2026-08-13) excludes both preload settings from authenticated status configuration hashes. Legacy FSCF proofs lack an explicit tracked-policy hash, so migrating them across harmless configuration drift also checks that configuration sources predate the index. That separate check recognizes core.preloadIndexBulk but not core.preloadIndex. Consequently, adding core.preloadIndex to an otherwise safe migration discards the authenticated tracked state. In the existing 258-path legacy fixture, enabling preload sends all 258 entries through bulk preload; disabling it performs 258 individual stats. The unmodified control refreshes only the reported path. Use the existing command-scoped acceleration predicate for legacy migration too. The proof-neutral key set and all source-epoch checks remain unchanged. Extend the migration regression to require preserved tracked state and no full preload, refresh, or manifest scan for both values. --- clean-status-config.c | 2 +- t/t7527-builtin-fsmonitor.sh | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/clean-status-config.c b/clean-status-config.c index 99b8d60377d817..3740bb929ef000 100644 --- a/clean-status-config.c +++ b/clean-status-config.c @@ -382,7 +382,7 @@ static int config_epoch_command_is_safe( { return starts_with(key, "advice.") || !strcmp(key, "user.name") || !strcmp(key, "user.email") || - !strcmp(key, "core.preloadindexbulk") || + config_is_command_acceleration(key, ctx) || config_is_command_transport(key, ctx); } diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 6b4c7c11a8b913..c1d5587122d3de 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -3660,6 +3660,41 @@ test_expect_success MACOS,UNTRACKED_CACHE,PERL_TEST_HELPERS,SEMANTIC_VERIFY_ANCH ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <.git/repeat.trace && + for legacy_preload in true false + do + cp .git/index.legacy .git/index && + rm -f .git/index.csts .git/index.csh1.* \ + .git/index.cswi.* && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked.txt \ + GIT_TRACE2_EVENT="$PWD/.git/legacy-preload-$legacy_preload.trace" \ + git -c user.name=Legacy \ + -c core.preloadIndex="$legacy_preload" \ + status --porcelain=v2 \ + >".git/legacy-preload-$legacy_preload.actual" && + test_cmp .git/legacy.expect \ + ".git/legacy-preload-$legacy_preload.actual" || + return 1 + done && + for legacy_preload in true false + do + test_trace2_data fsmonitor config/tracked-epoch-preserved 1 \ + <".git/legacy-preload-$legacy_preload.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <".git/legacy-preload-$legacy_preload.trace" && + ! test_trace2_data index preload/bulk_useful \ + "[1-9][0-9]*" \ + <".git/legacy-preload-$legacy_preload.trace" && + ! test_trace2_data index preload/sum_lstat \ + "\([2-9]\|[1-9][0-9][0-9]*\)" \ + <".git/legacy-preload-$legacy_preload.trace" && + ! test_trace2_data index refresh/sum_lstat \ + "\([2-9]\|[1-9][0-9][0-9]*\)" \ + <".git/legacy-preload-$legacy_preload.trace" || + return 1 + done && + cp .git/index.legacy .git/index && rm -f .git/index.csts .git/index.csh1.* .git/index.cswi.* && GIT_OPTIONAL_LOCKS=0 git -c advice.statusHints=true \ From 26874a9298d5fd82ec4d14e05ea3f6b0b3e2d578 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 22:30:29 -0500 Subject: [PATCH 17/73] fsmonitor: bound directory reuse by attribute sources 7c19bf467b (fsmonitor: retain checked manifests across scoped directory deltas, 2026-08-15) limits both the tracked entries and the distinct attribute sources examined for a directory event. The entry limit makes a flat directory's 65th tracked file trigger a second full manifest and a whole-worktree retry even when the directory has one attribute source. Bound the expensive attribute-source checks, not the in-memory entry walk. Keep every entry eligibility check and the independent 64-source limit. The provider callback already has to invalidate each affected entry, and none of those content checks is removed. Configuration, namespace, index, attribute, and exclude proofs remain unchanged. In five alternating runs with 2,048 unaffected sibling directories, median recovery fell from 278 ms to 155 ms for a 65-file cone, and from 316 ms to 206 ms for a 1,024-file cone. Both cases build one manifest and retry three directories instead of rebuilding the manifest and visiting 2,050 directories. Extend the regression to require the exact number of refreshed entries, retain the distinct-source limit, and exercise attribute and ignore changes while a large directory is being closed. --- clean-status-manifest.c | 5 +- t/t7519-status-fsmonitor.sh | 95 ++++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/clean-status-manifest.c b/clean-status-manifest.c index d668abf8da01b4..fe881e44e38edd 100644 --- a/clean-status-manifest.c +++ b/clean-status-manifest.c @@ -239,7 +239,7 @@ int clean_status_manifest_directory_unchanged( struct string_list candidates = STRING_LIST_INIT_DUP; struct strbuf candidate = STRBUF_INIT; const struct git_hash_algo *algo = istate->repo->hash_algo; - unsigned int first, count = 0, namespace_unstable = 0; + unsigned int first, namespace_unstable = 0; size_t len; int pos, pinned = 0, safe = 0; uint32_t required = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | @@ -303,12 +303,13 @@ int clean_status_manifest_directory_unchanged( strbuf_addstr(&candidate, directory); strbuf_addstr(&candidate, GITATTRIBUTES_FILE); string_list_insert(&candidates, candidate.buf); + /* Bound attribute-source I/O, not the affected in-memory entries. */ for (unsigned int i = first; i < istate->cache_nr && starts_with(istate->cache[i]->name, directory); i++) { const struct cache_entry *ce = istate->cache[i]; const char *slash = ce->name + len; - if (++count > 64 || ce_stage(ce) || ce_skip_worktree(ce) || + if (ce_stage(ce) || ce_skip_worktree(ce) || ce_intent_to_add(ce) || (ce->ce_flags & CE_VALID) || S_ISSPARSEDIR(ce->ce_mode)) goto done; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 3adb495d107543..50c352e73a523c 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3327,8 +3327,8 @@ test_expect_success HARDLINKS,!MINGW,!CYGWIN \ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'second closing-query change preserves verified sibling subtrees' ' test_when_finished \ - "rm -rf second-query-changed-file second-query-changed-directory" && - for event in file directory + "rm -rf second-query-changed-file second-query-changed-directory second-query-changed-large-directory" && + for event in file directory large-directory do test_create_repo "second-query-changed-$event" && ( @@ -3339,6 +3339,14 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_write_lines "*.ignored" >cached/.gitignore && printf "aaaa\n" >cached/tracked && test_write_lines ignored >cached/junk.ignored && + if test "$event" = large-directory + then + for descendant in $(test_seq 1 128) + do + test_write_lines "$descendant" \ + >"cached/retained-$descendant" || return 1 + done + fi && for sibling in $(test_seq 1 12) do mkdir "sibling-$sibling" && @@ -3348,6 +3356,10 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ >"sibling-$sibling/retained.ignored" || return 1 done && git add .gitignore cached/.gitignore cached/tracked sibling-* && + if test "$event" = large-directory + then + git add cached/retained-* + fi && git commit -m base && git config core.trustctime false && git config core.checkStat minimal && @@ -3373,7 +3385,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ -c core.trustctime=true -c core.checkStat=default \ status --porcelain=v2 >.git/expect && - if test "$event" = directory + if test "$event" != file then changed_path=cached/ else @@ -3395,7 +3407,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <.git/status.trace && test_trace2_data fsmonitor token_closure/apply_count 1 \ <.git/status.trace && - if test "$event" = directory + if test "$event" != file then test_trace2_data fsmonitor \ semantic/manifest-directory-reused 1 \ @@ -3437,8 +3449,14 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test "$initial_opened" -gt 8 && test $((retry_opened - initial_opened)) -gt 0 && test $((retry_opened - initial_opened)) -le 2 && - test_trace2_data index refresh/sum_lstat "[0-2]" \ - <.git/status.trace && + if test "$event" = large-directory + then + test_trace2_data index refresh/sum_lstat 130 \ + <.git/status.trace + else + test_trace2_data index refresh/sum_lstat "[0-2]" \ + <.git/status.trace + fi && test_trace2_data status \ fsmonitor_token/untracked-after-retry 1 \ <.git/status.trace && @@ -3449,6 +3467,63 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ done ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'directory closure rejects too many distinct attribute candidates' ' + test_when_finished "rm -rf directory-many-attribute-candidates" && + test_create_repo directory-many-attribute-candidates && + ( + cd directory-many-attribute-candidates && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached sibling && + for descendant in $(test_seq 1 64) + do + mkdir "cached/child-$descendant" && + printf "aaaa\n" \ + >"cached/child-$descendant/tracked" || return 1 + done && + test_write_lines retained >sibling/tracked && + git add cached sibling && + git commit -qm base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test_write_lines visible >sibling/visible && + git -c core.fsmonitor=false status --porcelain=v2 >.git/prime && + test_grep "^? sibling/visible$" .git/prime && + test-tool chmtime =-60 cached/child-1/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get cached/child-1/tracked) && + printf "bbbb\n" >cached/child-1/tracked && + test-tool chmtime =$mtime cached/child-1/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid cached/child-1/tracked && + GIT_OPTIONAL_LOCKS=0 git \ + -c core.fsmonitor=false -c core.untrackedCache=false \ + -c core.trustctime=true -c core.checkStat=default \ + status --porcelain=v2 >.git/expect && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/ \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "^1 \\.M .* cached/child-1/tracked$" .git/actual && + test_grep "^? sibling/visible$" .git/actual && + test_trace2_data fsmonitor semantic/manifest-scan-count 2 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/manifest-directory-reused 1 \ + <.git/status.trace && + ! test_trace2_data status \ + fsmonitor_token/reused-semantic-subtrees 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_fsmonitor_full_proof .git/index paired + ) +' + test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'directory closure rejects raced attributes and rechecks raced excludes' ' test_when_finished "rm -rf directory-race-attributes directory-race-ignore" && @@ -3465,13 +3540,19 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_write_lines "*.ignored" >cached/.gitignore && printf "aaaa\n" >cached/tracked && test_write_lines ignored >cached/junk.ignored && + for descendant in $(test_seq 1 128) + do + test_write_lines "$descendant" \ + >"cached/retained-$descendant" || return 1 + done && for sibling in $(test_seq 1 8) do mkdir "sibling-$sibling" && test_write_lines "$sibling" \ >"sibling-$sibling/tracked" || return 1 done && - git add .gitignore cached/.gitignore cached/tracked sibling-* && + git add .gitignore cached/.gitignore cached/tracked \ + cached/retained-* sibling-* && git commit -qm base && git config core.trustctime false && git config core.checkStat minimal && From fad7b3d2e6d46dbddd5ea76ef7a8957e5a9ff8bc Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 22:49:47 -0500 Subject: [PATCH 18/73] t7527: separate index and sidecar preload coverage f5d69f4ebb (status: admit clean sidecars for inactive configured filters, 2026-08-15) lets the first clean status in the preload test issue a sidecar. A later status can then return without reading the index, so the test's required config/coherent event is absent despite a valid clean-proof hit. Keep the existing test on the physical-index path by disabling optional locks for both the tuned and untuned readers. Require the index to stay byte-identical and the sidecar to remain absent, while retaining the strict coherence, no-manifest, and no-refresh assertions. Exercise the same five preload configurations separately against an issued inactive-filter sidecar in t7530. Require an exact independent oracle, a clean-proof hit with no index or directory scan, and unchanged index and sidecar bytes. This covers both paths without weakening the original proof-admission test. Production behavior is unchanged. --- t/t7527-builtin-fsmonitor.sh | 8 ++++++ t/t7530-status-clean-sidecar.sh | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index c1d5587122d3de..c10f1635ff5a98 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -6855,6 +6855,8 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <.git/prime.trace && test_grep FSCF .git/index && test_grep FSUC .git/index && + test_path_is_missing .git/index.csts && + cp .git/index .git/preload.index && for label in bulk preload both bulk-false preload-false do @@ -6865,10 +6867,13 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ bulk-false) set -- -c core.preloadIndexBulk=false ;; preload-false) set -- -c core.preloadIndex=false ;; esac && + GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCC \ GIT_TRACE2_EVENT="$PWD/.git/$label.trace" \ git "$@" status --porcelain=v2 >.git/$label && test_must_be_empty .git/$label && + test_cmp_bin .git/preload.index .git/index && + test_path_is_missing .git/index.csts && test_trace2_data fsmonitor config/coherent 1 \ <.git/$label.trace && ! test_trace2_data fsmonitor semantic/initial-mismatch 1 \ @@ -6879,10 +6884,13 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ <.git/$label.trace && ! test_trace2_data index refresh/sum_lstat \ "[1-9][0-9]*" <.git/$label.trace && + GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCC \ GIT_TRACE2_EVENT="$PWD/.git/$label-plain.trace" \ git status --porcelain=v2 >.git/$label-plain && test_must_be_empty .git/$label-plain && + test_cmp_bin .git/preload.index .git/index && + test_path_is_missing .git/index.csts && test_trace2_data fsmonitor config/coherent 1 \ <.git/$label-plain.trace && ! have_t2_data_event fsmonitor semantic/manifest-scan-count \ diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index c417b5547d3ad9..40b21af45cb1f9 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -318,6 +318,51 @@ test_expect_success DURABLE_FSMONITOR \ assert_clean_sidecar_hit sidecar-inactive-filter \ sidecar-inactive-filter inactive-filter.hit-again && + for label in bulk preload both bulk-false preload-false + do + case "$label" in + bulk) set -- -c core.preloadIndexBulk ;; + preload) set -- -c core.preloadIndex ;; + both) set -- -c core.preloadIndexBulk -c core.preloadIndex ;; + bulk-false) set -- -c core.preloadIndexBulk=false ;; + preload-false) set -- -c core.preloadIndex=false ;; + esac && + cp sidecar-inactive-filter/.git/index \ + "inactive-filter-$label.index" && + cp sidecar-inactive-filter/.git/index.csts \ + "inactive-filter-$label.sidecar" && + GIT_OPTIONAL_LOCKS=0 \ + git "$@" -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -C sidecar-inactive-filter status --porcelain=v2 \ + >"inactive-filter-$label.expect" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TRACE2_EVENT="$PWD/inactive-filter-$label.trace" \ + git "$@" -C sidecar-inactive-filter \ + status --porcelain=v2 \ + >"inactive-filter-$label.actual" && + test_cmp "inactive-filter-$label.expect" \ + "inactive-filter-$label.actual" && + test_cmp_bin "inactive-filter-$label.index" \ + sidecar-inactive-filter/.git/index && + test_cmp_bin "inactive-filter-$label.sidecar" \ + sidecar-inactive-filter/.git/index.csts && + test_trace2_data status clean-proof/hit 1 \ + <"inactive-filter-$label.trace" && + test_grep ! "\"label\":\"do_read_index\"" \ + "inactive-filter-$label.trace" && + test_grep ! "\"category\":\"index\",\"label\":\"refresh\"" \ + "inactive-filter-$label.trace" && + test_grep ! "\"category\":\"index\",\"label\":\"preload" \ + "inactive-filter-$label.trace" && + test_grep ! "\"label\":\"read_directory\"" \ + "inactive-filter-$label.trace" && + test_grep ! "\"key\":\"semantic/manifest-scan-count\"" \ + "inactive-filter-$label.trace" && + test_grep ! "\"label\":\"do_write_index\"" \ + "inactive-filter-$label.trace" || return 1 + done && + GIT_OPTIONAL_LOCKS=0 \ GIT_TRACE2_EVENT="$PWD/inactive-filter.config.trace" \ git -c filter.sidecar.required=false \ From 6ebfec2b43be861bf2745676a17eaade61d7f4bc Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 23:01:28 -0500 Subject: [PATCH 19/73] dir: share normalized exclude hashes without pthreads b4bd792a51 (dir: share normalized exclude hashes across identical blobs, 2026-08-15) initializes its sharing map only after creating a mutex. With NO_PTHREADS, the dummy mutex initializer returns ENOSYS, so identical ignore files are read and hashed separately even though the sole preload worker runs synchronously. Initialize the map directly when threads are unavailable. The dummy lock and destroy operations are no-ops, and the map is still released after the worker finishes. A failed real mutex initialization continues to disable sharing in threaded builds. The existing historical-normalized-excludes regression failed with no shared object or reused index stat. It and the neighboring hardlink and conversion checks pass with NO_PTHREADS for both object hashes. --- dir.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dir.c b/dir.c index 6dd004d3c81fc8..5338ee0c7cb72c 100644 --- a/dir.c +++ b/dir.c @@ -512,7 +512,8 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( fsmonitor_excludes_only, preload->pathspec); strbuf_release(&path); if (fsmonitor_excludes_only) { - if (!pthread_mutex_init(&preload->normalized_mutex, NULL)) { + if (!HAVE_THREADS || + !pthread_mutex_init(&preload->normalized_mutex, NULL)) { preload->normalized_mutex_initialized = 1; oidmap_init(&preload->normalized_excludes, 0); } From b6fbc8e1bd1f4952fd800b36b7981f73e2a23f18 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 23:07:47 -0500 Subject: [PATCH 20/73] t7519: distinguish serial and threaded index refreshes 28dac0af16 (stash: preserve authenticated worktree proofs during creation, 2026-08-14) and 9b689dfb57 (diff: close fsmonitor tokens after complete tracked refresh, 2026-08-14) check exact index-preload counts. With NO_PTHREADS, preload_index() returns before entering that trace region, so the tests reject the serial refresh even when its output and authenticated worktree proofs are correct. Keep the exact preload counts in threaded builds and require the whole preload region to be absent otherwise. Continue checking the ordinary refresh count, index immutability, provider-token pairing, and status output in both modes. --- t/t7519-status-fsmonitor.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 50c352e73a523c..b268244d8fdbe6 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1157,8 +1157,14 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ! test_trace2_data fsmonitor \ semantic/manifest-scan-count 1 \ <"$gitdir/status-$run.trace" && - test_trace2_data index preload/sum_lstat 1 \ - <"$gitdir/status-$run.trace" && + if test_have_prereq PTHREADS + then + test_trace2_data index preload/sum_lstat 1 \ + <"$gitdir/status-$run.trace" + else + test_region ! index preload \ + "$gitdir/status-$run.trace" + fi && test_trace2_data index refresh/sum_lstat 1 \ <"$gitdir/status-$run.trace" || return 1 done && @@ -2465,8 +2471,13 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ git -C "$worktree" diff \ >"$gitdir/next.actual" && test_must_be_empty "$gitdir/next.actual" && - test_trace2_data index preload/sum_lstat 0 \ - <"$gitdir/next.trace" || return 1 + if test_have_prereq PTHREADS + then + test_trace2_data index preload/sum_lstat 0 \ + <"$gitdir/next.trace" + else + test_region ! index preload "$gitdir/next.trace" + fi || return 1 done && git config filter.lfs.process "" && git config filter.lfs.clean false && From 1257a5f602616f0ea9705b2873cf9e0cddd99cf3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 23:12:25 -0500 Subject: [PATCH 21/73] t7519: check scoped recovery with the available worker count ef7d1509c1 (status: preload full recovery for scoped pathspecs, 2026-08-13) verifies recovery with 6, 8, 12, and 16 preload workers. Those requests are intentionally ignored in NO_PTHREADS builds, where the cache is checked synchronously by one worker. The test nevertheless requires the requested parallel count and fails before checking the remaining recovery cases. Retain the existing worker counts when threads are available and test one worker otherwise. Both modes still require exact status output, accepted token closure, and correct handling of outside-directory, ignore-file, and global changes. --- t/t7519-status-fsmonitor.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index b268244d8fdbe6..cec6bde177cf52 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -2161,7 +2161,13 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ GIT_OPTIONAL_LOCKS=0 \ git -c core.fsmonitor=false -c core.untrackedCache=false \ status --porcelain=v2 -- cached/deep >.git/scoped.expect && - for workers in 6 8 12 16 + if test_have_prereq PTHREADS + then + worker_counts="6 8 12 16" + else + worker_counts=1 + fi && + for workers in $worker_counts do rm -f .git/index.csts && GIT_OPTIONAL_LOCKS=0 \ From 3a86cb8af72a961d73d477b87257485afa323513 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 23:14:33 -0500 Subject: [PATCH 22/73] status: retain recovery and progress without pthreads b76a6c5d88 (status: show delayed progress during semantic refresh, 2026-08-10) protects progress updates with a mutex. The provider-reset repair added in dfc55f903c (diff: repair provider-reset history before publishing its index, 2026-08-15) similarly protects its warning flag. Both require pthread_mutex_init() to succeed, but the NO_PTHREADS stub returns ENOSYS. Interactive status can therefore abort when progress starts, and diff silently declines an otherwise valid durable repair. Initialize these mutexes only when threads are available. Serial builds run the callbacks on the main thread and use no-op lock and destroy operations. Threaded builds retain the existing failure handling, and the recovery proof, warning, snapshot, and index-lock checks are unchanged. Add a progress unit suite that checks repository selection and the start/update/stop lifecycle without installing signal handlers. It reproduces the abort before the fix. The existing provider-reset diff test also exercises the missing serial repair. Both object-hash variants pass the complete t7519 suite with NO_PTHREADS. --- Makefile | 1 + builtin/diff.c | 3 +- clean-status.c | 2 +- t/meson.build | 1 + t/unit-tests/u-clean-status-progress.c | 51 ++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 t/unit-tests/u-clean-status-progress.c diff --git a/Makefile b/Makefile index 46601baeddb8a8..5766c2c9226990 100644 --- a/Makefile +++ b/Makefile @@ -1579,6 +1579,7 @@ CLAR_TEST_SUITES += u-clean-status-history-store CLAR_TEST_SUITES += u-clean-status-identity CLAR_TEST_SUITES += u-clean-status-index CLAR_TEST_SUITES += u-clean-status-manifest +CLAR_TEST_SUITES += u-clean-status-progress CLAR_TEST_SUITES += u-clean-status-sidecar CLAR_TEST_SUITES += u-clean-status-store CLAR_TEST_SUITES += u-ctype diff --git a/builtin/diff.c b/builtin/diff.c index 7aa66f7fda581a..d384b3b7383604 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -366,7 +366,8 @@ static void refresh_index_quietly(void) clean_status_index_snapshot_release(&source); return; } - if (pthread_mutex_init(&diff_refresh_warning_mutex, NULL)) { + if (HAVE_THREADS && + pthread_mutex_init(&diff_refresh_warning_mutex, NULL)) { clean_status_index_snapshot_release(&source); return; } diff --git a/clean-status.c b/clean-status.c index 2c17894e949baf..7409daa8dc78b0 100644 --- a/clean-status.c +++ b/clean-status.c @@ -53,7 +53,7 @@ struct clean_status_progress *clean_status_start_progress( if (repo != progress_repo) return NULL; CALLOC_ARRAY(progress, 1); - if (pthread_mutex_init(&progress->mutex, NULL)) + if (HAVE_THREADS && pthread_mutex_init(&progress->mutex, NULL)) BUG("could not initialize clean status progress mutex"); progress->display = start_delayed_progress(repo, title, total); return progress; diff --git a/t/meson.build b/t/meson.build index 0f52f5637b3066..3a6263fefd00f9 100644 --- a/t/meson.build +++ b/t/meson.build @@ -7,6 +7,7 @@ clar_test_suites = [ 'unit-tests/u-clean-status-identity.c', 'unit-tests/u-clean-status-index.c', 'unit-tests/u-clean-status-manifest.c', + 'unit-tests/u-clean-status-progress.c', 'unit-tests/u-clean-status-sidecar.c', 'unit-tests/u-clean-status-store.c', 'unit-tests/u-ctype.c', diff --git a/t/unit-tests/u-clean-status-progress.c b/t/unit-tests/u-clean-status-progress.c new file mode 100644 index 00000000000000..04560d506c60ed --- /dev/null +++ b/t/unit-tests/u-clean-status-progress.c @@ -0,0 +1,51 @@ +#define USE_THE_REPOSITORY_VARIABLE +#define GIT_TEST_PROGRESS_ONLY + +#include "unit-test.h" +#include "clean-status.h" +#include "progress.h" +#include "repository.h" + +static int previous_progress_testing; + +void test_clean_status_progress__initialize(void) +{ + previous_progress_testing = progress_testing; + progress_testing = 1; + clean_status_enable_progress(NULL); +} + +void test_clean_status_progress__cleanup(void) +{ + clean_status_enable_progress(NULL); + progress_testing = previous_progress_testing; +} + +void test_clean_status_progress__requires_enabled_repository(void) +{ + struct repository other = { 0 }; + + cl_assert_equal_p(clean_status_start_progress( + the_repository, "disabled progress", 1), NULL); + clean_status_enable_progress(the_repository); + cl_assert_equal_p(clean_status_start_progress( + &other, "other repository", 1), NULL); +} + +void test_clean_status_progress__starts_updates_and_stops(void) +{ + struct clean_status_progress *progress; + + clean_status_enable_progress(the_repository); + progress = clean_status_start_progress( + the_repository, "clean status", 2); + cl_assert(progress != NULL); + clean_status_update_progress(progress, 0); + clean_status_update_progress(progress, 1); + clean_status_update_progress(progress, 1); + clean_status_stop_progress(&progress); + cl_assert_equal_p(progress, NULL); + clean_status_update_progress(NULL, 1); + clean_status_stop_progress(&progress); + clean_status_stop_progress(NULL); +} From e36890c2b6d7887b22dbe9842d89d92b7cd8d4c8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 15 Aug 2026 23:54:44 -0500 Subject: [PATCH 23/73] status: reuse bulk results after discarding legacy caches 7d5f592039 (status: reuse complete APFS untracked preload results, 2026-07-21) lets normal status reuse a complete bulk directory scan when no untracked cache exists. A native provider can discard an unauthenticated legacy cache while reading the index, but leaves an empty replacement behind. Even with bulk preload enabled, that replacement forces status to enumerate the entire worktree again. Record the discard on the replacement cache as process-local state. For a read-only, whole-worktree status, remove that empty cache only after the existing bulk-provider admission checks succeed. The normal bulk result can then supply untracked paths. Keep usable caches, writable recovery, explicit opt-outs, and ineligible query shapes on their existing paths. A failed scan or closing query still falls back to ordinary traversal; no untracked-cache proof or index is written. A 1.16-million-entry worktree spent 24.6 seconds in the duplicate traversal. The existing no-cache control reduced the same read-only command from 34.1 to 12.6 seconds. Cover the historical cache encoding, repeated immutable reads, independent status output, and the paired, writable, disabled, and unsupported-backend cases with both hashes. --- dir.c | 4 + dir.h | 2 + t/t7519-status-fsmonitor.sh | 223 ++++++++++++++++++++++++++++++++++++ wt-status.c | 39 +++++++ 4 files changed, 268 insertions(+) diff --git a/dir.c b/dir.c index 5338ee0c7cb72c..2405796d911378 100644 --- a/dir.c +++ b/dir.c @@ -4427,11 +4427,15 @@ int untracked_cache_adopt_legacy(struct index_state *istate) void untracked_cache_discard_legacy(struct index_state *istate) { + int had_root; + if (!istate->untracked || !legacy_ident_in_untracked(istate->untracked)) return; + had_root = !!istate->untracked->root; free_untracked_cache(istate->untracked); new_untracked_cache(istate, -1); + istate->untracked->fsmonitor_legacy_discarded = had_root; trace2_data_intmax("fsmonitor", istate->repo, "untracked/legacy-discarded", 1); } diff --git a/dir.h b/dir.h index 3b5f0c704197f9..7f16e89302c72e 100644 --- a/dir.h +++ b/dir.h @@ -221,6 +221,8 @@ struct untracked_cache { unsigned int use_fsmonitor : 1; /* A lost provider boundary requires ordinary directory validation. */ unsigned int fsmonitor_revalidation : 1; + /* Process-local: a populated legacy cache was discarded on read. */ + unsigned int fsmonitor_legacy_discarded : 1; }; /** diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index cec6bde177cf52..9a77b7e31bef3f 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -61,6 +61,26 @@ test_lazy_prereq HARDLINKS ' ln hardlink-a hardlink-b ' +test_lazy_prereq STATUS_BULK_PRELOAD ' + test_create_repo status-bulk-preload-prereq && + ( + cd status-bulk-preload-prereq && + test_write_lines tracked >tracked && + test_write_lines sibling >sibling && + git add tracked sibling && + git commit -qm base && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$PWD/.git/bulk.trace" \ + git -c core.fsmonitor=false \ + -c core.preloadIndexBulk=true \ + status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data index preload/bulk_result complete \ + <.git/bulk.trace + ) +' + test_expect_success 'FSMN parser fails closed' ' test-tool read-cache --test-fsmn-parser ' @@ -994,6 +1014,209 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'discarded legacy caches reuse an explicit bulk recovery pass' ' + test_when_finished "rm -rf legacy-discard-bulk" && + if test_have_prereq STATUS_BULK_PRELOAD + then + bulk_available=yes + else + bulk_available=no + fi && + test_create_repo legacy-discard-bulk && + ( + cd legacy-discard-bulk && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/deep sibling/empty && + test_write_lines "*.ignored" >.gitignore && + test_write_lines tracked >cached/deep/tracked && + test_write_lines sibling >sibling/empty/tracked && + test_write_lines hidden >cached/deep/hidden.ignored && + test_write_lines visible >cached/deep/visible && + git add .gitignore cached/deep/tracked sibling/empty/tracked && + git commit -qm base && + test-tool chmtime -120 .gitignore cached/deep/tracked \ + sibling/empty/tracked && + git update-index --refresh && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/paired && + test_grep "^? cached/deep/visible$" .git/paired && + test_grep ! hidden.ignored .git/paired && + test_fsmonitor_full_proof .git/index paired && + test_path_is_missing .git/index.csts && + find .git -maxdepth 1 -type f \ + \( -name "index.csh1.*" -o -name "index.cswi.*" \) \ + >.git/checkpoints && + test_must_be_empty .git/checkpoints && + cp .git/index .git/paired.index && + + cat >.git/restore-legacy-untracked.pl <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my $algorithm = $ARGV[0]; + my $rawsz = $algorithm eq "sha256" ? 32 : 20; + my $body = substr($index, 0, -$rawsz); + my $offset = index($body, "UNTR"); + die "missing UNTR extension\n" if $offset < 0; + my $size = unpack("N", substr($body, $offset + 4, 4)); + die "invalid UNTR extension size\n" + if $offset + 8 + $size > length($body); + my $payload = substr($body, $offset + 8, $size); + die "missing populated UNTR directory root\n" + unless index($payload, "cached\0") >= 0 && + index($payload, "visible\0") >= 0; + my $cursor = 0; + my $byte = ord(substr($payload, $cursor++, 1)); + my $ident_length = $byte & 127; + while ($byte & 128) { + die "truncated UNTR identity length\n" + if $cursor >= length($payload); + $ident_length++; + $byte = ord(substr($payload, $cursor++, 1)); + $ident_length = ($ident_length << 7) + ($byte & 127); + } + die "truncated UNTR identity\n" + if $cursor + $ident_length > length($payload); + my $ident = substr($payload, $cursor, $ident_length); + my $suffix = ", cache version 2\0"; + die "unexpected current UNTR identity\n" + unless substr($ident, -length($suffix)) eq $suffix; + substr($ident, -length($suffix), length($suffix), "\0"); + my $length = length($ident); + my @bytes = ($length & 127); + while ($length >>= 7) { + unshift @bytes, 128 | ((--$length) & 127); + } + my $replacement = pack("C*", @bytes) . $ident . + substr($payload, $cursor + $ident_length); + substr($body, $offset, 8 + $size, + "UNTR" . pack("N", length($replacement)) . $replacement); + my $fsuc = index($body, "FSUC"); + die "missing paired FSUC extension\n" if $fsuc < 0; + my $fsuc_size = unpack("N", substr($body, $fsuc + 4, 4)); + die "invalid FSUC extension size\n" + if $fsuc + 8 + $fsuc_size > length($body); + substr($body, $fsuc, 8 + $fsuc_size, ""); + print $body, + $algorithm eq "sha256" ? sha256($body) : sha1($body); + EOF + perl .git/restore-legacy-untracked.pl "$(test_oid algo)" \ + <.git/paired.index >.git/legacy.index && + cp .git/legacy.index .git/index && + test_grep FSMN .git/index && + test_grep UNTR .git/index && + test_grep FSCF .git/index && + test_grep ! FSUC .git/index && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expect && + test_cmp .git/paired .git/expect && + test_cmp_bin .git/legacy.index .git/index && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/disabled.trace" \ + git -c core.preloadIndexBulk=false \ + status --porcelain=v2 >.git/disabled && + test_cmp .git/expect .git/disabled && + test_cmp_bin .git/legacy.index .git/index && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/disabled.trace && + test_trace2_data fsmonitor untracked/legacy-preserved 1 \ + <.git/disabled.trace && + test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <.git/disabled.trace && + ! test_trace2_data status untracked/bulk-recovery 1 \ + <.git/disabled.trace && + ! test_trace2_data index preload/bulk_untracked_complete 1 \ + <.git/disabled.trace && + test_region dir read_directory .git/disabled.trace && + ! test_region index do_write_index .git/disabled.trace && + + cp .git/paired.index .git/index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/paired.trace" \ + git -c core.preloadIndexBulk=true \ + status --porcelain=v2 >.git/paired.actual && + test_cmp .git/expect .git/paired.actual && + test_cmp_bin .git/paired.index .git/index && + test_fsmonitor_full_proof .git/index paired && + ! test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <.git/paired.trace && + ! test_trace2_data status untracked/bulk-recovery 1 \ + <.git/paired.trace && + + cp .git/legacy.index .git/index && + for run in first second + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/$run.trace" \ + git -c core.preloadIndexBulk=true \ + status --porcelain=v2 \ + >".git/$run.actual" && + test_cmp .git/expect ".git/$run.actual" && + test_cmp_bin .git/legacy.index .git/index && + test_path_is_missing .git/index.csts && + find .git -maxdepth 1 -type f \ + \( -name "index.csh1.*" \ + -o -name "index.cswi.*" \) \ + >".git/$run.checkpoints" && + test_must_be_empty ".git/$run.checkpoints" && + test_trace2_data fsm_client query/trivial-response 1 \ + <".git/$run.trace" && + test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <".git/$run.trace" && + ! test_region index do_write_index ".git/$run.trace" && + if test "$bulk_available" = yes + then + test_trace2_data status untracked/bulk-recovery 1 \ + <".git/$run.trace" && + test_trace2_data index \ + preload/bulk_untracked_complete 1 \ + <".git/$run.trace" && + test_trace2_data index \ + preload/bulk_provider_applied \ + "[1-9][0-9]*" <".git/$run.trace" && + ! test_region dir read_directory \ + ".git/$run.trace" + else + ! test_trace2_data index \ + preload/bulk_untracked_complete 1 \ + <".git/$run.trace" && + test_region dir read_directory \ + ".git/$run.trace" + fi || return 1 + done && + + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/writable.trace" \ + git -c core.preloadIndexBulk=true \ + status --porcelain=v2 >.git/writable && + test_cmp .git/expect .git/writable && + test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <.git/writable.trace && + ! test_trace2_data status untracked/bulk-recovery 1 \ + <.git/writable.trace && + test_region index do_write_index .git/writable.trace && + test_fsmonitor_full_proof .git/index paired + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'index writers report missing authenticated untracked proofs' ' test_when_finished "rm -rf missing-untracked-proof" && diff --git a/wt-status.c b/wt-status.c index 603fcbd3a67704..a048f848f69a20 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1103,6 +1103,44 @@ static int wt_status_begin_attr_snapshot(struct wt_status *s) return ret; } +static void wt_status_prepare_bulk_recovery(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + struct untracked_cache *uc = istate->untracked; + + if (!uc || !uc->fsmonitor_legacy_discarded) + return; + uc->fsmonitor_legacy_discarded = 0; + if (use_optional_locks() || uc->root || + s->show_untracked_files != SHOW_NORMAL_UNTRACKED_FILES || + s->show_ignored_mode || s->pathspec.nr || + getenv(INDEX_ENVIRONMENT) || istate != istate->repo->index || + istate->split_index || istate->sparse_index != INDEX_EXPANDED || + !fstat_is_reliable() || + !repo_config_values(s->repo)->trust_ctime || + !repo_config_values(s->repo)->check_stat || + fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC || + !fsmonitor_pending_token_from_provider(istate) || + istate->fsmonitor_untracked_valid || + istate->fsmonitor_untracked_revalidation_authenticated || + istate->fsmonitor_legacy_untracked_fallback || + uc->fsmonitor_revalidation || + clean_status_filter_scope_needs_validation(istate) || + clean_status_manifest_global_fallback(istate) || + clean_status_worktree_manifest_needs_refresh(istate) || + !preload_index_bulk_can_close_provider(istate)) + return; + + /* + * The index read discarded an unauthenticated directory tree. A + * read-only caller cannot publish its replacement, so let the + * already enabled bulk scan supply complete untracked results. + * If that scan cannot close, ordinary traversal still supplies them. + */ + remove_untracked_cache(istate); + trace2_data_intmax("status", s->repo, "untracked/bulk-recovery", 1); +} + void wt_status_start_untracked_cache_preload(struct wt_status *s) { struct index_state *istate = s->repo->index; @@ -1118,6 +1156,7 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) wt_status_begin_attr_snapshot(s); /* Record the provider token before either filesystem traversal. */ refresh_fsmonitor(istate); + wt_status_prepare_bulk_recovery(s); if (s->certify_clean_status && !fsmonitor_has_pending_token(istate)) reopened_valid_token = From 5abe1ce4dfce9b88158e2a71182eb905e99edc87 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 00:03:09 -0500 Subject: [PATCH 24/73] preload-index: select bulk for read-only cache recovery e36890c2b6 (status: reuse bulk results after discarding legacy caches, 2026-08-15) avoids a second directory traversal when bulk preload is already enabled. Ordinary read-only status still takes the serial path because core.preloadIndexBulk defaults to false, even after losing a populated legacy cache and invalidating enough tracked entries to make the complete bulk scan useful. Let that exact recovery request supply the default while status closes its provider token. An explicit test override or configured boolean still wins, including false. Keep the existing useful-entry threshold, backend, index-shape, provider, attribute, exclusion, and closing-query checks. Clear the process-local request after closure so it cannot change later preload decisions. This does not enable bulk scans for healthy caches, ordinary sparse provider deltas, writable recovery, or unrelated commands. Document the narrow default exception and extend the legacy-cache regression to check automatic selection and each explicit opt-out. The automatic case previously performed a second walk even though the explicit bulk case already completed it correctly. --- Documentation/config/core.adoc | 5 ++- preload-index.c | 10 ++--- read-cache-ll.h | 2 + t/t7519-status-fsmonitor.sh | 70 ++++++++++++++++++++++------------ wt-status.c | 13 +++++-- 5 files changed, 67 insertions(+), 33 deletions(-) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 59bc4a818cceb8..6f8b4ac9372891 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -738,7 +738,10 @@ but may cost more than normal preload depending on filesystem and cache state. Inconclusive scans are discarded before continuing with the normal preload. Currently this is supported on APFS, ext-family filesystems, and XFS, and only has an effect when `core.preloadIndex` is enabled. Defaults -to false. +to false, except that a whole-worktree, read-only `git status` may use it +after the native file system monitor discards a populated legacy +untracked cache. Setting this option explicitly to false also disables +that recovery optimization. core.unsetenvvars:: Windows-only: comma-separated list of environment variables' diff --git a/preload-index.c b/preload-index.c index 95ebfff1d46787..5197ffa5e61913 100644 --- a/preload-index.c +++ b/preload-index.c @@ -322,11 +322,11 @@ static int preload_bulk_config_enabled(struct index_state *index) int control; control = git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", -1); - if (control < 0) - repo_config_get_bool(index->repo, "core.preloadindexbulk", - &enabled); - else - enabled = control; + if (control >= 0) + return control; + if (repo_config_get_bool(index->repo, "core.preloadindexbulk", + &enabled)) + enabled = index->preload_bulk_recovery_requested; return enabled; } diff --git a/read-cache-ll.h b/read-cache-ll.h index 409e4a9ca23c44..f721e1caf4e540 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -199,6 +199,8 @@ struct index_state { fsmonitor_legacy_untracked_fallback : 1, fsmonitor_pending_token_from_provider : 1, preload_untracked_complete : 1, + /* Read-only status request; never serialized. */ + preload_bulk_recovery_requested : 1, preload_bulk_provider_pending : 1, preload_bulk_excludes_digest_pending : 1, preload_bulk_excludes_digest_valid : 1; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 9a77b7e31bef3f..840d3689611a3b 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1015,7 +1015,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ' test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ - 'discarded legacy caches reuse an explicit bulk recovery pass' ' + 'discarded legacy caches select one bulk recovery pass' ' test_when_finished "rm -rf legacy-discard-bulk" && if test_have_prereq STATUS_BULK_PRELOAD then @@ -1122,26 +1122,43 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP test_cmp .git/paired .git/expect && test_cmp_bin .git/legacy.index .git/index && - GIT_OPTIONAL_LOCKS=0 \ - GIT_TEST_PRELOAD_INDEX=1 \ - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ - GIT_TRACE2_EVENT="$PWD/.git/disabled.trace" \ - git -c core.preloadIndexBulk=false \ - status --porcelain=v2 >.git/disabled && - test_cmp .git/expect .git/disabled && - test_cmp_bin .git/legacy.index .git/index && - test_trace2_data fsm_client query/trivial-response 1 \ - <.git/disabled.trace && - test_trace2_data fsmonitor untracked/legacy-preserved 1 \ - <.git/disabled.trace && - test_trace2_data fsmonitor untracked/legacy-discarded 1 \ - <.git/disabled.trace && - ! test_trace2_data status untracked/bulk-recovery 1 \ - <.git/disabled.trace && - ! test_trace2_data index preload/bulk_untracked_complete 1 \ - <.git/disabled.trace && - test_region dir read_directory .git/disabled.trace && - ! test_region index do_write_index .git/disabled.trace && + for disabled in bulk environment preload + do + case "$disabled" in + bulk) + set -- git -c core.preloadIndexBulk=false + ;; + environment) + set -- test_env GIT_TEST_PRELOAD_INDEX_BULK=0 \ + git -c core.preloadIndexBulk=true + ;; + preload) + set -- git -c core.preloadIndex=false \ + -c core.preloadIndexBulk=true + ;; + esac && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/$disabled.trace" \ + "$@" status --porcelain=v2 \ + >".git/$disabled.actual" && + test_cmp .git/expect ".git/$disabled.actual" && + test_cmp_bin .git/legacy.index .git/index && + test_trace2_data fsm_client query/trivial-response 1 \ + <".git/$disabled.trace" && + test_trace2_data fsmonitor untracked/legacy-preserved 1 \ + <".git/$disabled.trace" && + test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <".git/$disabled.trace" && + ! test_trace2_data status untracked/bulk-recovery 1 \ + <".git/$disabled.trace" && + ! test_trace2_data index preload/bulk_untracked_complete 1 \ + <".git/$disabled.trace" && + test_region dir read_directory ".git/$disabled.trace" && + ! test_region index do_write_index \ + ".git/$disabled.trace" || return 1 + done && cp .git/paired.index .git/index && GIT_OPTIONAL_LOCKS=0 \ @@ -1159,14 +1176,19 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP <.git/paired.trace && cp .git/legacy.index .git/index && - for run in first second + for run in first second auto do + if test "$run" = auto + then + set -- git + else + set -- git -c core.preloadIndexBulk=true + fi && GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_PRELOAD_INDEX=1 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$PWD/.git/$run.trace" \ - git -c core.preloadIndexBulk=true \ - status --porcelain=v2 \ + "$@" status --porcelain=v2 \ >".git/$run.actual" && test_cmp .git/expect ".git/$run.actual" && test_cmp_bin .git/legacy.index .git/index && diff --git a/wt-status.c b/wt-status.c index a048f848f69a20..f194d8001ff36b 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1108,6 +1108,7 @@ static void wt_status_prepare_bulk_recovery(struct wt_status *s) struct index_state *istate = s->repo->index; struct untracked_cache *uc = istate->untracked; + istate->preload_bulk_recovery_requested = 0; if (!uc || !uc->fsmonitor_legacy_discarded) return; uc->fsmonitor_legacy_discarded = 0; @@ -1127,14 +1128,19 @@ static void wt_status_prepare_bulk_recovery(struct wt_status *s) uc->fsmonitor_revalidation || clean_status_filter_scope_needs_validation(istate) || clean_status_manifest_global_fallback(istate) || - clean_status_worktree_manifest_needs_refresh(istate) || - !preload_index_bulk_can_close_provider(istate)) + clean_status_worktree_manifest_needs_refresh(istate)) return; + istate->preload_bulk_recovery_requested = 1; + if (!preload_index_bulk_can_close_provider(istate)) { + istate->preload_bulk_recovery_requested = 0; + return; + } /* * The index read discarded an unauthenticated directory tree. A * read-only caller cannot publish its replacement, so let the - * already enabled bulk scan supply complete untracked results. + * bulk scan supply both tracked and complete untracked results. + * Explicit configuration still takes precedence over this request. * If that scan cannot close, ordinary traversal still supplies them. */ remove_untracked_cache(istate); @@ -2371,6 +2377,7 @@ int wt_status_refresh_index(struct wt_status *s, proof = wt_status_prepare_semantic_verify(s, refresh_flags); ret = wt_status_close_fsmonitor_token( s, proof, refresh_flags, require_untracked, 0); + istate->preload_bulk_recovery_requested = 0; if (istate->preload_untracked == &s->untracked) { s->untracked_from_preload = istate->preload_untracked_complete; From 3fcb4c5a12b6c699f1a85cc3f5601d6c6578d530 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 00:40:08 -0500 Subject: [PATCH 25/73] status: recover discarded empty legacy caches in one pass In e36890c2b6 (status: reuse bulk results after discarding legacy caches, 2026-08-15), we remembered a discarded legacy cache only if it had a directory tree. A foreign index writer can leave a valid legacy UNTR extension with zero directory nodes and no FSUC or FSCF proof. The 1,160,465-entry checkout which exposed the duplicate walk has exactly that state, so the populated-cache restriction missed the real workload. Remember every actual replacement of a matching legacy cache. Newly initialized and current-format empty caches still do not qualify. Keep the existing read-only, provider, configuration, filter, and bulk-scan admission checks unchanged. Extend the regression with an empty legacy cache and a clean provider response, checking both explicit and automatic recovery. Also verify that a current-format empty cache retains ordinary traversal. On the real checkout, the complete recovery path reduces an explicitly enabled bulk status from 26.2 to 8.6 seconds by removing its second directory walk, with identical output and no index or sidecar writes. --- Documentation/config/core.adoc | 2 +- dir.c | 5 +- dir.h | 2 +- t/t7519-status-fsmonitor.sh | 104 ++++++++++++++++++++++++++++++++- wt-status.c | 2 +- 5 files changed, 105 insertions(+), 10 deletions(-) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 6f8b4ac9372891..79af4e6038f255 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -739,7 +739,7 @@ state. Inconclusive scans are discarded before continuing with the normal preload. Currently this is supported on APFS, ext-family filesystems, and XFS, and only has an effect when `core.preloadIndex` is enabled. Defaults to false, except that a whole-worktree, read-only `git status` may use it -after the native file system monitor discards a populated legacy +after the native file system monitor discards a legacy untracked cache. Setting this option explicitly to false also disables that recovery optimization. diff --git a/dir.c b/dir.c index 2405796d911378..0c474dfad0c46b 100644 --- a/dir.c +++ b/dir.c @@ -4427,15 +4427,12 @@ int untracked_cache_adopt_legacy(struct index_state *istate) void untracked_cache_discard_legacy(struct index_state *istate) { - int had_root; - if (!istate->untracked || !legacy_ident_in_untracked(istate->untracked)) return; - had_root = !!istate->untracked->root; free_untracked_cache(istate->untracked); new_untracked_cache(istate, -1); - istate->untracked->fsmonitor_legacy_discarded = had_root; + istate->untracked->fsmonitor_legacy_discarded = 1; trace2_data_intmax("fsmonitor", istate->repo, "untracked/legacy-discarded", 1); } diff --git a/dir.h b/dir.h index 7f16e89302c72e..527ba58cc653e2 100644 --- a/dir.h +++ b/dir.h @@ -221,7 +221,7 @@ struct untracked_cache { unsigned int use_fsmonitor : 1; /* A lost provider boundary requires ordinary directory validation. */ unsigned int fsmonitor_revalidation : 1; - /* Process-local: a populated legacy cache was discarded on read. */ + /* Process-local: a legacy cache was discarded on read. */ unsigned int fsmonitor_legacy_discarded : 1; }; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 840d3689611a3b..0a7ad5aeee185f 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1062,6 +1062,8 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP local $/; my $index = ; my $algorithm = $ARGV[0]; + my $empty = $ARGV[1] && $ARGV[1] =~ /^(current-)?empty$/; + my $current = $ARGV[1] && $ARGV[1] eq "current-empty"; my $rawsz = $algorithm eq "sha256" ? 32 : 20; my $body = substr($index, 0, -$rawsz); my $offset = index($body, "UNTR"); @@ -1089,14 +1091,21 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP my $suffix = ", cache version 2\0"; die "unexpected current UNTR identity\n" unless substr($ident, -length($suffix)) eq $suffix; - substr($ident, -length($suffix), length($suffix), "\0"); + substr($ident, -length($suffix), length($suffix), "\0") + unless $current; my $length = length($ident); my @bytes = ($length & 127); while ($length >>= 7) { unshift @bytes, 128 | ((--$length) & 127); } - my $replacement = pack("C*", @bytes) . $ident . - substr($payload, $cursor + $ident_length); + my $tail = substr($payload, $cursor + $ident_length); + if ($empty) { + my $exclude = index($tail, "\0", 76 + 2 * $rawsz); + die "missing UNTR per-directory exclude name\n" + if $exclude < 0; + $tail = substr($tail, 0, $exclude + 1) . "\0"; + } + my $replacement = pack("C*", @bytes) . $ident . $tail; substr($body, $offset, 8 + $size, "UNTR" . pack("N", length($replacement)) . $replacement); my $fsuc = index($body, "FSUC"); @@ -1105,6 +1114,14 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP die "invalid FSUC extension size\n" if $fsuc + 8 + $fsuc_size > length($body); substr($body, $fsuc, 8 + $fsuc_size, ""); + if ($empty) { + my $fscf = index($body, "FSCF"); + die "missing FSCF extension\n" if $fscf < 0; + my $fscf_size = unpack("N", substr($body, $fscf + 4, 4)); + die "invalid FSCF extension size\n" + if $fscf + 8 + $fscf_size > length($body); + substr($body, $fscf, 8 + $fscf_size, ""); + } print $body, $algorithm eq "sha256" ? sha256($body) : sha1($body); EOF @@ -1224,6 +1241,87 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP fi || return 1 done && + perl .git/restore-legacy-untracked.pl "$(test_oid algo)" \ + current-empty <.git/paired.index >.git/current.index && + cp .git/current.index .git/index && + test_grep FSMN .git/index && + test_grep UNTR .git/index && + test_grep ! FSCF .git/index && + test_grep ! FSUC .git/index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/current.trace" \ + git status --porcelain=v2 >.git/current.actual && + test_cmp .git/expect .git/current.actual && + test_cmp_bin .git/current.index .git/index && + test_path_is_missing .git/index.csts && + find .git -maxdepth 1 -type f \ + \( -name "index.csh1.*" -o -name "index.cswi.*" \) \ + >.git/current.checkpoints && + test_must_be_empty .git/current.checkpoints && + ! test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <.git/current.trace && + ! test_trace2_data status untracked/bulk-recovery 1 \ + <.git/current.trace && + ! test_region index do_write_index .git/current.trace && + test_region dir read_directory .git/current.trace && + + perl .git/restore-legacy-untracked.pl "$(test_oid algo)" \ + empty <.git/paired.index >.git/empty.index && + cp .git/empty.index .git/index && + test_grep FSMN .git/index && + test_grep UNTR .git/index && + test_grep ! FSCF .git/index && + test_grep ! FSUC .git/index && + for run in empty-explicit empty-auto + do + if test "$run" = empty-auto + then + set -- git + else + set -- git -c core.preloadIndexBulk=true + fi && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/$run.trace" \ + "$@" status --porcelain=v2 \ + >".git/$run.actual" && + test_cmp .git/expect ".git/$run.actual" && + test_cmp_bin .git/empty.index .git/index && + test_path_is_missing .git/index.csts && + find .git -maxdepth 1 -type f \ + \( -name "index.csh1.*" \ + -o -name "index.cswi.*" \) \ + >".git/$run.checkpoints" && + test_must_be_empty ".git/$run.checkpoints" && + test_trace2_data fsmonitor untracked/legacy-preserved 1 \ + <".git/$run.trace" && + test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <".git/$run.trace" && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <".git/$run.trace" && + ! test_region index do_write_index ".git/$run.trace" && + if test "$bulk_available" = yes + then + test_trace2_data status untracked/bulk-recovery 1 \ + <".git/$run.trace" && + test_trace2_data index \ + preload/bulk_untracked_complete 1 \ + <".git/$run.trace" && + ! test_region dir read_directory \ + ".git/$run.trace" + else + ! test_trace2_data index \ + preload/bulk_untracked_complete 1 \ + <".git/$run.trace" && + test_region dir read_directory \ + ".git/$run.trace" + fi || return 1 + done && + cp .git/legacy.index .git/index && + GIT_TEST_PRELOAD_INDEX=1 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$PWD/.git/writable.trace" \ diff --git a/wt-status.c b/wt-status.c index f194d8001ff36b..356bc948c0711d 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1137,7 +1137,7 @@ static void wt_status_prepare_bulk_recovery(struct wt_status *s) } /* - * The index read discarded an unauthenticated directory tree. A + * The index read discarded an unauthenticated legacy cache. A * read-only caller cannot publish its replacement, so let the * bulk scan supply both tracked and complete untracked results. * Explicit configuration still takes precedence over this request. From 5e6cc3058cee863a900fd4447624a5095c1c316d Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 11:46:47 -0500 Subject: [PATCH 26/73] t7519: isolate the bulk recovery test environment The environment-override case added in 5abe1ce4df (preload-index: select bulk for read-only cache recovery, 2026-08-16) selects test_env through "$@" and prefixes it with read-only and fake-provider variables. Assignments before a shell function need not have the same lifetime as an external command's environment. Linux CI exposes both a supposedly read-only control that rewrites the index and a final writable control that unexpectedly enters read-only bulk recovery. Use the external env command for the bulk override, and explicitly enable optional locks for the writable control. This keeps the two test modes independent without changing production behavior. --- t/t7519-status-fsmonitor.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 0a7ad5aeee185f..8d49966f859b4e 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1146,7 +1146,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP set -- git -c core.preloadIndexBulk=false ;; environment) - set -- test_env GIT_TEST_PRELOAD_INDEX_BULK=0 \ + set -- env GIT_TEST_PRELOAD_INDEX_BULK=0 \ git -c core.preloadIndexBulk=true ;; preload) @@ -1322,6 +1322,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP done && cp .git/legacy.index .git/index && + GIT_OPTIONAL_LOCKS=1 \ GIT_TEST_PRELOAD_INDEX=1 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$PWD/.git/writable.trace" \ From c6b2f26cf50bf3af04091c1343b11f11c505d457 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 11:57:15 -0500 Subject: [PATCH 27/73] fsmonitor: avoid full scans for partially proved private indexes In 8abeda59ec (fsmonitor: preserve authenticated proofs across ordinary commands, 2026-08-15), the conservative temporary-index path required both a complete proof and partial manifest history to be absent. A sparse add to a copied index can retain only the manifest-complete and full-index flags. That history cannot authenticate the index, but it still excludes the conservative path. The next sparse add or write-tree therefore rebuilds the complete worktree manifest. Use the existing strong invalidation path whenever a genuine private index lacks a complete proof. Keep the physical-index, index-lock, device/inode, provider, filesystem, and split-index guards unchanged. This does not promote partial history into a valid proof: tracked entries and the untracked cache are invalidated before normal processing. Extend the primary and linked-worktree regression with repeated sparse staging through one copied index. Compare its tree with independently staged, fsmonitor-disabled controls, and retain alias, attribute, and required-filter failure checks. Both hash formats reproduce the unwanted scan without this change and pass with it. --- fsmonitor.c | 3 +- t/t7519-status-fsmonitor.sh | 199 +++++++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 7 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 1800ae7174f6a0..5dccb81b25f62b 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -1241,8 +1241,7 @@ static void invalidate_fsmonitor_for_bootstrap( return; } if (getenv(INDEX_ENVIRONMENT) && - !clean_status_has_persistent_fsmonitor_semantic_history(istate) && - !clean_status_has_worktree_manifest_history(istate)) { + !clean_status_has_persistent_fsmonitor_semantic_history(istate)) { char *physical = xstrfmt("%s/index", repo_get_git_dir(istate->repo)); char *selected = real_pathdup(repo_get_index_file(istate->repo), 0); char *canonical = real_pathdup(physical, 0); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 8d49966f859b4e..096702c35025f2 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1646,6 +1646,9 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ >"$gitdir/canonical-tree" && test_region index do_write_index \ "$gitdir/canonical-tree.trace" && + ! test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/canonical-tree.trace" && ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/canonical-tree.trace" && perl "$PWD/.git/check-write-tree-proof.pl" \ @@ -1681,6 +1684,9 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ git -C "$worktree" ls-tree HEAD hook-generated \ >"$gitdir/hook-all-entry" && test_grep "hook-generated$" "$gitdir/hook-all-entry" && + ! test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/hook-all.trace" && ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/hook-all.trace" && perl "$PWD/.git/check-write-tree-proof.pl" \ @@ -1701,27 +1707,210 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ cp "$gitdir/index" "$gitdir/readonly.index" && cp "$gitdir/index" "$gitdir/snapshot.index" && test_write_lines snapshot >"$worktree/snapshot-new" && - printf "%s\n" snapshot-new | + printf "%s\0" snapshot-new | + GIT_OPTIONAL_LOCKS=0 \ GIT_INDEX_FILE="$gitdir/snapshot.index" \ + GIT_LITERAL_PATHSPECS=1 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ - GIT_TRACE2_EVENT="$gitdir/snapshot-add.trace" \ + GIT_TRACE2_EVENT="$gitdir/snapshot-prime.trace" \ git -C "$worktree" add --sparse \ - --pathspec-from-file=- && + --pathspec-from-file=- \ + --pathspec-file-nul && perl "$PWD/.git/check-write-tree-unbound.pl" \ <"$gitdir/snapshot.index" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + if test_have_prereq HARDLINKS && + test_have_prereq SYMLINKS + then + cp "$gitdir/snapshot.index" "$gitdir/index" && + ln "$gitdir/index" "$gitdir/physical-hardlink" && + ln -s "$gitdir/index" "$gitdir/physical-symlink" && + cp "$gitdir/index" "$gitdir/index.lock" && + for alias in index physical-hardlink \ + physical-symlink index.lock + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/$alias" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/alias-$alias.trace" \ + git -C "$worktree" status \ + --porcelain=v2 \ + >"$gitdir/alias-$alias" && + ! test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/alias-$alias.trace" && + test_cmp_bin "$gitdir/snapshot.index" \ + "$gitdir/index" || return 1 + done && + rm -f "$gitdir/physical-hardlink" \ + "$gitdir/physical-symlink" \ + "$gitdir/index.lock" && + cp "$gitdir/readonly.index" "$gitdir/index" + fi && + test_write_lines next >"$worktree/snapshot-next" && + printf "%s\0" snapshot-next | + GIT_OPTIONAL_LOCKS=0 \ GIT_INDEX_FILE="$gitdir/snapshot.index" \ - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_LITERAL_PATHSPECS=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/snapshot-add.trace" \ + git -C "$worktree" add --sparse \ + --pathspec-from-file=- \ + --pathspec-file-nul && + test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/snapshot-add.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/snapshot-add.trace" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/snapshot.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$gitdir/snapshot-tree.trace" \ git -C "$worktree" write-tree \ >"$gitdir/snapshot-tree" && test_file_not_empty "$gitdir/snapshot-tree" && + test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/snapshot-tree.trace" && ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <"$gitdir/snapshot-tree.trace" && git -C "$worktree" ls-tree \ - "$(cat "$gitdir/snapshot-tree")" snapshot-new \ + "$(cat "$gitdir/snapshot-tree")" \ + snapshot-new snapshot-next \ >"$gitdir/snapshot-entry" && test_grep "snapshot-new$" "$gitdir/snapshot-entry" && + test_grep "snapshot-next$" "$gitdir/snapshot-entry" && + cp "$gitdir/readonly.index" "$gitdir/control.index" && + printf "%s\0" snapshot-new snapshot-next | + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/control.index" \ + GIT_LITERAL_PATHSPECS=1 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -C "$worktree" add --sparse \ + --pathspec-from-file=- \ + --pathspec-file-nul && + GIT_INDEX_FILE="$gitdir/control.index" \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -C "$worktree" write-tree \ + >"$gitdir/snapshot-expect" && + test_cmp "$gitdir/snapshot-expect" "$gitdir/snapshot-tree" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + test_write_lines "*.filtered filter=snapshot" \ + >"$worktree/.gitattributes" && + test_write_lines raw >"$worktree/snapshot.filtered" && + printf "%s\0" .gitattributes snapshot.filtered | + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/snapshot.index" \ + GIT_LITERAL_PATHSPECS=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/filter-add.trace" \ + git -c filter.snapshot.clean="sed s/raw/converted/" \ + -c filter.snapshot.required=true \ + -C "$worktree" add --sparse \ + --pathspec-from-file=- \ + --pathspec-file-nul && + test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/filter-add.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/filter-add.trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/snapshot.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/filter-tree.trace" \ + git -c filter.snapshot.clean="sed s/raw/converted/" \ + -c filter.snapshot.required=true \ + -C "$worktree" write-tree \ + >"$gitdir/filter-tree" && + test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/filter-tree.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/filter-tree.trace" && + cp "$gitdir/readonly.index" "$gitdir/filter-control.index" && + printf "%s\0" snapshot-new snapshot-next \ + .gitattributes snapshot.filtered | + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/filter-control.index" \ + GIT_LITERAL_PATHSPECS=1 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c filter.snapshot.clean="sed s/raw/converted/" \ + -c filter.snapshot.required=true \ + -C "$worktree" add --sparse \ + --pathspec-from-file=- \ + --pathspec-file-nul && + GIT_INDEX_FILE="$gitdir/filter-control.index" \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c filter.snapshot.clean="sed s/raw/converted/" \ + -c filter.snapshot.required=true \ + -C "$worktree" write-tree \ + >"$gitdir/filter-expect" && + test_cmp "$gitdir/filter-expect" "$gitdir/filter-tree" && + git -C "$worktree" cat-file blob \ + "$(cat "$gitdir/filter-tree"):snapshot.filtered" \ + >"$gitdir/filter-actual-blob" && + test_write_lines converted >"$gitdir/filter-expect-blob" && + test_cmp "$gitdir/filter-expect-blob" \ + "$gitdir/filter-actual-blob" && + test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + test_write_lines raw >"$worktree/required-failure.filtered" && + printf "%s\0" required-failure.filtered \ + >"$gitdir/required-failure.paths" && + cp "$gitdir/snapshot.index" "$gitdir/filter-failure.index" && + cp "$gitdir/filter-failure.index" \ + "$gitdir/filter-failure.before" && + test_must_fail env \ + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/filter-failure.index" \ + GIT_LITERAL_PATHSPECS=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/filter-failure.trace" \ + git -c filter.snapshot.clean=false \ + -c filter.snapshot.required=true \ + -C "$worktree" add --sparse \ + --pathspec-from-file="$gitdir/required-failure.paths" \ + --pathspec-file-nul \ + 2>"$gitdir/filter-failure.error" && + test_grep "clean filter .snapshot. failed" \ + "$gitdir/filter-failure.error" && + test_trace2_data fsmonitor \ + semantic/temporary-index-stat-fallback 1 \ + <"$gitdir/filter-failure.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/filter-failure.trace" && + test_cmp_bin "$gitdir/filter-failure.before" \ + "$gitdir/filter-failure.index" && + cp "$gitdir/readonly.index" \ + "$gitdir/filter-control-failure.index" && + test_must_fail env \ + GIT_OPTIONAL_LOCKS=0 \ + GIT_INDEX_FILE="$gitdir/filter-control-failure.index" \ + GIT_LITERAL_PATHSPECS=1 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c filter.snapshot.clean=false \ + -c filter.snapshot.required=true \ + -C "$worktree" add --sparse \ + --pathspec-from-file="$gitdir/required-failure.paths" \ + --pathspec-file-nul \ + 2>"$gitdir/filter-control-failure.error" && + test_grep "clean filter .snapshot. failed" \ + "$gitdir/filter-control-failure.error" && + test_cmp_bin "$gitdir/readonly.index" \ + "$gitdir/filter-control-failure.index" && test_cmp_bin "$gitdir/readonly.index" "$gitdir/index" && + rm -f "$worktree/.gitattributes" \ + "$worktree/snapshot.filtered" \ + "$worktree/required-failure.filtered" && GIT_INDEX_FILE="$gitdir/manifestless.index" \ git -C "$worktree" read-tree HEAD && test_grep ! FSCF "$gitdir/manifestless.index" && From b208bc4f2298bdd37279f3b8e8835a07b82254fb Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 12:10:18 -0500 Subject: [PATCH 28/73] t7519: disarm completed query-barrier cleanups The query-barrier tests added by ef6cdb5039 (diff: reuse pinned observations during provider-reset repair, 2026-08-15) and 7c19bf467b (fsmonitor: retain checked manifests across scoped directory deltas, 2026-08-15) leave their subshell EXIT traps armed after successfully waiting for the background Git process. Under dash with --verbose-log -x, those now-empty cleanups can emit xtrace lines outside the test body's log redirection. The linux32 and linux-TEST-vars jobs pass all 97 assertions, but prove rejects the six extra lines as invalid TAP. Disarm each trap after a successful wait and clearing the saved PID. Keep it armed while the child is running, so a failed assertion still terminates and reaps the process. No production behavior changes. --- t/t7519-status-fsmonitor.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 096702c35025f2..27f8a22692afb7 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3079,6 +3079,7 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ printf x >"$resume" && wait "$fsmonitor_query_pid" && fsmonitor_query_pid= && + trap - 0 && test_must_be_empty .git/actual && test_cmp_bin .git/competing.index .git/index && ! test_region index do_write_index .git/diff.trace && @@ -4153,6 +4154,7 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ printf x >"$resume" && wait "$fsmonitor_query_pid" && fsmonitor_query_pid= && + trap - 0 && test_cmp .git/expect .git/actual && test_grep "^1 \\.M .* cached/tracked$" .git/actual && test_grep "^? sibling-1/visible$" .git/actual && From 8def7dcd435be69b494239d576de7bd3db73cec2 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 12:17:49 -0500 Subject: [PATCH 29/73] t7519: verify reuse after writable legacy-cache repair In e36890c2b6 (status: reuse bulk results after discarding legacy caches, 2026-08-15), writable status deliberately kept the ordinary directory walk so that it could rebuild durable untracked metadata. The regression verifies the resulting paired proof, but not whether the next process can reuse it. Follow that repair with a read-only status. Require the same independent oracle output and a coherent paired proof, with no new manifest scan, physical directory opens, tracked-file stats, bulk walk, or index write. Also compare the physical index and all existing history sidecars before and after the command. Cached untracked-directory nodes may still be visited to produce the requested output. --- t/t7519-status-fsmonitor.sh | 38 ++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 27f8a22692afb7..79443f84c1bd23 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1334,7 +1334,43 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP ! test_trace2_data status untracked/bulk-recovery 1 \ <.git/writable.trace && test_region index do_write_index .git/writable.trace && - test_fsmonitor_full_proof .git/index paired + test_fsmonitor_full_proof .git/index paired && + cp .git/index .git/writable.index && + find .git -maxdepth 1 -type f \ + \( -name "index.csts" -o -name "index.csh1.*" \ + -o -name "index.cswi.*" \) | + sort >.git/sidecars.before && + git hash-object --no-filters --stdin-paths \ + <.git/sidecars.before >.git/sidecar-hashes.before && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/follower.trace" \ + git status --porcelain=v2 >.git/follower && + test_cmp .git/expect .git/follower && + test_cmp_bin .git/writable.index .git/index && + test_fsmonitor_full_proof .git/index paired && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/follower.trace && + ! test_trace2_data fsmonitor untracked/legacy-discarded 1 \ + <.git/follower.trace && + ! test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9][0-9]*" <.git/follower.trace && + ! test_trace2_data read_directory opendir \ + "[1-9][0-9]*" <.git/follower.trace && + ! test_trace2_data index preload/bulk_dirs \ + "[1-9][0-9]*" <.git/follower.trace && + ! test_trace2_data index preload/sum_lstat \ + "[1-9][0-9]*" <.git/follower.trace && + ! test_region index do_write_index .git/follower.trace && + find .git -maxdepth 1 -type f \ + \( -name "index.csts" -o -name "index.csh1.*" \ + -o -name "index.cswi.*" \) | + sort >.git/sidecars.after && + test_cmp .git/sidecars.before .git/sidecars.after && + git hash-object --no-filters --stdin-paths \ + <.git/sidecars.after >.git/sidecar-hashes.after && + test_cmp .git/sidecar-hashes.before .git/sidecar-hashes.after ) ' From d079f66b8340c8c86a46dfc073622b36f0a6cf35 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 12:23:42 -0500 Subject: [PATCH 30/73] fsmonitor: avoid repository-wide bootstrap for bounded readers The physical-index recovery added in 2181cb2ea9 (fsmonitor: seed missing-history baselines from legacy tokens, 2026-07-24) can rebuild the complete attribute manifest before an index-only or path-limited reader starts its actual work. Repeating such commands against an incomplete proof repeats the same repository-wide scan. Let a narrowly classified reader request conservative bootstrap for its first index read. Expire the tracked-entry semantics and untracked cache instead of reconstructing a proof, so the requested paths still receive ordinary content checks. Restrict this to the canonical, non-split, non-sparse index with a reliable native provider. Cached diffs, bounded regular-file diffs, explicit check-attr requests, and stage-only ls-files requests opt in; other command forms keep the existing recovery path. Check the parsed index representation before skipping external-history restoration, which would otherwise hash a stale checkpoint before the fallback runs. Suppress diff's opportunistic index refresh only when this noncertifying fallback was actually used. Exercise main and linked worktrees in both hash formats. Compare each reader with a provider-disabled oracle, including same-stat content changes, changed attributes, and a failing required filter. Require no manifest reconstruction, checkpoint digest, or physical index write, and retain the original path for an on-disk split index and an unbounded worktree diff. --- builtin/check-attr.c | 12 ++ builtin/diff.c | 53 ++++++- builtin/ls-files.c | 29 ++++ clean-status-history.c | 4 +- fsmonitor.c | 79 ++++++++++ fsmonitor.h | 5 + t/t7534-status-scoped-readers.sh | 263 +++++++++++++++++++++++++++++++ 7 files changed, 442 insertions(+), 3 deletions(-) create mode 100755 t/t7534-status-scoped-readers.sh diff --git a/builtin/check-attr.c b/builtin/check-attr.c index 217d83ea7d5de0..f7083585fbfa95 100644 --- a/builtin/check-attr.c +++ b/builtin/check-attr.c @@ -3,6 +3,7 @@ #include "config.h" #include "attr.h" #include "environment.h" +#include "fsmonitor.h" #include "gettext.h" #include "object-name.h" #include "quote.h" @@ -115,6 +116,7 @@ int cmd_check_attr(int argc, struct attr_check *check; struct object_id initialized_oid; int cnt, i, doubledash, filei; + int scoped_bootstrap = 0; if (!is_bare_repository(the_repository)) setup_work_tree(the_repository); @@ -127,9 +129,19 @@ int cmd_check_attr(int argc, prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; + if (!stdin_paths && !source) { + for (i = 0; i < argc && strcmp(argv[i], "--"); i++) + ; + scoped_bootstrap = i < argc && + argc - i - 1 > 0 && argc - i - 1 <= 64; + } + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(the_repository->index); if (repo_read_index(the_repository) < 0) { die("invalid cache"); } + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(the_repository->index); if (cached_attrs) git_attr_set_direction(GIT_ATTR_INDEX); diff --git a/builtin/diff.c b/builtin/diff.c index d384b3b7383604..643a4925e12fbd 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -20,6 +20,7 @@ #include "environment.h" #include "gettext.h" #include "fsmonitor-ll.h" +#include "fsmonitor.h" #include "fsmonitor-settings.h" #include "tag.h" #include "trace2.h" @@ -32,6 +33,7 @@ #include "revision.h" #include "log-tree.h" #include "setup.h" +#include "symlinks.h" #include "thread-utils.h" #include "oid-array.h" #include "tree.h" @@ -51,6 +53,32 @@ static const char builtin_diff_usage[] = "\n" COMMON_DIFF_OPTIONS_HELP; +static int scoped_diff_bootstrap_used; + +static int diff_has_bounded_regular_pathspec(const struct pathspec *pathspec) +{ + int i; + + if (pathspec->nr <= 0 || pathspec->nr > 64 || + pathspec->has_wildcard || + (pathspec->magic & + (PATHSPEC_GLOB | PATHSPEC_ICASE | + PATHSPEC_EXCLUDE | PATHSPEC_ATTR))) + return 0; + for (i = 0; i < pathspec->nr; i++) { + const struct pathspec_item *item = &pathspec->items[i]; + struct stat st; + + if (!item->match || item->len <= 0 || + item->match[item->len - 1] == '/' || + !strcmp(item->match, ".") || + has_symlink_leading_path(item->match, item->len) || + lstat(item->match, &st) || !S_ISREG(st.st_mode)) + return 0; + } + return 1; +} + static const char *blob_path(struct object_array_entry *entry) { return entry->path ? entry->path : entry->name; @@ -152,6 +180,8 @@ static void builtin_diff_index(struct rev_info *revs, int argc, const char **argv) { unsigned int option = 0; + int scoped_bootstrap; + while (1 < argc) { const char *arg = argv[1]; if (!strcmp(arg, "--cached") || !strcmp(arg, "--staged")) @@ -170,8 +200,13 @@ static void builtin_diff_index(struct rev_info *revs, revs->max_count != -1 || revs->min_age != -1 || revs->max_age != -1) usage(builtin_diff_usage); - if (!(option & DIFF_INDEX_CACHED)) { + if (!(option & DIFF_INDEX_CACHED)) setup_work_tree(the_repository); + scoped_bootstrap = (option & DIFF_INDEX_CACHED) || + diff_has_bounded_regular_pathspec(&revs->diffopt.pathspec); + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(the_repository->index); + if (!(option & DIFF_INDEX_CACHED)) { if (repo_read_index_preload(the_repository, &revs->diffopt.pathspec, 0) < 0) { die_errno("repo_read_index_preload"); @@ -179,6 +214,9 @@ static void builtin_diff_index(struct rev_info *revs, } else if (repo_read_index(the_repository) < 0) { die_errno("repo_read_cache"); } + if (scoped_bootstrap) + scoped_diff_bootstrap_used |= + fsmonitor_end_scoped_bootstrap(the_repository->index); run_diff_index(revs, option); } @@ -459,6 +497,7 @@ static void refresh_index_quietly(void) static void builtin_diff_files(struct rev_info *revs, int argc, const char **argv) { unsigned int options = 0; + int scoped_bootstrap; while (1 < argc && argv[1][0] == '-') { if (!strcmp(argv[1], "--base")) @@ -489,10 +528,17 @@ static void builtin_diff_files(struct rev_info *revs, int argc, const char **arg diff_merges_set_dense_combined_if_unset(revs); setup_work_tree(the_repository); + scoped_bootstrap = + diff_has_bounded_regular_pathspec(&revs->diffopt.pathspec); + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(the_repository->index); if (repo_read_index_preload(the_repository, &revs->diffopt.pathspec, 0) < 0) { die_errno("repo_read_index_preload"); } + if (scoped_bootstrap) + scoped_diff_bootstrap_used |= + fsmonitor_end_scoped_bootstrap(the_repository->index); run_diff_files(revs, options); } @@ -639,6 +685,8 @@ int cmd_diff(int argc, int result; struct symdiff sdiff; + scoped_diff_bootstrap_used = 0; + /* * We could get N tree-ish in the rev.pending_objects list. * Also there could be M blobs there, and P pathspecs. --cached may @@ -875,7 +923,8 @@ int cmd_diff(int argc, ent.objects, ent.nr, first_non_parent); result = diff_result_code(&rev); - if (1 < rev.diffopt.skip_stat_unmatch) + if (1 < rev.diffopt.skip_stat_unmatch && + !scoped_diff_bootstrap_used) refresh_index_quietly(); release_revisions(&rev); object_array_clear(&ent); diff --git a/builtin/ls-files.c b/builtin/ls-files.c index b044520f9e3c39..5c2656c43f5fe1 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -12,6 +12,7 @@ #include "config.h" #include "convert.h" #include "environment.h" +#include "fsmonitor.h" #include "quote.h" #include "dir.h" #include "gettext.h" @@ -587,6 +588,28 @@ static int option_parse_exclude_standard(const struct option *opt, return 0; } +static int ls_files_has_bounded_stage_request(int argc, const char **argv) +{ + int i; + int stage = 0; + + for (i = 1; i < argc && strcmp(argv[i], "--"); i++) { + if (!strcmp(argv[i], "--stage") || !strcmp(argv[i], "-s")) + stage = 1; + else if (strcmp(argv[i], "-z")) + return 0; + } + if (!stage || i == argc || argc - i - 1 <= 0 || + argc - i - 1 > 64) + return 0; + for (i++; i < argc; i++) { + if (!*argv[i] || starts_with(argv[i], ":(") || + strpbrk(argv[i], "*?[")) + return 0; + } + return 1; +} + int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix, @@ -666,6 +689,7 @@ int cmd_ls_files(int argc, OPT_END() }; int ret = 0; + int scoped_bootstrap; show_usage_with_options_if_asked(argc, argv, ls_files_usage, builtin_ls_files_options); @@ -678,8 +702,13 @@ int cmd_ls_files(int argc, prefix_len = strlen(prefix); repo_config(repo, git_default_config, NULL); + scoped_bootstrap = ls_files_has_bounded_stage_request(argc, argv); + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(repo->index); if (repo_read_index(repo) < 0) die("index file corrupt"); + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(repo->index); argc = parse_options(argc, argv, prefix, builtin_ls_files_options, ls_files_usage, 0); diff --git a/clean-status-history.c b/clean-status-history.c index e02d8830ecb67d..073182c9ed8f27 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -1503,7 +1503,9 @@ int clean_status_restore_external_history(struct index_state *istate) int provider_reset_recovery = 0; int restored = 0; - if (!clean_status_external_history_enabled(istate) || !state || + /* Split and sparse representations are known only after parsing. */ + if (fsmonitor_scoped_bootstrap_is_active(istate) || + !clean_status_external_history_enabled(istate) || !state || state->disk_config_invalid || !state->config_enforced || !state->current_config_valid || !state->current_semantic_valid || diff --git a/fsmonitor.c b/fsmonitor.c index 5dccb81b25f62b..b1c07929d98581 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -16,6 +16,7 @@ #include "hashmap.h" #include "hex-ll.h" #include "name-hash.h" +#include "replace-object.h" #include "repository.h" #include "run-command.h" #include "strbuf.h" @@ -30,6 +31,75 @@ struct trace_key trace_fsmonitor = TRACE_KEY_INIT(FSMONITOR); +static struct index_state *scoped_bootstrap_index; +static int scoped_bootstrap_eligible; +static int scoped_bootstrap_used; + +static int fsmonitor_scoped_bootstrap_is_eligible(struct index_state *istate) +{ + struct repository *repo = istate->repo; + struct stat st; + char *physical, *selected, *canonical; + int eligible; + + if (istate != repo->index || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || + getenv(INDEX_ENVIRONMENT) || + getenv(GIT_WORK_TREE_ENVIRONMENT) || + getenv(GIT_COMMON_DIR_ENVIRONMENT) || + getenv(DB_ENVIRONMENT) || + getenv(ALTERNATE_DB_ENVIRONMENT) || + !fstat_is_reliable() || + fsm_settings__get_mode(repo) != FSMONITOR_MODE_IPC || + repo_config_get_split_index(repo) > 0 || + repo_config_values(repo)->apply_sparse_checkout || + repo_has_replace_refs_uncached(repo)) + return 0; + + physical = xstrfmt("%s/index", repo_get_git_dir(repo)); + selected = real_pathdup(repo_get_index_file(repo), 0); + canonical = real_pathdup(physical, 0); + eligible = selected && canonical && + !fspathcmp(selected, canonical) && + !lstat(physical, &st) && S_ISREG(st.st_mode) && + st.st_nlink == 1; + free(canonical); + free(selected); + free(physical); + return eligible; +} + +void fsmonitor_begin_scoped_bootstrap(struct index_state *istate) +{ + if (scoped_bootstrap_index) + BUG("nested scoped fsmonitor bootstrap"); + scoped_bootstrap_index = istate; + scoped_bootstrap_eligible = + fsmonitor_scoped_bootstrap_is_eligible(istate); + scoped_bootstrap_used = 0; +} + +int fsmonitor_scoped_bootstrap_is_active(const struct index_state *istate) +{ + return scoped_bootstrap_index == istate && + scoped_bootstrap_eligible && !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + !repo_config_values(istate->repo)->apply_sparse_checkout; +} + +int fsmonitor_end_scoped_bootstrap(struct index_state *istate) +{ + int used; + + if (scoped_bootstrap_index != istate) + BUG("scoped fsmonitor bootstrap index changed"); + used = scoped_bootstrap_used; + scoped_bootstrap_index = NULL; + scoped_bootstrap_eligible = 0; + scoped_bootstrap_used = 0; + return used; +} + static void assert_index_minimum(struct index_state *istate, size_t pos) { if (pos > istate->cache_nr) @@ -1268,6 +1338,15 @@ static void invalidate_fsmonitor_for_bootstrap( } } + if (fsmonitor_scoped_bootstrap_is_active(istate)) { + fsmonitor_invalidate_semantics(istate); + untracked_cache_invalidate_all(istate); + scoped_bootstrap_used = 1; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/scoped-reader-stat-fallback", 1); + return; + } + if (physical_history_unavailable) { int authenticated_manifest = clean_status_has_authenticated_worktree_manifest(istate); diff --git a/fsmonitor.h b/fsmonitor.h index ef178d61a225d9..be8136767a9bbe 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -56,6 +56,11 @@ static inline int fsmonitor_stat_can_be_valid(const struct stat *st) void fsmonitor_invalidate_semantics(struct index_state *istate); +/* Bound conservative bootstrap to one index read; never issue a proof. */ +void fsmonitor_begin_scoped_bootstrap(struct index_state *istate); +int fsmonitor_scoped_bootstrap_is_active(const struct index_state *istate); +int fsmonitor_end_scoped_bootstrap(struct index_state *istate); + /* * Check if refresh_fsmonitor has been called at least once. * refresh_fsmonitor is idempotent. Returns true if fsmonitor is diff --git a/t/t7534-status-scoped-readers.sh b/t/t7534-status-scoped-readers.sh new file mode 100755 index 00000000000000..4d2081b4def702 --- /dev/null +++ b/t/t7534-status-scoped-readers.sh @@ -0,0 +1,263 @@ +#!/bin/sh + +test_description='bounded readers do not certify partial fsmonitor history' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +test_scoped_partial_proof () { + perl - "$1" <<-\EOF + open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; + binmode $input; + local $/; + my $index = <$input>; + my $offset = index($index, "FSCF"); + die "missing FSCF extension\n" if $offset < 0; + my $flags = unpack("N", substr($index, $offset + 16, 4)); + die "unexpected FSCF flags $flags\n" if $flags != 9; + EOF +} + +test_scoped_remove_fscf () { + perl - "$1" "$2" <<-\EOF + use Digest::SHA qw(sha1 sha256); + open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; + binmode $input; + binmode STDOUT; + local $/; + my $index = <$input>; + my $rawsz = $ARGV[1] eq "sha256" ? 32 : 20; + my $offset = index($index, "FSCF"); + die "missing FSCF extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + substr($index, $offset, 8 + $size, ""); + my $payload = substr($index, 0, -$rawsz); + print $payload, $rawsz == 32 ? sha256($payload) : sha1($payload); + EOF +} + +assert_scoped_reader () { + scoped_label=$1 && + scoped_locks=$2 && + shift 2 && + cp "$gitdir/index" "$gitdir/$scoped_label.index" && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + -C "$worktree" "$@" \ + >"$gitdir/$scoped_label.expect" && + GIT_OPTIONAL_LOCKS=$scoped_locks \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$scoped_label.trace" \ + git -C "$worktree" "$@" \ + >"$gitdir/$scoped_label.actual" && + test_cmp "$gitdir/$scoped_label.expect" \ + "$gitdir/$scoped_label.actual" && + test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <"$gitdir/$scoped_label.trace" && + test_grep ! "\"label\":\"history_logical_digest\"" \ + "$gitdir/$scoped_label.trace" && + ! test_trace2_data fsmonitor history/external-restored 1 \ + <"$gitdir/$scoped_label.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/$scoped_label.trace" && + test_region ! index do_write_index \ + "$gitdir/$scoped_label.trace" && + test_cmp_bin "$gitdir/$scoped_label.index" "$gitdir/index" && + test_cmp_bin "$gitdir/checkpoint.pristine" "$scoped_checkpoint" +} + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'bounded physical-index readers reject incomplete history without a manifest' ' + test_when_finished "rm -rf scoped-readers scoped-readers-linked" && + test_create_repo scoped-readers && + ( + cd scoped-readers && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines "tracked text" >.gitattributes && + test_write_lines aaaa >tracked && + test_write_lines side >sibling && + git add .gitattributes tracked sibling && + git commit -m base && + git worktree add --detach ../scoped-readers-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../scoped-readers-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + test-tool chmtime -120 "$worktree/.gitattributes" \ + "$worktree/tracked" "$worktree/sibling" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/checkpoint.trace" \ + git -C "$worktree" status --short \ + >"$gitdir/checkpoint.status" && + test_must_be_empty "$gitdir/checkpoint.status" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/checkpoint.trace" && + find "$gitdir" -maxdepth 1 -type f \ + -name "index.csh1.*" >"$gitdir/checkpoints" && + test_line_count = 1 "$gitdir/checkpoints" && + scoped_checkpoint=$(cat "$gitdir/checkpoints") && + cp "$scoped_checkpoint" "$gitdir/checkpoint.pristine" && + cp "$gitdir/index" "$gitdir/private.index" && + test_write_lines staged >"$worktree/staged" && + GIT_INDEX_FILE="$gitdir/private.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" add --sparse -- staged && + test_scoped_partial_proof "$gitdir/private.index" && + cp "$gitdir/private.index" "$gitdir/index" && + assert_scoped_reader cached-readonly 0 \ + diff --no-ext-diff --no-textconv \ + --cached HEAD --name-only -z && + assert_scoped_reader cached-default 1 \ + diff --no-ext-diff --no-textconv \ + --cached HEAD --name-only -z && + assert_scoped_reader worktree-readonly 0 \ + diff --no-ext-diff --no-textconv -- tracked && + assert_scoped_reader worktree-default 1 \ + diff --no-ext-diff --no-textconv -- tracked && + test_must_be_empty "$gitdir/worktree-default.actual" && + if test "$worktree" != "$PWD" + then + cp "$gitdir/index" "$gitdir/partial.saved" && + test_scoped_remove_fscf "$gitdir/index" \ + "$(test_oid algo)" >"$gitdir/stale.index" && + mv "$gitdir/stale.index" "$gitdir/index" && + test_grep ! FSCF "$gitdir/index" && + assert_scoped_reader stale-linked-checkpoint 0 \ + diff --no-ext-diff --no-textconv -- tracked && + cp "$gitdir/partial.saved" "$gitdir/index" || + return 1 + fi && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/root-path.trace" \ + git -C "$worktree" diff \ + --no-ext-diff --no-textconv -- . \ + >"$gitdir/root-path.actual" && + ! test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <"$gitdir/root-path.trace" && + test_cmp_bin "$gitdir/private.index" "$gitdir/index" && + assert_scoped_reader attributes 0 \ + check-attr -a -- tracked sibling && + assert_scoped_reader cached-attributes 0 \ + check-attr --cached -a -- tracked && + assert_scoped_reader index-stage-readonly 0 \ + ls-files --stage -- tracked staged && + assert_scoped_reader index-stage-default 1 \ + ls-files --stage -- tracked staged && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/fsmonitor-mode.trace" \ + git -C "$worktree" ls-files -f -- tracked \ + >"$gitdir/fsmonitor-mode" && + ! test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <"$gitdir/fsmonitor-mode.trace" && + test_cmp_bin "$gitdir/private.index" "$gitdir/index" && + git -C "$worktree" config core.trustctime false && + git -C "$worktree" config core.checkStat minimal && + mtime=$(test-tool chmtime --get "$worktree/tracked") && + test_write_lines bbbb >"$worktree/tracked" && + test-tool chmtime =$mtime "$worktree/tracked" && + assert_scoped_reader same-stat 0 \ + diff --no-ext-diff --no-textconv -- tracked && + test_grep "^diff --git a/tracked b/tracked$" \ + "$gitdir/same-stat.actual" && + test_write_lines "tracked custom=changed" \ + >"$worktree/.gitattributes" && + assert_scoped_reader changed-attributes 0 \ + check-attr -a -- tracked && + test_grep "tracked: custom: changed" \ + "$gitdir/changed-attributes.actual" && + test_write_lines "tracked filter=required" \ + >"$worktree/.gitattributes" && + git -C "$worktree" config filter.required.clean false && + git -C "$worktree" config filter.required.required true && + cp "$gitdir/index" "$gitdir/filter.before" && + test_must_fail env GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/required-filter.trace" \ + git -C "$worktree" diff \ + --no-ext-diff --no-textconv -- tracked \ + >"$gitdir/required-filter.actual" \ + 2>"$gitdir/required-filter.error" && + test_must_fail env GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + -C "$worktree" diff \ + --no-ext-diff --no-textconv -- tracked \ + >"$gitdir/required-filter.expect" \ + 2>"$gitdir/required-filter.oracle-error" && + test_grep "clean filter .required. failed" \ + "$gitdir/required-filter.error" && + test_grep "clean filter .required. failed" \ + "$gitdir/required-filter.oracle-error" && + test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <"$gitdir/required-filter.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/required-filter.trace" && + test_cmp_bin "$gitdir/filter.before" "$gitdir/index" && + git -C "$worktree" config --unset filter.required.clean && + git -C "$worktree" config --unset filter.required.required && + git -C "$worktree" config --unset core.trustctime && + git -C "$worktree" config --unset core.checkStat && + cp "$gitdir/index" "$gitdir/unsplit.before" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --split-index && + test_must_fail git -C "$worktree" \ + config --get core.splitIndex && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" rev-parse --shared-index-path \ + >"$gitdir/split.shared" && + test_file_not_empty "$gitdir/split.shared" && + cp "$gitdir/index" "$gitdir/split.before" && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + -C "$worktree" diff \ + --no-ext-diff --no-textconv -- tracked \ + >"$gitdir/split.expect" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/split.trace" \ + git -C "$worktree" diff \ + --no-ext-diff --no-textconv -- tracked \ + >"$gitdir/split.actual" && + test_cmp "$gitdir/split.expect" "$gitdir/split.actual" && + ! test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <"$gitdir/split.trace" && + test_cmp_bin "$gitdir/split.before" "$gitdir/index" && + cp "$gitdir/unsplit.before" "$gitdir/index" || + return 1 + done + ) +' + +test_done From 04abe794a3c7b2ad85f549d734dbe863d2e67c5f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 12:40:31 -0500 Subject: [PATCH 31/73] dir: persist unresolved fsmonitor events as invalidation In d067582291 (status: reuse closed proofs for scoped queries, 2026-08-11), a changed file can leave its cached directory valid while queuing a targeted refresh in memory. The UNTR format records the valid bit and cached names, but not that queue. If another command writes the index before consuming the event, a later status can trust the advanced provider token and omit a newly created untracked file. Mark each still-dirty directory invalid before serializing it. The existing invalid-directory path discards its stale names and omits the valid bitmap bit. Its validated descendants remain available, but a new reader must inspect the affected directory instead of treating the lost queue as completed work. No index-format change is needed. Add a regression in which update-index receives the event and rewrites the paired index, then a fresh read-only status receives no new events. Require the same untracked output as an independent provider-disabled reader, and verify that the reader does not write the index. The old code reports a clean tree in both object formats. --- dir.c | 7 +++++ t/t7519-status-fsmonitor.sh | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/dir.c b/dir.c index 0c474dfad0c46b..5c9a4a4f6b96d5 100644 --- a/dir.c +++ b/dir.c @@ -5199,6 +5199,13 @@ static void write_one_dir(struct untracked_cache_dir *untracked, uint8_t intlen; int i = wd->index++; + /* Pending provider paths are process-local and cannot survive index I/O. */ + if (untracked->fsmonitor_dirty) { + untracked->valid = 0; + untracked->valid_recursive = 0; + untracked->fsmonitor_dirty = 0; + } + /* * untracked_nr should be reset whenever valid is clear, but * for safety.. diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 79443f84c1bd23..eb1711f7149e94 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -5297,4 +5297,63 @@ test_expect_success PTHREADS,UNTRACKED_CACHE,SHA1 'load cache-tree and untracked ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'untracked provider events do not disappear across an index rewrite' ' + test_when_finished "rm -rf untracked-provider-index-rewrite" && + test_create_repo untracked-provider-index-rewrite && + ( + cd untracked-provider-index-rewrite && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines tracked >cached/tracked && + test_write_lines outside >outside && + git add cached/tracked outside && + git commit -qm base && + test-tool chmtime -120 cached/tracked outside && + git update-index --refresh && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + + test_write_lines visible >cached/new-visible && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/new-visible \ + GIT_TRACE2_EVENT="$PWD/.git/writer.trace" \ + git update-index --force-write-index && + test_trace2_data fsmonitor apply_count 1 <.git/writer.trace && + test_region index do_write_index .git/writer.trace && + test_fsmonitor_full_proof .git/index paired && + cp .git/index .git/index.snapshot && + + git --no-optional-locks \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 >.git/expect && + test_grep "^? cached/new-visible$" .git/expect && + test_cmp_bin .git/index.snapshot .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/reader.trace" \ + git --no-optional-locks status --porcelain=v2 \ + >.git/actual && + test_cmp .git/expect .git/actual && + test_cmp_bin .git/index.snapshot .git/index && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/reader.trace && + test_trace2_data fsmonitor apply_count 0 \ + <.git/reader.trace && + ! test_trace2_data read_directory opendir 0 \ + <.git/reader.trace && + ! test_region index do_write_index .git/reader.trace + ) +' + test_done From 4922f4b11ecb04d37efd99b0f886db7600e00b2f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 12:58:02 -0500 Subject: [PATCH 32/73] read-cache: allow authenticated same-path replacements The provider-reset recovery in 49d03b6235 (fsmonitor: retain untracked candidates across expired provider tokens, 2026-08-14) can replace a semantically unchanged index entry without invalidating its untracked directory. An ordinary indexed patch needs the same operation, but must not borrow the reset-revalidation flag to obtain it. Add an explicit add_index_entry() option for this case. Require the canonical strong-stat index, native fsmonitor, a complete semantic proof, equal authenticated tracked and untracked tokens, and a safe same-name regular-file replacement. Existing callers keep their old behavior. The replacement still clears CE_FSMONITOR_VALID. Keep the untracked cache's existing invalid and dirty state intact. In 04abe794a3 (dir: persist unresolved fsmonitor events as invalidation, 2026-08-16), serialization was made to expire unresolved dirty nodes, so preserving this state cannot turn pending work into a valid cache. --- read-cache-ll.h | 1 + read-cache.c | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/read-cache-ll.h b/read-cache-ll.h index f721e1caf4e540..242474ce9861f5 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -423,6 +423,7 @@ static inline int index_pos_to_insert_pos(uintmax_t pos) #define ADD_CACHE_NEW_ONLY 16 /* Do not replace existing ones */ #define ADD_CACHE_KEEP_CACHE_TREE 32 /* Do not invalidate cache-tree */ #define ADD_CACHE_RENORMALIZE 64 /* Pass along HASH_RENORMALIZE */ +#define ADD_CACHE_PRESERVE_CLEAN_HISTORY 128 /* Preserve safe replacements */ int add_index_entry(struct index_state *, struct cache_entry *ce, int option); void rename_index_entry_at(struct index_state *, int pos, const char *new_name); diff --git a/read-cache.c b/read-cache.c index 7cda0ff5a8f9db..7e85e45dc75a94 100644 --- a/read-cache.c +++ b/read-cache.c @@ -33,6 +33,7 @@ #include "path.h" #include "preload-index.h" #include "read-cache.h" +#include "replace-object.h" #include "repository.h" #include "resolve-undo.h" #include "revision.h" @@ -143,12 +144,42 @@ static void set_index_entry(struct index_state *istate, int nr, struct cache_ent add_name_hash(istate, ce); } -static void replace_index_entry(struct index_state *istate, int nr, struct cache_entry *ce) +static void replace_index_entry(struct index_state *istate, int nr, + struct cache_entry *ce, int options) { struct cache_entry *old = istate->cache[nr]; + int preserve_paired_history = + (options & ADD_CACHE_PRESERVE_CLEAN_HISTORY) && + fstat_is_reliable() && istate == istate->repo->index && + !alternate_index_output && !getenv(INDEX_ENVIRONMENT) && + !getenv(GIT_WORK_TREE_ENVIRONMENT) && + !getenv(GIT_COMMON_DIR_ENVIRONMENT) && + !getenv(DB_ENVIRONMENT) && + !getenv(ALTERNATE_DB_ENVIRONMENT) && + !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + !repo_config_values(istate->repo)->apply_sparse_checkout && + istate->repo->config_values_private_.trust_ctime && + istate->repo->config_values_private_.check_stat && + fsm_settings__get_mode(istate->repo) == FSMONITOR_MODE_IPC && + !repo_has_replace_refs_uncached(istate->repo) && + istate->fsmonitor_token_valid && + istate->fsmonitor_untracked_valid && + istate->fsmonitor_untracked_extension_seen && + !istate->fsmonitor_untracked_extension_invalid && + istate->fsmonitor_last_update && + istate->fsmonitor_untracked_token && + !strcmp(istate->fsmonitor_last_update, + istate->fsmonitor_untracked_token) && + clean_status_has_persistent_fsmonitor_semantic_history(istate) && + clean_status_revalidated_token_matches(istate); + /* Keep invalid nodes invalid; write_one_dir() expires pending events. */ int preserve_untracked = istate->untracked && - istate->untracked->fsmonitor_revalidation && - istate->untracked->root && istate->untracked->root->valid && + istate->untracked->root && + ((istate->untracked->fsmonitor_revalidation && + istate->untracked->root->valid) || + (preserve_paired_history && + istate->untracked->use_fsmonitor)) && S_ISREG(old->ce_mode) && S_ISREG(ce->ce_mode) && clean_status_index_entry_is_semantically_safe(istate, old, ce); @@ -162,6 +193,9 @@ static void replace_index_entry(struct index_state *istate, int nr, struct cache ce->ce_flags &= ~CE_FSMONITOR_VALID; else mark_fsmonitor_invalid(istate, ce); + if (preserve_paired_history && preserve_untracked) + trace2_data_intmax("fsmonitor", istate->repo, + "apply/untracked-replacement-preserved", 1); istate->cache_changed |= CE_ENTRY_CHANGED; } @@ -236,7 +270,7 @@ void refresh_index_entry_stat(struct index_state *istate, int nr, struct stat *st) { replace_index_entry(istate, nr, make_refreshed_cache_entry( - istate, istate->cache[nr], st, 1)); + istate, istate->cache[nr], st, 1), 0); } static unsigned int st_mode_from_ce(const struct cache_entry *ce) @@ -1351,7 +1385,7 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e /* existing match? Just replace it. */ if (pos >= 0) { if (!new_only) - replace_index_entry(istate, pos, ce); + replace_index_entry(istate, pos, ce, option); return 0; } pos = -pos-1; @@ -1708,7 +1742,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, clean_status_fsmonitor_semantic_baseline_pending( istate)); - replace_index_entry(istate, i, new_entry); + replace_index_entry(istate, i, new_entry, 0); if (fsmonitor_valid) mark_fsmonitor_valid(istate, istate->cache[i]); From 428d8292400731c7f9d6367fee88c84378afe505 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 12:58:13 -0500 Subject: [PATCH 33/73] stash: retain paired history for scoped regular-file changes A path-specific stash removes and re-adds an existing index entry while applying its reverse patch. That unnecessarily invalidates the rooted untracked cache even when the path, type, attributes, and provider proof are unchanged. The next indexed stash apply then has to reconstruct the missing history. Enable authenticated history before the first canonical indexed-apply read. Preflight the complete patch batch and use the explicit safe replacement option only for ordinary same-path regular-file edits. An unsafe member retains the existing two-phase removal behavior for the whole batch. Attribute and ignore changes, active filters, renames, conflicts, alternate indexes, and weak stat configurations remain conservative. Give scoped stash inspection its own history guard, without changing the deliberate invalidation for dirty whole-worktree stashes. Reuse the existing digest suspension while writing the synthetic stash tree, and permit the final physical indexed restore to use unpack-trees' guarded semantic transfer. The synthetic index never receives that permission. Compare the complete stash cycle with an independent disabled-provider repository, including both stash trees, staged entries, and worktree bytes. Cover an unrelated staged sibling, mixed attribute/ignore patch orders, required-filter failure, alternate indexes, and pending untracked changes. Retained proofs must stay paired without losing untracked output or requiring a complete attribute-manifest scan. --- apply.c | 86 +++- builtin/apply.c | 25 ++ builtin/stash.c | 52 ++- t/t7533-status-scoped-stash.sh | 749 +++++++++++++++++++++++++++++++++ 4 files changed, 901 insertions(+), 11 deletions(-) create mode 100755 t/t7533-status-scoped-stash.sh diff --git a/apply.c b/apply.c index 1f5dda3b6f3fcc..5aad604fc9c6d1 100644 --- a/apply.c +++ b/apply.c @@ -13,12 +13,14 @@ #include "git-compat-util.h" #include "abspath.h" #include "base85.h" +#include "clean-status.h" #include "config.h" #include "odb.h" #include "delta.h" #include "diff.h" #include "dir.h" #include "environment.h" +#include "fsmonitor-settings.h" #include "gettext.h" #include "hex.h" #include "xdiff-interface.h" @@ -31,6 +33,7 @@ #include "path.h" #include "quote.h" #include "read-cache.h" +#include "replace-object.h" #include "repository.h" #include "rerere.h" #include "apply.h" @@ -4440,9 +4443,66 @@ static void patch_stats(struct apply_state *state, struct patch *patch) } } +static int patch_preserves_clean_history(struct apply_state *state, + struct patch *patch) +{ + struct index_state *istate = state->repo->index; + const struct cache_entry *old; + int pos; + + if (!state->update_index || state->ita_only || state->threeway || + state->apply_with_reject || state->fake_ancestor || + state->index_file || !fstat_is_reliable() || + getenv(INDEX_ENVIRONMENT) || + getenv(GIT_WORK_TREE_ENVIRONMENT) || + getenv(GIT_COMMON_DIR_ENVIRONMENT) || + getenv(DB_ENVIRONMENT) || getenv(ALTERNATE_DB_ENVIRONMENT) || + istate != istate->repo->index || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || + repo_config_values(istate->repo)->apply_sparse_checkout || + !istate->repo->config_values_private_.trust_ctime || + !istate->repo->config_values_private_.check_stat || + fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC || + repo_has_replace_refs_uncached(istate->repo) || + patch->is_new > 0 || patch->is_delete > 0 || patch->is_copy || + patch->is_rename || patch->conflicted_threeway || + !patch->old_name || !patch->new_name || + strcmp(patch->old_name, patch->new_name) || + !S_ISREG(patch->old_mode) || !S_ISREG(patch->new_mode) || + create_ce_mode(patch->old_mode) != + create_ce_mode(patch->new_mode) || + !clean_status_external_history_enabled(istate) || + !clean_status_has_persistent_fsmonitor_semantic_history(istate) || + !clean_status_revalidated_token_matches(istate) || + !istate->fsmonitor_token_valid || + !istate->fsmonitor_untracked_valid || + !istate->fsmonitor_untracked_extension_seen || + istate->fsmonitor_untracked_extension_invalid || + !istate->fsmonitor_last_update || + !istate->fsmonitor_untracked_token || + strcmp(istate->fsmonitor_last_update, + istate->fsmonitor_untracked_token) || + !istate->untracked || !istate->untracked->use_fsmonitor || + !istate->untracked->root) + return 0; + + pos = index_name_pos(istate, patch->old_name, + strlen(patch->old_name)); + if (pos < 0) + return 0; + old = istate->cache[pos]; + return S_ISREG(old->ce_mode) && + old->ce_mode == create_ce_mode(patch->new_mode) && + clean_status_index_entry_is_semantically_safe( + istate, old, old); +} + static int remove_file(struct apply_state *state, struct patch *patch, int rmdir_empty) { - if (state->update_index && !state->ita_only) { + if (state->update_index && !state->ita_only && + !patch_preserves_clean_history(state, patch)) { + if (clean_status_external_history_enabled(state->repo->index)) + clean_status_invalidate_current_proof(state->repo->index); if (remove_file_from_index(state->repo->index, patch->old_name) < 0) return error(_("unable to remove %s from index"), patch->old_name); } @@ -4455,6 +4515,7 @@ static int remove_file(struct apply_state *state, struct patch *patch, int rmdir } static int add_index_file(struct apply_state *state, + struct patch *patch, const char *path, unsigned mode, void *buf, @@ -4463,6 +4524,7 @@ static int add_index_file(struct apply_state *state, struct stat st; struct cache_entry *ce; int namelen = strlen(path); + int options = ADD_CACHE_OK_TO_ADD; ce = make_empty_cache_entry(state->repo->index, namelen); memcpy(ce->name, path, namelen); @@ -4497,7 +4559,13 @@ static int add_index_file(struct apply_state *state, "for newly created file %s"), path); } } - if (add_index_entry(state->repo->index, ce, ADD_CACHE_OK_TO_ADD) < 0) { + if (patch_preserves_clean_history(state, patch)) { + options |= ADD_CACHE_OK_TO_REPLACE | + ADD_CACHE_PRESERVE_CLEAN_HISTORY; + } else if (clean_status_external_history_enabled(state->repo->index)) { + clean_status_invalidate_current_proof(state->repo->index); + } + if (add_index_entry(state->repo->index, ce, options) < 0) { discard_cache_entry(ce); return error(_("unable to add cache entry for %s"), path); } @@ -4697,7 +4765,7 @@ static int create_file(struct apply_state *state, struct patch *patch) if (patch->conflicted_threeway) return add_conflicted_stages_file(state, patch); else if (state->check_index || (state->ita_only && patch->is_new > 0)) - return add_index_file(state, path, mode, buf, size); + return add_index_file(state, patch, path, mode, buf, size); return 0; } @@ -4828,6 +4896,18 @@ static int write_out_results(struct apply_state *state, struct patch *list) struct patch *l; struct string_list cpath = STRING_LIST_INIT_DUP; + if (state->update_index && + clean_status_external_history_enabled(state->repo->index)) { + for (l = list; l; l = l->next) { + if (l->rejected || + !patch_preserves_clean_history(state, l)) { + clean_status_invalidate_current_proof( + state->repo->index); + break; + } + } + } + for (phase = 0; phase < 2; phase++) { l = list; while (l) { diff --git a/builtin/apply.c b/builtin/apply.c index d642a402516f30..4cc2ac83369b35 100644 --- a/builtin/apply.c +++ b/builtin/apply.c @@ -1,7 +1,12 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "clean-status-config.h" +#include "clean-status.h" +#include "environment.h" +#include "fsmonitor-settings.h" #include "gettext.h" #include "hash.h" +#include "replace-object.h" #include "apply.h" static const char * const apply_usage[] = { @@ -17,6 +22,7 @@ int cmd_apply(int argc, int force_apply = 0; int options = 0; int ret; + struct clean_status_config_digest clean_digest; struct apply_state state; if (init_apply_state(&state, the_repository, prefix)) @@ -43,6 +49,25 @@ int cmd_apply(int argc, if (check_apply_state(&state, force_apply)) exit(128); + if (state.apply && state.check_index && !state.threeway && + !state.apply_with_reject && !state.ita_only && + !state.fake_ancestor && !state.index_file && + !getenv(INDEX_ENVIRONMENT) && + !getenv(GIT_WORK_TREE_ENVIRONMENT) && + !getenv(GIT_COMMON_DIR_ENVIRONMENT) && + !getenv(DB_ENVIRONMENT) && + !getenv(ALTERNATE_DB_ENVIRONMENT) && fstat_is_reliable() && + !repo_config_values(the_repository)->apply_sparse_checkout && + the_repository->config_values_private_.trust_ctime && + the_repository->config_values_private_.check_stat && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC && + !repo_has_replace_refs_uncached(the_repository) && + !clean_status_config_read_repository(the_repository, + &clean_digest)) { + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, &clean_digest); + } + ret = apply_all_patches(&state, argc, argv, options); clear_apply_state(&state); diff --git a/builtin/stash.c b/builtin/stash.c index f5bb8b37ac18c3..3eb63d18faa0fd 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -6,6 +6,7 @@ #include "clean-status-config.h" #include "config.h" #include "environment.h" +#include "fsmonitor-settings.h" #include "gettext.h" #include "hash.h" #include "hex.h" @@ -22,6 +23,7 @@ #include "entry.h" #include "preload-index.h" #include "read-cache.h" +#include "replace-object.h" #include "repository.h" #include "rerere.h" #include "revision.h" @@ -334,7 +336,8 @@ static int clear_stash(int argc, const char **argv, const char *prefix, return do_clear_stash(); } -static int reset_tree(struct object_id *i_tree, int update, int reset) +static int reset_tree(struct object_id *i_tree, int update, int reset, + int preserve_semantic_history) { int nr_trees = 1; struct unpack_trees_options opts; @@ -359,6 +362,22 @@ static int reset_tree(struct object_id *i_tree, int update, int reset) opts.head_idx = 1; opts.src_index = the_repository->index; opts.dst_index = the_repository->index; + opts.preserve_semantic_history = preserve_semantic_history && + !update && !reset && fstat_is_reliable() && + !getenv(INDEX_ENVIRONMENT) && + !getenv(GIT_WORK_TREE_ENVIRONMENT) && + !getenv(GIT_COMMON_DIR_ENVIRONMENT) && + !getenv(DB_ENVIRONMENT) && + !getenv(ALTERNATE_DB_ENVIRONMENT) && + !repo_config_values(the_repository)->apply_sparse_checkout && + the_repository->config_values_private_.trust_ctime && + the_repository->config_values_private_.check_stat && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC && + !repo_has_replace_refs_uncached(the_repository) && + the_repository->index->fsmonitor_untracked_valid && + clean_status_has_persistent_fsmonitor_semantic_history( + the_repository->index) && + clean_status_revalidated_token_matches(the_repository->index); opts.merge = 1; opts.reset = reset ? UNPACK_RESET_PROTECT_UNTRACKED : 0; opts.update = update; @@ -754,7 +773,7 @@ static int do_apply_stash(const char *prefix, struct stash_info *info, } if (has_index) { - if (reset_tree(&index_tree, 0, 0)) + if (reset_tree(&index_tree, 0, 0, 1)) ret = -1; } else { unstage_changes_unless_new(&c_tree); @@ -1473,7 +1492,7 @@ static int stash_working_tree(struct stash_info *info, const struct pathspec *ps copy_pathspec(&rev.prune_data, ps); set_alternate_index_output(stash_index_path.buf); - if (reset_tree(&info->i_tree, 0, 0)) { + if (reset_tree(&info->i_tree, 0, 0, 0)) { ret = -1; goto done; } @@ -1629,7 +1648,9 @@ static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_b } } else { if (stash_working_tree(info, ps, - !ps->nr && !include_untracked)) { + !include_untracked && + clean_status_external_history_enabled( + the_repository->index))) { if (!quiet) fprintf_ln(stderr, _("Cannot save the current " "worktree state")); @@ -1703,6 +1724,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q { int ret = 0; int preserve_clean_history = !ps->nr && !include_untracked; + int preserve_scoped_history = 0; struct lock_file index_lock = LOCK_INIT; struct stash_info info = STASH_INFO_INIT; struct strbuf patch = STRBUF_INIT; @@ -1731,12 +1753,26 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q goto done; } + preserve_scoped_history = ps->nr && !include_untracked && + !patch_mode && !only_staged && keep_index != 1 && + !getenv(INDEX_ENVIRONMENT) && + !getenv(GIT_WORK_TREE_ENVIRONMENT) && + !getenv(GIT_COMMON_DIR_ENVIRONMENT) && + !getenv(DB_ENVIRONMENT) && + !getenv(ALTERNATE_DB_ENVIRONMENT) && fstat_is_reliable() && + !repo_config_values(the_repository)->apply_sparse_checkout && + the_repository->config_values_private_.trust_ctime && + the_repository->config_values_private_.check_stat && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC && + !repo_has_replace_refs_uncached(the_repository); + /* - * Keep whole-worktree history bound while inspecting the worktree. - * If changes are found, invalidate it before stash machinery - * mutates the index or worktree. + * Keep authenticated history bound while inspecting the worktree. + * Whole-worktree changes still invalidate their proof below. A scoped + * regular-file replacement keeps it only when each writer proves that + * its provider, semantic inputs, and untracked cache remain paired. */ - if (preserve_clean_history) { + if (preserve_clean_history || preserve_scoped_history) { clean_status_enable_external_history(the_repository); clean_status_set_config_digest(the_repository, &stash_clean_digest); diff --git a/t/t7533-status-scoped-stash.sh b/t/t7533-status-scoped-stash.sh new file mode 100755 index 00000000000000..81bdf080cabdf4 --- /dev/null +++ b/t/t7533-status-scoped-stash.sh @@ -0,0 +1,749 @@ +#!/bin/sh + +test_description='authenticated fsmonitor history across scoped stash writers' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +scoped_stash_full_proof () { + perl - "$1" <<-\EOF + open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; + binmode $input; + local $/; + my $index = <$input>; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + die "invalid $name token\n" if $end < 0; + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "mismatched provider tokens\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"} && + $tokens{"FSMN"} eq $tokens{"FSCF"}; + EOF +} + +scoped_stash_prime () { + worktree=$1 && + gitdir=$(git -C "$worktree" rev-parse --absolute-git-dir) && + test-tool chmtime -120 "$worktree/tracked" "$worktree/sibling" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + scoped_stash_full_proof "$gitdir/index" +} + +scoped_stash_setup () { + test_create_repo "$1" && + ( + cd "$1" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + git config core.untrackedCache true && + git config core.fsmonitor true && + scoped_stash_prime "$PWD" + ) +} + +scoped_stash_control_git () { + git -C "$scoped_control" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default "$@" +} + +scoped_stash_control () { + scoped_source=$1 && + scoped_control=$2 && + scoped_output=$3 && + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + clone -q --no-hardlinks "$scoped_source" \ + "$scoped_control" && + test_write_lines scoped >"$scoped_control/tracked" && + scoped_stash_control_git add -- tracked && + scoped_stash_control_git stash push -q \ + -m independent-control -- tracked && + scoped_stash_control_git rev-parse \ + "stash@{0}^{tree}" "stash@{0}^2^{tree}" \ + >"$scoped_output/control.trees" && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 \ + >"$scoped_output/control.pushed" && + scoped_stash_control_git stash apply --index -q "stash@{0}" && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 \ + >"$scoped_output/control.applied" && + scoped_stash_control_git --no-optional-locks \ + ls-files --stage \ + >"$scoped_output/control.staged" +} + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'scoped stash push and indexed apply preserve paired worktree proofs' ' + test_when_finished "rm -rf scoped-stash scoped-stash-linked \ + scoped-stash-control-1 scoped-stash-control-2" && + test_create_repo scoped-stash && + ( + cd scoped-stash && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + git worktree add --detach ../scoped-stash-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + control_nr=0 && + for worktree in "$PWD" "$PWD/../scoped-stash-linked" + do + control_nr=$((control_nr + 1)) && + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + scoped_stash_prime "$worktree" && + scoped_stash_control "$worktree" \ + "$PWD/../scoped-stash-control-$control_nr" \ + "$gitdir" && + test_write_lines scoped >"$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git -C "$worktree" add -- tracked && + scoped_stash_full_proof "$gitdir/index" && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/push.trace" \ + git -C "$worktree" stash push -q \ + -m scoped-proof -- tracked && + scoped_stash_full_proof "$gitdir/index" && + test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <"$gitdir/push.trace" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/push.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/push.trace" && + + stash=$(git -C "$worktree" rev-parse stash@{0}) && + git -C "$worktree" rev-parse \ + "$stash^{tree}" "$stash^2^{tree}" \ + >"$gitdir/push.trees" && + test_cmp "$gitdir/control.trees" \ + "$gitdir/push.trees" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 \ + >"$gitdir/push.actual" && + test_cmp "$gitdir/control.pushed" \ + "$gitdir/push.actual" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/apply.trace" \ + git -C "$worktree" stash apply --index -q \ + "$stash" && + scoped_stash_full_proof "$gitdir/index" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/apply.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/apply.trace" && + + cp "$gitdir/index" "$gitdir/index.before" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/actual" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >"$gitdir/expect" && + test_cmp "$gitdir/expect" "$gitdir/actual" && + test_cmp "$gitdir/control.applied" \ + "$gitdir/actual" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + ls-files --stage \ + >"$gitdir/staged.actual" && + test_cmp "$gitdir/control.staged" \ + "$gitdir/staged.actual" && + test_cmp "$scoped_control/tracked" \ + "$worktree/tracked" && + test_cmp "$scoped_control/sibling" \ + "$worktree/sibling" && + test_grep "^1 M\\. .* tracked$" "$gitdir/actual" && + test_cmp_bin "$gitdir/index.before" "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/status.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/status.trace" || return 1 + done + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'scoped stash push preserves an unrelated staged sibling' ' + test_when_finished "rm -rf scoped-stash-staged \ + scoped-stash-staged-control" && + scoped_stash_setup scoped-stash-staged && + ( + cd scoped-stash-staged && + gitdir="$PWD/.git" && + scoped_control="$PWD/../scoped-stash-staged-control" && + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + clone -q --no-hardlinks "$PWD" \ + "$scoped_control" && + test_write_lines scoped >"$scoped_control/tracked" && + test_write_lines independently-staged \ + >"$scoped_control/sibling" && + scoped_stash_control_git add -- tracked sibling && + scoped_stash_control_git stash push -q \ + -m independent-staged-control -- tracked && + scoped_stash_control_git rev-parse \ + "stash@{0}^{tree}" "stash@{0}^2^{tree}" \ + >.git/control.trees && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 >.git/control.status && + scoped_stash_control_git --no-optional-locks \ + ls-files --stage >.git/control.staged && + test_grep "^1 M\\. .* sibling$" .git/control.status && + + test_write_lines scoped >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add -- tracked && + test_write_lines independently-staged >sibling && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=sibling \ + git add -- sibling && + scoped_stash_full_proof .git/index && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/staged-push.trace" \ + git stash push -q -m staged-proof -- tracked && + scoped_stash_full_proof .git/index && + test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/staged-push.trace && + ! test_trace2_data fsmonitor \ + untracked/proof-missing 1 <.git/staged-push.trace && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <.git/staged-push.trace && + git rev-parse "stash@{0}^{tree}" \ + "stash@{0}^2^{tree}" >.git/staged.trees && + test_cmp .git/control.trees .git/staged.trees && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/staged.status && + test_cmp .git/control.status .git/staged.status && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + ls-files --stage >.git/staged.entries && + test_cmp .git/control.staged .git/staged.entries && + test_cmp "$scoped_control/tracked" tracked && + test_cmp "$scoped_control/sibling" sibling + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'scoped stash invalidates a changed attribute-source proof' ' + test_when_finished "rm -rf scoped-stash-attributes" && + test_create_repo scoped-stash-attributes && + ( + cd scoped-stash-attributes && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + test_write_lines sibling >sibling && + test_write_lines "*.asset text" >.gitattributes && + git add tracked sibling .gitattributes && + git commit -qm base && + git config core.untrackedCache true && + git config core.fsmonitor true && + scoped_stash_prime "$PWD" && + test_write_lines "*.asset -text" >.gitattributes && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + git add .gitattributes && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git status --porcelain=v2 >.git/staged && + scoped_stash_full_proof .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$PWD/.git/attributes.trace" \ + git stash push -q -m attributes -- .gitattributes && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/attributes.trace && + ! scoped_stash_full_proof .git/index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git status --porcelain=v2 >.git/actual && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expect && + test_cmp .git/expect .git/actual + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'mixed apply batches reject attribute and ignore proofs in either order' ' + test_when_finished "rm -rf scoped-stash-mixed-*" && + for source in .gitattributes .gitignore + do + for order in regular-first source-first + do + repo="scoped-stash-mixed-${source#.}-$order" && + test_create_repo "$repo" && + ( + cd "$repo" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + test_write_lines sibling >sibling && + test_write_lines "*.asset text" \ + >.gitattributes && + test_write_lines ignored-before >.gitignore && + git add tracked sibling .gitattributes \ + .gitignore && + git commit -qm base && + git config core.untrackedCache true && + git config core.fsmonitor true && + test_write_lines mixed >tracked && + case "$source" in + .gitattributes) + test_write_lines "*.asset -text" \ + >"$source" + ;; + .gitignore) + test_write_lines ignored-after \ + >"$source" + ;; + esac && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + diff -- tracked \ + >.git/regular.patch && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + diff -- "$source" \ + >.git/source.patch && + if test "$order" = regular-first + then + cat .git/regular.patch \ + .git/source.patch \ + >.git/mixed.patch + else + cat .git/source.patch \ + .git/regular.patch \ + >.git/mixed.patch + fi && + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + checkout -- tracked "$source" && + scoped_stash_prime "$PWD" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/mixed.trace" \ + git apply --index .git/mixed.patch && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/mixed.trace && + ! scoped_stash_full_proof .git/index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 \ + >.git/actual && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 \ + >.git/expect && + test_cmp .git/expect .git/actual && + test_grep " tracked$" .git/actual && + test_grep " $source$" .git/actual + ) || return 1 + done + done +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'scoped stash never preserves an active required clean filter' ' + test_when_finished "rm -rf scoped-stash-filter" && + test_create_repo scoped-stash-filter && + ( + cd scoped-stash-filter && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + test_write_lines sibling >sibling && + test_write_lines "tracked text" >.gitattributes && + git add tracked sibling .gitattributes && + git commit -qm base && + git config core.untrackedCache true && + git config core.fsmonitor true && + scoped_stash_prime "$PWD" && + git config filter.scoped.clean cat && + git config filter.scoped.smudge cat && + git config filter.scoped.required true && + test_write_lines "tracked filter=scoped" >.gitattributes && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + git add -- .gitattributes && + test_write_lines filtered >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add tracked && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git status --porcelain=v2 >.git/staged && + ! scoped_stash_full_proof .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/filter.trace" \ + git stash push -q -m filtered -- tracked && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/filter.trace && + ! scoped_stash_full_proof .git/index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git status --porcelain=v2 >.git/actual && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expect && + test_cmp .git/expect .git/actual && + + git config filter.scoped.clean false && + test_write_lines rejected >tracked && + cp .git/index .git/required.before && + git rev-parse refs/stash >.git/stash.before && + test_must_fail env \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/required.trace" \ + git stash push -q -m rejected -- tracked \ + >.git/required.out 2>.git/required.err && + test_grep "clean filter .scoped. failed" \ + .git/required.err && + test_cmp_bin .git/required.before .git/index && + git rev-parse refs/stash >.git/stash.after && + test_cmp .git/stash.before .git/stash.after && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/required.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'whole-worktree stash retains its deliberate proof invalidation' ' + test_when_finished "rm -rf scoped-stash-whole" && + scoped_stash_setup scoped-stash-whole && + ( + cd scoped-stash-whole && + test_write_lines dirty >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/whole.trace" \ + git stash push -q -m whole && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/whole.trace && + ! scoped_stash_full_proof .git/index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git status --porcelain=v2 >.git/actual && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expect && + test_cmp .git/expect .git/actual + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'alternate indexed apply cannot transfer a primary worktree proof' ' + test_when_finished "rm -rf scoped-stash-alternate" && + scoped_stash_setup scoped-stash-alternate && + ( + cd scoped-stash-alternate && + test_write_lines alternate >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git diff >.git/alternate.patch && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout -- tracked && + scoped_stash_full_proof .git/index && + cp .git/index .git/index.before && + cp .git/index .git/alternate.index && + GIT_INDEX_FILE="$PWD/.git/alternate.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/alternate.trace" \ + git apply --cached .git/alternate.patch && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/alternate.trace && + test_cmp_bin .git/index.before .git/index && + scoped_stash_full_proof .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'nested scoped stash never resurrects dirty untracked siblings' ' + test_when_finished "rm -rf scoped-stash-nested \ + scoped-stash-nested-control" && + test_create_repo scoped-stash-nested && + ( + cd scoped-stash-nested && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + mkdir nested && + test_write_lines base >nested/tracked && + git add -- nested/tracked && + git commit -qm nested-base && + scoped_control="$PWD/../scoped-stash-nested-control" && + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + clone -q --no-hardlinks "$PWD" "$scoped_control" && + test_write_lines existing >nested/existing-untracked && + test_write_lines existing \ + >"$scoped_control/nested/existing-untracked" && + git config core.untrackedCache true && + git config core.fsmonitor true && + test-tool chmtime -120 nested/tracked && + scoped_stash_prime "$PWD" && + test_grep "^? nested/existing-untracked$" .git/prime && + GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ + test-tool dump-untracked-cache >.git/initial-cache && + test_grep "^/nested/ .* valid" .git/initial-cache && + test_grep "^existing-untracked$" .git/initial-cache && + + test_write_lines staged >nested/tracked && + test_write_lines staged >"$scoped_control/nested/tracked" && + scoped_stash_control_git add -- nested/tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=nested/tracked \ + git add -- nested/tracked && + scoped_stash_full_proof .git/index && + + test_write_lines new >nested/new-untracked && + test_write_lines new \ + >"$scoped_control/nested/new-untracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=nested/new-untracked \ + GIT_TRACE2_EVENT="$PWD/.git/nested-writer.trace" \ + git update-index --force-write-index && + test_trace2_data fsmonitor apply_count 1 \ + <.git/nested-writer.trace && + test_region index do_write_index .git/nested-writer.trace && + scoped_stash_full_proof .git/index && + GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ + test-tool dump-untracked-cache >.git/dirty-cache && + test_grep "^/nested/ .* recurse$" .git/dirty-cache && + ! test_grep "^/nested/ .* valid" .git/dirty-cache && + ! test_grep "^existing-untracked$" .git/dirty-cache && + cp .git/index .git/before-reader && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 >.git/control-before && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/nested-reader.trace" \ + git --no-optional-locks status --porcelain=v2 \ + >.git/candidate-before && + test_cmp .git/control-before .git/candidate-before && + test_grep "^? nested/existing-untracked$" \ + .git/candidate-before && + test_grep "^? nested/new-untracked$" \ + .git/candidate-before && + test_cmp_bin .git/before-reader .git/index && + ! test_region index do_write_index .git/nested-reader.trace && + + test_write_lines unstaged >nested/tracked && + test_write_lines unstaged \ + >"$scoped_control/nested/tracked" && + scoped_stash_control_git stash push -q \ + -m nested-control -- nested/tracked && + scoped_stash_control_git rev-parse \ + "stash@{0}^{tree}" "stash@{0}^2^{tree}" \ + >.git/control-trees && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 >.git/control-pushed && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=nested/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/nested-push.trace" \ + git stash push -q -m nested-proof -- nested/tracked && + scoped_stash_full_proof .git/index && + test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/nested-push.trace && + ! test_trace2_data fsmonitor \ + untracked/proof-missing 1 <.git/nested-push.trace && + git rev-parse "stash@{0}^{tree}" "stash@{0}^2^{tree}" \ + >.git/candidate-trees && + test_cmp .git/control-trees .git/candidate-trees && + cp .git/index .git/after-push && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git --no-optional-locks status --porcelain=v2 \ + >.git/candidate-pushed && + test_cmp .git/control-pushed .git/candidate-pushed && + test_grep "^? nested/existing-untracked$" \ + .git/candidate-pushed && + test_grep "^? nested/new-untracked$" \ + .git/candidate-pushed && + test_cmp_bin .git/after-push .git/index && + + scoped_stash_control_git stash apply --index -q "stash@{0}" && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 >.git/control-applied && + scoped_stash_control_git --no-optional-locks \ + ls-files --stage >.git/control-staged && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=nested/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/nested-apply.trace" \ + git stash apply --index -q "stash@{0}" && + scoped_stash_full_proof .git/index && + ! test_trace2_data fsmonitor \ + untracked/proof-missing 1 <.git/nested-apply.trace && + cp .git/index .git/after-apply && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git --no-optional-locks status --porcelain=v2 \ + >.git/candidate-applied && + test_cmp .git/control-applied .git/candidate-applied && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git --no-optional-locks ls-files --stage \ + >.git/candidate-staged && + test_cmp .git/control-staged .git/candidate-staged && + test_grep "^? nested/existing-untracked$" \ + .git/candidate-applied && + test_grep "^? nested/new-untracked$" \ + .git/candidate-applied && + test_cmp "$scoped_control/nested/tracked" nested/tracked && + test_cmp_bin .git/after-apply .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'cached apply safely expires a pending nested provider event' ' + test_when_finished "rm -rf scoped-stash-soft-dirty \ + scoped-stash-soft-dirty-control" && + test_create_repo scoped-stash-soft-dirty && + ( + cd scoped-stash-soft-dirty && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + mkdir nested && + test_write_lines base >nested/tracked && + git add -- nested/tracked && + git commit -qm nested-base && + scoped_control="$PWD/../scoped-stash-soft-dirty-control" && + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + clone -q --no-hardlinks "$PWD" "$scoped_control" && + test_write_lines existing >nested/existing-untracked && + test_write_lines existing \ + >"$scoped_control/nested/existing-untracked" && + git config core.untrackedCache true && + git config core.fsmonitor true && + test-tool chmtime -120 nested/tracked && + scoped_stash_prime "$PWD" && + test_grep "^? nested/existing-untracked$" .git/prime && + GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ + test-tool dump-untracked-cache >.git/initial-cache && + test_grep "^/nested/ .* valid" .git/initial-cache && + test_grep "^existing-untracked$" .git/initial-cache && + + test_write_lines indexed >"$scoped_control/nested/tracked" && + scoped_stash_control_git diff -- nested/tracked \ + >.git/nested.patch && + scoped_stash_control_git checkout -- nested/tracked && + test_cmp "$scoped_control/nested/tracked" nested/tracked && + test_write_lines visible >nested/new-visible && + test_write_lines visible \ + >"$scoped_control/nested/new-visible" && + scoped_stash_control_git apply --cached \ + "$PWD/.git/nested.patch" && + scoped_stash_control_git --no-optional-locks \ + status --porcelain=v2 >.git/control.status && + scoped_stash_control_git --no-optional-locks \ + ls-files --stage >.git/control.staged && + test_grep "^1 MM .* nested/tracked$" .git/control.status && + test_grep "^? nested/existing-untracked$" \ + .git/control.status && + test_grep "^? nested/new-visible$" .git/control.status && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=nested/new-visible \ + GIT_TRACE2_EVENT="$PWD/.git/apply.trace" \ + git apply --cached .git/nested.patch && + test_trace2_data fsmonitor apply_count 1 <.git/apply.trace && + test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/apply.trace && + ! test_trace2_data fsmonitor \ + untracked/proof-missing 1 <.git/apply.trace && + test_region index do_write_index .git/apply.trace && + scoped_stash_full_proof .git/index && + GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ + test-tool dump-untracked-cache >.git/serialized-cache && + test_grep "^/nested/ .* recurse$" .git/serialized-cache && + ! test_grep "^/nested/ .* valid" .git/serialized-cache && + ! test_grep "^existing-untracked$" \ + .git/serialized-cache && + cp .git/index .git/after-apply && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/reader.trace" \ + git --no-optional-locks status --porcelain=v2 \ + >.git/candidate.status && + test_cmp .git/control.status .git/candidate.status && + test_grep "^? nested/existing-untracked$" \ + .git/candidate.status && + test_grep "^? nested/new-visible$" .git/candidate.status && + test_cmp_bin .git/after-apply .git/index && + ! test_region index do_write_index .git/reader.trace && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + git --no-optional-locks ls-files --stage \ + >.git/candidate.staged && + test_cmp .git/control.staged .git/candidate.staged && + test_cmp "$scoped_control/nested/tracked" nested/tracked && + test_cmp_bin .git/after-apply .git/index + ) +' + +test_done From 116bc025705e916d351f9cba75b3e20d3460fe00 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 13:03:19 -0500 Subject: [PATCH 34/73] clean-status: normalize redundant disabled submodule recursion In 062e636055 (status: preserve proofs across guarded Codex invocations, 2026-08-14), harmless command-scoped overrides were excluded from the clean-status configuration fingerprint. A redundant submodule.recurse=false override still creates a separate proof domain. After a guarded checkout rewrites a linked worktree's index, ordinary diffs can repeatedly hash the stale checkpoint from the original domain, especially when index.skipHash leaves no usable physical checksum. Track the last known submodule.recurse value while reading config. Ignore a command-scoped false value only when recursion already defaults to false or the preceding effective value is known to be false. Preserve true overrides, effective changes, invalid values, and unknown scopes in the fingerprint. This does not broaden the configuration-epoch shortcut or change the semantic and tracked-policy hashes. Cover precedence and boolean spellings with both hash algorithms. Add a linked-worktree regression with a zero index checksum and authenticated sidecars: the guarded checkout must leave a directly reusable proof, and two ordinary diffs must match an independent oracle without hashing the old checkpoint. A persistent true setting remains a counterexample. --- clean-status-config.c | 15 ++++ clean-status-config.h | 2 + t/t7519-status-fsmonitor.sh | 119 +++++++++++++++++++++++++++ t/unit-tests/u-clean-status-config.c | 88 ++++++++++++++++++++ 4 files changed, 224 insertions(+) diff --git a/clean-status-config.c b/clean-status-config.c index 3740bb929ef000..f927baa8ca4023 100644 --- a/clean-status-config.c +++ b/clean-status-config.c @@ -139,6 +139,21 @@ static int config_is_command_status_guard( digest->fsmonitor_value_enabled = boolean > 0; return redundant; } + if (!strcmp(key, "submodule.recurse")) { + int boolean = git_parse_maybe_bool(value); + int known_scope = ctx && ctx->kvi && + ctx->kvi->scope > CONFIG_SCOPE_UNKNOWN && + ctx->kvi->scope <= CONFIG_SCOPE_COMMAND; + int redundant = command && !boolean && + (!digest->submodule_recurse_seen || + digest->submodule_recurse_known_false); + + /* Recursion defaults to off; retain every effective change. */ + digest->submodule_recurse_seen = 1; + digest->submodule_recurse_known_false = + known_scope && !boolean; + return redundant; + } return command && value && ((!strcmp(key, "safe.barerepository") && diff --git a/clean-status-config.h b/clean-status-config.h index 779a45e720aed3..0c381ecd14a27b 100644 --- a/clean-status-config.h +++ b/clean-status-config.h @@ -25,6 +25,8 @@ struct clean_status_config_digest { unsigned fsmonitor_value_seen : 1; unsigned fsmonitor_value_boolean : 1; unsigned fsmonitor_value_enabled : 1; + unsigned submodule_recurse_seen : 1; + unsigned submodule_recurse_known_false : 1; }; void clean_status_config_init(struct clean_status_config_digest *digest, diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index eb1711f7149e94..3cc9ff531d0255 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -4304,6 +4304,125 @@ test_expect_success MACOS,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success MACOS,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'redundant disabled recursion preserves linked pre-commit diff proofs' ' + test_when_finished "rm -rf precommit-linked-proof precommit-linked-worktree" && + test_create_repo precommit-linked-proof && + ( + cd precommit-linked-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + git worktree add --detach ../precommit-linked-worktree HEAD && + git config index.skipHash true && + git config core.untrackedCache true && + git config core.fsmonitor true && + worktree="$PWD/../precommit-linked-worktree" && + gitdir=$(git -C "$worktree" rev-parse --absolute-git-dir) && + test-tool chmtime -120 "$worktree/tracked" "$worktree/sibling" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/checkpoint.trace" \ + git -C "$worktree" status >"$gitdir/checkpoint.out" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/checkpoint.trace" && + test_fsmonitor_full_proof "$gitdir/index" paired && + test_trailing_hash "$gitdir/index" >"$gitdir/index.hash" && + test_oid zero >"$gitdir/zero" && + test_cmp "$gitdir/zero" "$gitdir/index.hash" && + find "$gitdir" -maxdepth 1 -type f -name "index.csh1.*" \ + >"$gitdir/checkpoints" && + find "$gitdir" -maxdepth 1 -type f -name "index.cswi.*" \ + >"$gitdir/witnesses" && + test_line_count = 1 "$gitdir/checkpoints" && + test_line_count = 1 "$gitdir/witnesses" && + checkpoint=$(cat "$gitdir/checkpoints") && + witness=$(cat "$gitdir/witnesses") && + cp "$checkpoint" "$gitdir/checkpoint.before" && + cp "$witness" "$gitdir/witness.before" && + + test_write_lines dirty >"$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/checkout.trace" \ + git -C "$worktree" -c submodule.recurse=0 \ + checkout -- . && + test_region index do_write_index "$gitdir/checkout.trace" && + test_fsmonitor_full_proof "$gitdir/index" paired && + test_trailing_hash "$gitdir/index" >"$gitdir/index.hash" && + test_cmp "$gitdir/zero" "$gitdir/index.hash" && + test_cmp_bin "$gitdir/checkpoint.before" "$checkpoint" && + test_cmp_bin "$gitdir/witness.before" "$witness" && + cp "$gitdir/index" "$gitdir/index.before-diff" && + git -C "$worktree" --no-optional-locks \ + -c core.fsmonitor=false -c core.untrackedCache=false \ + -c core.trustctime=true -c core.checkStat=default \ + diff --no-ext-diff --no-textconv --ignore-submodules \ + >"$gitdir/expect" && + test_cmp_bin "$gitdir/index.before-diff" "$gitdir/index" && + for attempt in first second + do + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/diff-$attempt.trace" \ + git -C "$worktree" diff \ + --no-ext-diff --no-textconv --ignore-submodules \ + >"$gitdir/diff-$attempt.actual" && + test_cmp "$gitdir/expect" "$gitdir/diff-$attempt.actual" && + test_cmp_bin "$gitdir/index.before-diff" "$gitdir/index" && + test_cmp_bin "$gitdir/checkpoint.before" "$checkpoint" && + test_cmp_bin "$gitdir/witness.before" "$witness" && + test_grep ! "\"label\":\"history_logical_digest\"" \ + "$gitdir/diff-$attempt.trace" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/diff-$attempt.trace" && + ! test_trace2_data fsmonitor history/external-restored 1 \ + <"$gitdir/diff-$attempt.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/diff-$attempt.trace" && + ! test_region index do_write_index \ + "$gitdir/diff-$attempt.trace" || return 1 + done && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/checkout.trace" && + + git -C "$worktree" config submodule.recurse true && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/enabled.prime" && + test_must_be_empty "$gitdir/enabled.prime" && + test_fsmonitor_full_proof "$gitdir/index" paired && + test_write_lines dirty >"$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/enabled.checkout.trace" \ + git -C "$worktree" -c submodule.recurse=false \ + checkout -- . && + test_trace2_data fsmonitor config/coherent 0 \ + <"$gitdir/enabled.checkout.trace" && + cp "$gitdir/index" "$gitdir/enabled.index" && + git -C "$worktree" --no-optional-locks \ + -c core.fsmonitor=false -c core.untrackedCache=false \ + -c core.trustctime=true -c core.checkStat=default \ + diff --no-ext-diff --no-textconv --ignore-submodules \ + >"$gitdir/enabled.expect" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" diff \ + --no-ext-diff --no-textconv --ignore-submodules \ + >"$gitdir/enabled.actual" && + test_cmp "$gitdir/enabled.expect" "$gitdir/enabled.actual" && + test_cmp_bin "$gitdir/enabled.index" "$gitdir/index" + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'global second closing-query change rejects verified subtree reuse' ' test_when_finished "rm -rf second-query-global" && diff --git a/t/unit-tests/u-clean-status-config.c b/t/unit-tests/u-clean-status-config.c index 2df1ac6fad30e0..6d19a46d079995 100644 --- a/t/unit-tests/u-clean-status-config.c +++ b/t/unit-tests/u-clean-status-config.c @@ -335,6 +335,94 @@ void test_clean_status_config__only_safe_command_guards_are_normalized(void) } } +void test_clean_status_config__only_redundant_disabled_submodule_recursion_is_normalized(void) +{ + static const struct { + const char *first; + enum config_scope first_scope; + const char *second; + enum config_scope second_scope; + const char *override; + int normalized; + } cases[] = { + { NULL, 0, NULL, 0, "false", 1 }, + { NULL, 0, NULL, 0, "0", 1 }, + { NULL, 0, NULL, 0, "off", 1 }, + { NULL, 0, NULL, 0, "no", 1 }, + { "false", CONFIG_SCOPE_LOCAL, NULL, 0, "0", 1 }, + { "no", CONFIG_SCOPE_GLOBAL, NULL, 0, "off", 1 }, + { "true", CONFIG_SCOPE_LOCAL, NULL, 0, "false", 0 }, + { "invalid", CONFIG_SCOPE_LOCAL, NULL, 0, "false", 0 }, + { "false", CONFIG_SCOPE_UNKNOWN, NULL, 0, "false", 0 }, + { "false", CONFIG_SCOPE_SUBMODULE, NULL, 0, "false", 0 }, + { "true", CONFIG_SCOPE_GLOBAL, + "false", CONFIG_SCOPE_LOCAL, "false", 1 }, + { "false", CONFIG_SCOPE_GLOBAL, + "true", CONFIG_SCOPE_LOCAL, "false", 0 }, + { NULL, 0, NULL, 0, "true", 0 }, + { NULL, 0, NULL, 0, "invalid", 0 }, + { "false", CONFIG_SCOPE_LOCAL, NULL, 0, "true", 0 }, + { "true", CONFIG_SCOPE_COMMAND, + "false", CONFIG_SCOPE_COMMAND, "false", 1 }, + }; + static const int algorithms[] = { GIT_HASH_SHA1, GIT_HASH_SHA256 }; + struct key_value_info kvi = KVI_INIT; + struct config_context ctx = { .kvi = &kvi }; + + for (size_t a = 0; a < ARRAY_SIZE(algorithms); a++) { + const struct git_hash_algo *algo = &hash_algos[algorithms[a]]; + + for (size_t i = 0; i < ARRAY_SIZE(cases); i++) { + struct clean_status_config_digest baseline, digest; + + clean_status_config_init(&baseline, algo); + clean_status_config_init(&digest, algo); + if (cases[i].first) { + kvi.scope = cases[i].first_scope; + clean_status_config_add(&baseline, "submodule.recurse", + cases[i].first, &ctx); + clean_status_config_add(&digest, "submodule.recurse", + cases[i].first, &ctx); + } + if (cases[i].second) { + kvi.scope = cases[i].second_scope; + clean_status_config_add(&baseline, "submodule.recurse", + cases[i].second, &ctx); + clean_status_config_add(&digest, "submodule.recurse", + cases[i].second, &ctx); + } + clean_status_config_final(&baseline); + kvi.scope = CONFIG_SCOPE_COMMAND; + clean_status_config_add(&digest, "submodule.recurse", + cases[i].override, &ctx); + clean_status_config_final(&digest); + cl_assert_equal_i(hasheq(digest.hash, baseline.hash, algo), + cases[i].normalized); + cl_assert(hasheq(digest.semantic_hash, + baseline.semantic_hash, algo)); + cl_assert(hasheq(digest.tracked_policy_hash, + baseline.tracked_policy_hash, algo)); + } + + { + struct clean_status_config_digest baseline, digest; + + clean_status_config_init(&baseline, algo); + clean_status_config_init(&digest, algo); + clean_status_config_add(&baseline, "submodule.recurse", + "false", NULL); + clean_status_config_add(&digest, "submodule.recurse", + "false", NULL); + clean_status_config_final(&baseline); + kvi.scope = CONFIG_SCOPE_COMMAND; + clean_status_config_add(&digest, "submodule.recurse", + "false", &ctx); + clean_status_config_final(&digest); + cl_assert(!hasheq(digest.hash, baseline.hash, algo)); + } + } +} + void test_clean_status_config__command_empty_attributes_do_not_change_proof(void) { static const enum config_scope persistent_scopes[] = { From e658bba676e36fa3699cb81cac1c9b668260dccb Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 13:22:16 -0500 Subject: [PATCH 35/73] tests: integrate scoped proof coverage with both harnesses The new reader and stash suites run directly, but Meson's explicit test list does not include them. Its configure-time inventory check rejects the source tree before any tests can run. Register both scripts next to the existing status and preload coverage, and keep the existing history unit-test entries in the order required by check-meson. The nested stash regressions also negate test_grep at the shell level. Use the helper's negative-match form so a grep error cannot be mistaken for an expected absence. This satisfies test-greplint without changing the assertions or product code. The old source reproduces both inventory omissions and all four lint errors. The corrected inventory matches the test files, check-meson and the complete test-lint target pass, and the nine stash cases pass with SHA-1 and SHA-256. --- t/meson.build | 4 +++- t/t7533-status-scoped-stash.sh | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/t/meson.build b/t/meson.build index 3a6263fefd00f9..e486e988bc9aaa 100644 --- a/t/meson.build +++ b/t/meson.build @@ -2,8 +2,8 @@ clar_test_suites = [ 'unit-tests/u-attr-fingerprint.c', 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', - 'unit-tests/u-clean-status-history.c', 'unit-tests/u-clean-status-history-store.c', + 'unit-tests/u-clean-status-history.c', 'unit-tests/u-clean-status-identity.c', 'unit-tests/u-clean-status-index.c', 'unit-tests/u-clean-status-manifest.c', @@ -965,6 +965,8 @@ integration_tests = [ 't7530-status-clean-sidecar.sh', 't7531-semantic-verify.sh', 't7532-preload-index-linux.sh', + 't7533-status-scoped-stash.sh', + 't7534-status-scoped-readers.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t7533-status-scoped-stash.sh b/t/t7533-status-scoped-stash.sh index 81bdf080cabdf4..659cc220889bb9 100755 --- a/t/t7533-status-scoped-stash.sh +++ b/t/t7533-status-scoped-stash.sh @@ -577,8 +577,8 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ test-tool dump-untracked-cache >.git/dirty-cache && test_grep "^/nested/ .* recurse$" .git/dirty-cache && - ! test_grep "^/nested/ .* valid" .git/dirty-cache && - ! test_grep "^existing-untracked$" .git/dirty-cache && + test_grep ! "^/nested/ .* valid" .git/dirty-cache && + test_grep ! "^existing-untracked$" .git/dirty-cache && cp .git/index .git/before-reader && scoped_stash_control_git --no-optional-locks \ status --porcelain=v2 >.git/control-before && @@ -723,8 +723,8 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ test-tool dump-untracked-cache >.git/serialized-cache && test_grep "^/nested/ .* recurse$" .git/serialized-cache && - ! test_grep "^/nested/ .* valid" .git/serialized-cache && - ! test_grep "^existing-untracked$" \ + test_grep ! "^/nested/ .* valid" .git/serialized-cache && + test_grep ! "^existing-untracked$" \ .git/serialized-cache && cp .git/index .git/after-apply && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ From bc0e68c4258b2a5a1dc9cafdb9fbee8ac40a1c9c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 13:42:39 -0500 Subject: [PATCH 36/73] t7534: force the same-stat oracle to read file contents The same-stat check added by d079f66b83 (fsmonitor: avoid repository-wide bootstrap for bounded readers, 2026-08-16) compares the bounded reader with an fsmonitor-disabled control. It rewrites a file without changing its size and restores the original mtime. Without USE_NSEC, the control compares ctime at second resolution. An edit in the same second can therefore leave its cached stat data unchanged. The control reports a clean file while the bounded reader correctly detects the changed contents, making the test fail spuriously. For this control only, copy the index and reinsert the tracked entry with its original mode and object ID using update-index --cacheinfo. The new entry has no cached stat data, so the ordinary diff must read the file contents. The staged contents and physical index are unchanged, and the existing index and sidecar immutability checks still apply. --- t/t7534-status-scoped-readers.sh | 44 +++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/t/t7534-status-scoped-readers.sh b/t/t7534-status-scoped-readers.sh index 4d2081b4def702..249a4ac7813de9 100755 --- a/t/t7534-status-scoped-readers.sh +++ b/t/t7534-status-scoped-readers.sh @@ -46,13 +46,43 @@ assert_scoped_reader () { scoped_locks=$2 && shift 2 && cp "$gitdir/index" "$gitdir/$scoped_label.index" && - GIT_OPTIONAL_LOCKS=0 \ - git -c core.fsmonitor=false \ - -c core.untrackedCache=false \ - -c core.trustctime=true \ - -c core.checkStat=default \ - -C "$worktree" "$@" \ - >"$gitdir/$scoped_label.expect" && + if test "$scoped_label" = same-stat + then + scoped_oracle_index="$gitdir/$scoped_label.oracle.index" && + cp "$gitdir/index" "$scoped_oracle_index" && + GIT_INDEX_FILE="$scoped_oracle_index" \ + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -C "$worktree" ls-files --stage -- tracked \ + >"$gitdir/$scoped_label.stage" && + test_line_count = 1 "$gitdir/$scoped_label.stage" && + read scoped_mode scoped_oid scoped_stage scoped_path \ + <"$gitdir/$scoped_label.stage" && + test "$scoped_stage" = 0 && + test "$scoped_path" = tracked && + GIT_INDEX_FILE="$scoped_oracle_index" \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -C "$worktree" update-index \ + --cacheinfo "$scoped_mode,$scoped_oid,$scoped_path" && + GIT_INDEX_FILE="$scoped_oracle_index" \ + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + -C "$worktree" "$@" \ + >"$gitdir/$scoped_label.expect" + else + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + -C "$worktree" "$@" \ + >"$gitdir/$scoped_label.expect" + fi && GIT_OPTIONAL_LOCKS=$scoped_locks \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$gitdir/$scoped_label.trace" \ From 39647c8865314b2229f9ca6416478aee8ccd4215 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 13:59:10 -0500 Subject: [PATCH 37/73] t7534: force the required-filter control to read the file bc0e68c425 (t7534: force the same-stat oracle to read file contents, 2026-08-16) gives the same-stat comparison an index whose tracked entry has no cached stat data. The later required-filter comparison still uses the physical index, even though the same deliberately hidden edit remains in the worktree. On a filesystem whose cached timestamps match, that control can report a clean file without invoking the required filter. The candidate reads the file and correctly reports the filter failure, so the comparison fails for the wrong reason. Reuse the content-backed scratch index for the required-filter control. The candidate continues to read the physical index, and the existing filter-error and index-immutability assertions remain unchanged. --- t/t7534-status-scoped-readers.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/t/t7534-status-scoped-readers.sh b/t/t7534-status-scoped-readers.sh index 249a4ac7813de9..d4451eba636c2d 100755 --- a/t/t7534-status-scoped-readers.sh +++ b/t/t7534-status-scoped-readers.sh @@ -231,7 +231,9 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP --no-ext-diff --no-textconv -- tracked \ >"$gitdir/required-filter.actual" \ 2>"$gitdir/required-filter.error" && - test_must_fail env GIT_OPTIONAL_LOCKS=0 \ + test_must_fail env \ + GIT_INDEX_FILE="$gitdir/same-stat.oracle.index" \ + GIT_OPTIONAL_LOCKS=0 \ git -c core.fsmonitor=false \ -c core.untrackedCache=false \ -c core.trustctime=true \ From 284a8ae410943c720dae2d37dd91ebba2a5829bd Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 14:05:43 -0500 Subject: [PATCH 38/73] t7533: keep the staged-sibling fixture out of split-index mode 428d829240 (stash: retain paired history for scoped regular-file changes, 2026-08-16) adds a full-proof test for an unrelated staged sibling. Its setup helper clears GIT_TEST_SPLIT_INDEX inside a subshell, but the test starts another subshell after that helper returns. The linux-TEST-vars job therefore restores the inherited split-index setting. The two ordinary adds enable split index and downgrade the proof before the stash operation under test. The full-proof assertion fails for a fixture that no longer exercises the intended index form. Clear GIT_TEST_SPLIT_INDEX in the subshell that runs the scenario, as the other full-proof tests do. Keep the proof and independent-control assertions intact. The original fails and the corrected test passes under the inherited CI setting with both object formats. --- t/t7533-status-scoped-stash.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/t/t7533-status-scoped-stash.sh b/t/t7533-status-scoped-stash.sh index 659cc220889bb9..67bf189c6e7d31 100755 --- a/t/t7533-status-scoped-stash.sh +++ b/t/t7533-status-scoped-stash.sh @@ -216,6 +216,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP scoped_stash_setup scoped-stash-staged && ( cd scoped-stash-staged && + sane_unset GIT_TEST_SPLIT_INDEX && gitdir="$PWD/.git" && scoped_control="$PWD/../scoped-stash-staged-control" && git -c core.fsmonitor=false \ From 810ab59f8d19ae85fe258e37b864314c014a3a8d Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 14:58:01 -0500 Subject: [PATCH 39/73] am: retain authenticated worktree proofs before refresh e3c9a5b818 (fsmonitor: preserve proofs through filters, replay, and index locks, 2026-08-15) initializes authenticated history for cherry-pick and revert. But am invokes the apply machinery directly and never supplies the current clean-status configuration before its pre-apply refresh writes the index. The existing FSCF remains eligible for serialization, but the writer cannot bind it to the current configuration. It therefore clears the token and stat bindings even when the provider token has not changed. Each subsequent read-only status rebuilds the worktree manifest. Initialize the same guarded native-provider configuration and history before am first reads the index. Keep the existing checks for temporary three-way indexes, attribute changes, filters, and unsafe replacements. Cover primary and linked worktrees with an unchanged provider token, immediate physical-proof checks, independent commit and content oracles, and two immutable read-only followers. A patch that changes attribute semantics must still invalidate its proof. --- builtin/am.c | 11 +++ t/t7519-status-fsmonitor.sh | 154 +++++++++++++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/builtin/am.c b/builtin/am.c index e9623b8307793f..0c1039070325e9 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -9,9 +9,12 @@ #include "builtin.h" #include "abspath.h" #include "advice.h" +#include "clean-status-config.h" +#include "clean-status.h" #include "config.h" #include "editor.h" #include "environment.h" +#include "fsmonitor-settings.h" #include "gettext.h" #include "hex.h" #include "parse-options.h" @@ -2315,6 +2318,7 @@ int cmd_am(int argc, struct repository *repo UNUSED) { struct am_state state; + struct clean_status_config_digest clean_digest; int binary = -1; int keep_cr = -1; int patch_format = PATCH_FORMAT_UNKNOWN; @@ -2464,6 +2468,13 @@ int cmd_am(int argc, /* Ensure a valid committer ident can be constructed */ git_committer_info(IDENT_STRICT); + if (fstat_is_reliable() && !getenv(INDEX_ENVIRONMENT) && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC && + !clean_status_config_read_repository(the_repository, &clean_digest)) { + clean_status_enable_external_history(the_repository); + clean_status_set_config_digest(the_repository, &clean_digest); + } + if (repo_read_index_preload(the_repository, NULL, 0) < 0) die(_("failed to read the index")); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 3cc9ff531d0255..ced1e7c1cc438d 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -757,7 +757,7 @@ prepare_builtin_closure_repo () { } test_fsmonitor_full_proof () { - perl - "$1" "$2" <<-\EOF + perl - "$@" <<-\EOF binmode STDIN; open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; binmode $input; @@ -788,6 +788,8 @@ test_fsmonitor_full_proof () { "pending:$suffix" : $tokens{"FSMN"}; die "mismatched untracked token\n" unless $tokens{"FSUC"} eq $untracked; + die "unexpected provider token\n" if defined($ARGV[2]) && + $tokens{"FSMN"} ne $ARGV[2]; EOF } @@ -5475,4 +5477,154 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'am preserves complete worktree proofs at an unchanged provider token' ' + test_when_finished "rm -rf am-provider-proof am-provider-proof-linked \ + am-provider-proof.patch \ + am-provider-proof.expected-tree" && + test_create_repo am-provider-proof && + ( + cd am-provider-proof && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + test_write_lines sibling >sibling && + git add tracked sibling && + git commit -qm base && + test_write_lines patched >tracked && + git add tracked && + git commit -qm patched && + git rev-parse HEAD^{tree} >../am-provider-proof.expected-tree && + git format-patch -1 --stdout >../am-provider-proof.patch && + git reset --hard -q HEAD^ && + git worktree add --detach -q \ + ../am-provider-proof-linked HEAD && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../am-provider-proof-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + test-tool chmtime -120 \ + "$worktree/tracked" "$worktree/sibling" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + test_fsmonitor_full_proof "$gitdir/index" paired && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index \ + --force-write-index && + test_fsmonitor_full_proof "$gitdir/index" paired \ + "builtin:test:1" && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/am.trace" \ + git -C "$worktree" am --quiet \ + "$PWD/../am-provider-proof.patch" && + test_fsmonitor_full_proof "$gitdir/index" paired \ + "builtin:test:1" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/am.trace" && + git -C "$worktree" rev-parse HEAD^{tree} \ + >"$gitdir/actual-tree" && + test_cmp "$PWD/../am-provider-proof.expected-tree" \ + "$gitdir/actual-tree" && + test_write_lines patched >"$gitdir/expected-tracked" && + test_cmp "$gitdir/expected-tracked" \ + "$worktree/tracked" && + cp "$gitdir/index" "$gitdir/index.snapshot" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 >"$gitdir/expected" && + test_must_be_empty "$gitdir/expected" && + test_cmp_bin "$gitdir/index.snapshot" "$gitdir/index" && + for reader in first second + do + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$reader.trace" \ + git --no-optional-locks -C "$worktree" \ + status --porcelain=v2 \ + >"$gitdir/$reader.actual" && + test_cmp "$gitdir/expected" \ + "$gitdir/$reader.actual" && + test_cmp_bin "$gitdir/index.snapshot" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$reader.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$reader.trace" && + test_region ! index do_write_index \ + "$gitdir/$reader.trace" || return 1 + done || return 1 + done + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'am invalidates proofs when a patch changes attribute semantics' ' + test_when_finished "rm -rf am-provider-attributes \ + am-provider-attributes.patch" && + test_create_repo am-provider-attributes && + ( + cd am-provider-attributes && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + test_write_lines sibling >sibling && + test_write_lines "tracked text" >.gitattributes && + git add tracked sibling .gitattributes && + git commit -qm base && + test_write_lines patched >tracked && + test_write_lines "tracked -text" >.gitattributes && + git add tracked .gitattributes && + git commit -qm "change attribute semantics" && + git rev-parse HEAD^{tree} >.git/expected-tree && + git format-patch -1 --stdout >../am-provider-attributes.patch && + git reset --hard -q HEAD^ && + git config core.untrackedCache true && + git config core.fsmonitor true && + test-tool chmtime -120 tracked sibling .gitattributes && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --force-write-index && + test_fsmonitor_full_proof .git/index paired \ + "builtin:test:1" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/am.trace" \ + git am --quiet ../am-provider-attributes.patch && + ! test_fsmonitor_full_proof .git/index paired && + ! test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <.git/am.trace && + git rev-parse HEAD^{tree} >.git/actual-tree && + test_cmp .git/expected-tree .git/actual-tree && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git --no-optional-locks status --porcelain=v2 \ + >.git/actual && + git --no-optional-locks \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 >.git/expected && + test_cmp .git/expected .git/actual + ) +' + test_done From fec9824447dce1f522b9907e788eada494a73a0c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 16:01:16 -0500 Subject: [PATCH 40/73] t7519: age the staged entry before checkpoint reissue 5f2a52ecb8 (t7519: cover stash creation from linked external history, 2026-08-14) ages the initial tracked files before creating its linked history checkpoint, but later adds a new file with a fresh timestamp. On builds without USE_NSEC, the final writable status can reach the checkpoint writer in the same second as that new index entry. The writer correctly rejects the racy index, so the test intermittently misses its expected history/external-stored marker. Set the new file's mtime to 120 seconds before the current time before adding it. This keeps the fixture eligible without weakening the checkpoint guard. A future-mtime control reproduces the rejection on both the current and parent-equivalent native builds in SHA-1 and SHA-256. The aged fixture publishes the checkpoint in both hash formats. --- t/t7519-status-fsmonitor.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index ced1e7c1cc438d..315b2ca3578372 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -4850,6 +4850,7 @@ test_expect_success MACOS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHO >"$gitdir/witnesses" && test_line_count = 1 "$gitdir/witnesses" && test_write_lines staged >"$worktree/existing/staged" && + test-tool chmtime =-120 "$worktree/existing/staged" && git -C "$worktree" add existing/staged && GIT_INDEX_FILE="$gitdir/index" \ git -C "$worktree" status --porcelain=v2 \ From 687de436f150abf15b8eaa32143d3fd8afc7b88c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 17:10:24 -0500 Subject: [PATCH 41/73] stash: leave room for the object ID terminator d9b6634589 (stash: be careful what we store, 2023-10-11) added a stash-like check before updating refs/stash. It formats the candidate object ID in a GIT_MAX_HEXSZ-byte stack buffer, but oid_to_hex_r() also writes a terminating NUL. SHA-1 fits in the buffer; SHA-256 writes one byte past its end when storing an otherwise valid stash. Reserve the extra byte required by the formatting API. Add an explicit SHA-256 repository test for stash store and ordinary push so the usual SHA-1 sanitizer jobs also exercise a maximum-width object ID. The old code aborts with a stack-buffer-overflow in hash_to_hex_algop_r(); the corrected buffer preserves the existing stash validation and updates. --- builtin/stash.c | 2 +- t/t3903-stash.sh | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/builtin/stash.c b/builtin/stash.c index 3eb63d18faa0fd..03838fee5424d2 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -1169,7 +1169,7 @@ static int do_store_stash(const struct object_id *w_commit, const char *stash_ms int quiet) { struct stash_info info; - char revision[GIT_MAX_HEXSZ]; + char revision[GIT_MAX_HEXSZ + 1]; oid_to_hex_r(revision, w_commit); assert_stash_like(&info, revision); diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index 8ac681e41df64a..1401b82e4b1360 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -953,6 +953,31 @@ test_expect_success 'store called with non-stash commit' ' test_must_fail git stash store HEAD ' +test_expect_success 'stash store and push support explicit SHA-256 repositories' ' + test_when_finished "rm -rf stash-explicit-sha256" && + git init --object-format=sha256 stash-explicit-sha256 && + ( + cd stash-explicit-sha256 && + git config core.fsmonitor false && + test "$(git rev-parse --show-object-format)" = sha256 && + echo original >tracked && + git add tracked && + git commit -m base && + echo stored >tracked && + oid=$(git stash create) && + test "${#oid}" -eq 64 && + git stash store -m stored "$oid" && + test "$oid" = "$(git rev-parse refs/stash)" && + test "$oid" = "$(git reflog --format=%H -1 refs/stash)" && + git stash clear && + echo pushed >tracked && + git stash push -m pushed -- tracked && + test "$(cat tracked)" = original && + git stash pop && + test "$(cat tracked)" = pushed + ) +' + test_expect_success 'store updates stash ref and reflog' ' git stash clear && git reset --hard && From ef3af6c9783b46da009e2ef19098f7ac3bebbf16 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 17:51:27 -0500 Subject: [PATCH 42/73] t7530: probe fsmonitor without requiring an index write e4f0e5486f (status: issue sidecars after a verified full scan, 2026-07-28) added a prerequisite which runs status and expects a builtin fsmonitor token in the physical index. With a non-racy index, status can instead save external history and leave that index alone. The daemon works, but dump-fsmonitor sees no physical FSMN extension and the prerequisite skips every sidecar test. Query the running daemon with an explicit token, as t7527 already does, instead of requiring status to write the index. Keep the status smoke test and make its index non-racy so the probe covers this no-write case. The sidecar tests continue to check proof publication and reuse; no production behavior changes. --- t/t7530-status-clean-sidecar.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 40b21af45cb1f9..3714810732f746 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -20,11 +20,14 @@ test_lazy_prereq DURABLE_FSMONITOR ' ( cd durable-fsmonitor-probe && test_commit base tracked && + test-tool chmtime =-120 tracked && + git -c core.fsmonitor=false update-index --refresh && git config core.fsmonitor true && git fsmonitor--daemon start --start-timeout=10 && git status --porcelain=v2 >/dev/null && - test-tool dump-fsmonitor >token && - grep "^fsmonitor last update builtin:" token + test-tool fsmonitor-client query --token 0 >token && + nul_to_q token.filtered && + grep "^builtin:" token.filtered result=$? git fsmonitor--daemon stop >/dev/null 2>&1 || : exit $result From e94acd7cc2c7087a6e097e2b87251b04c374e59d Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sun, 16 Aug 2026 18:47:23 -0500 Subject: [PATCH 43/73] t7530: isolate external exclude FIFO parents The FIFO tests added by 61f7705f80 (status: answer exact clean status before index deserialization, 2026-07-27) put their configured core.excludesFile directly in the shared temporary directory. Exclude proofs compare the parent directory's full stat data while reading a source. An unrelated process creating or removing a temporary file can therefore invalidate the initial capture. In the raced test, status can then take the conservative fast-excludes fallback and exit successfully before reaching the synchronization barrier. The test fails without having replaced the excludes file. Give each FIFO fixture a private temporary directory outside its worktree. The intended regular-file-to-FIFO replacement still changes the authenticated source and its parent, and all existing race and fallback assertions remain intact. Leave the production proof checks unchanged. --- t/t7530-status-clean-sidecar.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 3714810732f746..6de465a64bda35 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -2211,9 +2211,11 @@ test_expect_success PIPE,DURABLE_FSMONITOR \ 'an existing exclude FIFO cannot block fast-path capture' ' test_when_finished "stop_daemon sidecar-exclude-fifo" && setup_repo sidecar-exclude-fifo && - exclude_file=$(mktemp \ + exclude_dir=$(mktemp -d \ "${TMPDIR:-/tmp}/git-status-exclude-fifo.XXXXXX") && - test_when_finished "rm -f \"$exclude_file\"" && + test_when_finished "rm -rf \"$exclude_dir\"" && + exclude_file=$exclude_dir/global && + : >"$exclude_file" && git -C sidecar-exclude-fifo config core.excludesFile \ "$exclude_file" && issue_sidecar sidecar-exclude-fifo && @@ -2231,9 +2233,10 @@ test_expect_success PIPE,DURABLE_FSMONITOR \ test_when_finished "stop_daemon sidecar-exclude-race" && test_when_finished "cleanup_fast_race" && setup_repo sidecar-exclude-race && - exclude_file=$(mktemp \ + exclude_dir=$(mktemp -d \ "${TMPDIR:-/tmp}/git-status-exclude-race.XXXXXX") && - test_when_finished "rm -f \"$exclude_file\"" && + test_when_finished "rm -rf \"$exclude_dir\"" && + exclude_file=$exclude_dir/global && test_write_lines ignored >"$exclude_file" && git -C sidecar-exclude-race config core.excludesFile \ "$exclude_file" && From cb6cbd4f5ebdd0981b07adde3a88d6bff32db08c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 02:19:10 -0500 Subject: [PATCH 44/73] status: defer Linux scoped history hashing until a repair 5684c457ea (status: refresh external history before exact proofs, 2026-08-10) captures the original logical index before refreshing its entries. That lets a later checkpoint authenticate the pre-repair source without losing a required physical stat update. On Linux, where the physical index has no durable identity, an ordinary tracked-only scoped status repeats that full-index hash even when it needs no repair. Two calls on an 858,001-entry dirty index spend about 350 ms each doing so. Defer the capture for explicit --untracked-files=no queries over a bounded set of literal, semantically safe tracked files. Require a full physical index, authenticated current provider and filter proofs, paired recursively valid untracked history, and a pinned original index. Keep the ordinary content checks, collection, and optional index lock. Omit the checkpoint only after those checks leave no persistent entry change, the complete proof still matches, and the entire index is non-racy. If refresh repairs an entry, read the original index through a duplicate of its pinned descriptor and recover the original logical hash. Then use the existing checkpoint and physical-index writers. Recheck the source under index.lock before publication and immediately before writing; if the source changed, roll back only our lock. This also handles a skipped index checksum without treating zero as a durable identity or allowing one zero-checksum writer to overwrite another. On a separately re-primed, 1,111,063-entry Linux index with a scripted provider, balanced old/new runs drop from 828-836 ms to 216 ms. The old logical digest takes 456-460 ms; the new path does not compute it. Both versions match an independent provider-disabled status and leave the physical index and existing checkpoints unchanged. The index read alone still takes 134-138 ms, and the whole-index racy check remains. Cover genuinely dirty and clean zero-stat entries, forwarded provider tokens, main and linked worktrees, both object formats, racy entries, active filters, and a competing zero-checksum writer. Other platforms, query shapes, and ordinary checkpoint publication keep their existing behavior. --- builtin/commit.c | 247 +++++++++++++++- clean-status-history.c | 67 +++++ clean-status.h | 6 + read-cache-ll.h | 3 + read-cache.c | 30 +- t/t7519-status-fsmonitor.sh | 563 ++++++++++++++++++++++++++++++++++++ 6 files changed, 908 insertions(+), 8 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 04bfacdb08354e..1d0f21138f8ca0 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -9,6 +9,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "abspath.h" #include "advice.h" #include "config.h" #include "lockfile.h" @@ -35,9 +36,11 @@ #include "preload-index.h" #include "read-cache.h" #include "refs.h" +#include "replace-object.h" #include "repository.h" #include "string-list.h" #include "submodule.h" +#include "symlinks.h" #include "rerere.h" #include "unpack-trees.h" #include "column.h" @@ -51,6 +54,7 @@ #include "pretty.h" #include "trace2.h" #include "trailer.h" +#include "wrapper.h" static const char * const builtin_commit_usage[] = { N_("git commit [-a | --interactive | --patch] [-s] [-v] [-u[]] [--amend]\n" @@ -1676,6 +1680,185 @@ static int clean_status_sidecar_needs_reissue(struct repository *repo, return reissue; } +#ifdef __linux__ +static int clean_status_scoped_history_test_barrier(void) +{ + const char *ready = + getenv("GIT_TEST_CLEAN_STATUS_SCOPED_HISTORY_BARRIER_READY"); + const char *resume = + getenv("GIT_TEST_CLEAN_STATUS_SCOPED_HISTORY_BARRIER_RESUME"); + const char *scripted = getenv("GIT_TEST_FSMONITOR_QUERY_SEQUENCE"); + struct stat fifo, opened; + char resumed; + int fd, failed; + + if (!ready && !resume) + return 0; + if (!scripted || !*scripted || !ready || !*ready || + !resume || !*resume || lstat(resume, &fifo) || + !S_ISFIFO(fifo.st_mode) || fifo.st_uid != geteuid()) + return -1; + fd = open(ready, O_WRONLY | O_CREAT | O_EXCL | + O_NOFOLLOW | O_CLOEXEC, 0600); + if (fd < 0) + return -1; + failed = fstat(fd, &opened) || !S_ISREG(opened.st_mode) || + opened.st_uid != geteuid() || opened.st_nlink != 1 || + write_in_full(fd, "ready\n", 6) != 6; + if (close(fd) || failed) + return -1; + fd = open(resume, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0) + return -1; + failed = fstat(fd, &opened) || !S_ISFIFO(opened.st_mode) || + opened.st_uid != geteuid() || + opened.st_dev != fifo.st_dev || opened.st_ino != fifo.st_ino || + read_in_full(fd, &resumed, 1) != 1; + if (close(fd) || failed) + return -1; + return 0; +} + +static int clean_status_scoped_pathspec_is_bounded( + const struct wt_status *status) +{ + const struct pathspec *pathspec = &status->pathspec; + struct index_state *istate = status->repo->index; + int i; + + if (pathspec->nr <= 0 || pathspec->nr > 64 || + pathspec->has_wildcard || + (pathspec->magic & + (PATHSPEC_GLOB | PATHSPEC_ICASE | + PATHSPEC_EXCLUDE | PATHSPEC_ATTR))) + return 0; + for (i = 0; i < pathspec->nr; i++) { + const struct pathspec_item *item = &pathspec->items[i]; + const struct cache_entry *ce; + struct stat st; + int pos; + + if (!item->match || item->len <= 0 || + item->match[item->len - 1] == '/' || + !strcmp(item->match, ".") || + has_symlink_leading_path(item->match, item->len) || + lstat(item->match, &st) || !S_ISREG(st.st_mode)) + return 0; + pos = index_name_pos(istate, item->match, item->len); + if (pos < 0) + return 0; + ce = istate->cache[pos]; + if (!S_ISREG(ce->ce_mode) || ce_stage(ce) || + ce_intent_to_add(ce) || ce_skip_worktree(ce) || + (ce->ce_flags & CE_VALID) || + !clean_status_index_entry_is_semantically_safe( + istate, ce, ce)) + return 0; + } + return 1; +} +#endif + +static int clean_status_scoped_provider_is_current( + const struct index_state *istate) +{ + return clean_status_has_persistent_fsmonitor_semantic_history(istate) && + clean_status_has_current_full_fsmonitor_proof(istate) && + istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + !istate->fsmonitor_last_update_pending && + istate->fsmonitor_untracked_extension_seen && + !istate->fsmonitor_untracked_extension_invalid && + istate->fsmonitor_untracked_valid && + istate->fsmonitor_untracked_token && + !strcmp(istate->fsmonitor_last_update, + istate->fsmonitor_untracked_token) && + istate->untracked && istate->untracked->root && + istate->untracked->root->valid && + istate->untracked->root->valid_recursive && + !istate->untracked->root->fsmonitor_dirty && + !istate->untracked->fsmonitor_dirty_paths.len && + !istate->fsmonitor_untracked_must_persist && + !clean_status_worktree_manifest_needs_refresh(istate) && + !clean_status_manifest_global_fallback(istate) && + !clean_status_external_history_was_restored(istate); +} + +static int clean_status_defer_scoped_history_capture( + const struct wt_status *status, + struct clean_status_index_snapshot *snapshot) +{ +#ifdef __linux__ + struct repository *repo = status->repo; + struct index_state *istate = repo->index; + struct stat st; + char *physical, *selected, *canonical; + int eligible; + + /* + * Legacy checkpoints cannot authenticate a deferred source digest. + * Keep their ordinary lock and content verification; only a later + * clean proof may decide that publishing a checkpoint is unnecessary. + */ + if (clean_status_identity_is_durable() || + !untracked_files_arg || strcmp(untracked_files_arg, "no") || + status->show_untracked_files != SHOW_NO_UNTRACKED_FILES || + status->show_ignored_mode || status->submodule_summary || + !clean_status_scoped_pathspec_is_bounded(status) || + getenv(INDEX_ENVIRONMENT) || + getenv(GIT_WORK_TREE_ENVIRONMENT) || + getenv(GIT_COMMON_DIR_ENVIRONMENT) || + getenv(DB_ENVIRONMENT) || + getenv(ALTERNATE_DB_ENVIRONMENT) || + istate != repo->index || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || + repo_config_get_split_index(repo) > 0 || + repo_config_values(repo)->apply_sparse_checkout || + !fstat_is_reliable() || + !repo->config_values_private_.trust_ctime || + !repo->config_values_private_.check_stat || + fsm_settings__get_mode(repo) != FSMONITOR_MODE_IPC || + (istate->cache_changed & + ~(FSMONITOR_CHANGED | UNTRACKED_CHANGED)) || + !clean_status_scoped_provider_is_current(istate) || + repo_has_replace_refs_uncached(repo)) + return 0; + + physical = xstrfmt("%s/index", repo_get_git_dir(repo)); + selected = real_pathdup(repo_get_index_file(repo), 0); + canonical = real_pathdup(physical, 0); + eligible = selected && canonical && + !fspathcmp(selected, canonical) && + !lstat(physical, &st) && S_ISREG(st.st_mode) && + st.st_nlink == 1 && + !clean_status_index_snapshot_pin_proof_epoch(snapshot, istate); + if (!eligible) + clean_status_index_snapshot_release(snapshot); + free(canonical); + free(selected); + free(physical); + return eligible; +#else + (void)status; + (void)snapshot; + return 0; +#endif +} + +static int clean_status_scoped_history_can_rollback( + const struct wt_status *status, + const struct clean_status_index_snapshot *snapshot) +{ + struct index_state *istate = status->repo->index; + + return clean_status_scoped_provider_is_current(istate) && + !(istate->cache_changed & + ~(FSMONITOR_CHANGED | UNTRACKED_CHANGED)) && + clean_status_index_snapshot_still_matches_proof_epoch( + snapshot, istate) && + !has_racy_timestamp(istate); +} + int cmd_status(int argc, const char **argv, const char *prefix, @@ -1698,6 +1881,11 @@ struct repository *repo UNUSED) int repository_inputs_changed = 0; int reissue_after_write = 0; int save_history_after_write = 0; + int deferred_scoped_history = 0; + int guarded_scoped_history_source = 0; + struct clean_status_index_snapshot scoped_history_source = { + .fd = -1, + }; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1834,9 +2022,22 @@ struct repository *repo UNUSED) if (use_optional_locks()) clean_status_require_external_history_source(the_repository); repo_read_index(the_repository); - if (use_optional_locks()) - clean_status_capture_external_history_source( - the_repository->index); + if (use_optional_locks()) { + deferred_scoped_history = + clean_status_defer_scoped_history_capture( + &s, &scoped_history_source); + guarded_scoped_history_source = deferred_scoped_history; + if (deferred_scoped_history) { + trace2_data_intmax("fsmonitor", the_repository, + "history/scoped-source-capture-deferred", 1); +#ifdef __linux__ + if (clean_status_scoped_history_test_barrier()) + die("invalid clean status scoped-history test barrier"); +#endif + } else + clean_status_capture_external_history_source( + the_repository->index); + } if (normal_clean_query && use_optional_locks() && clean_status_identity_is_durable() && (reissue_clean_sidecar || @@ -1876,7 +2077,36 @@ struct repository *repo UNUSED) wt_status_collect(&s); - if (0 <= fd) { + if (0 <= fd && guarded_scoped_history_source && + !clean_status_index_snapshot_still_matches_proof_epoch( + &scoped_history_source, the_repository->index)) { + rollback_lock_file(&index_lock); + fd = -1; + trace2_data_intmax("fsmonitor", the_repository, + "history/scoped-source-epoch-mismatch", 1); + } + if (0 <= fd && deferred_scoped_history) { + if (clean_status_scoped_history_can_rollback( + &s, &scoped_history_source)) { + rollback_lock_file(&index_lock); + fd = -1; + trace2_data_intmax("fsmonitor", the_repository, + "history/scoped-source-capture-skipped", 1); + } else { + trace2_data_intmax("fsmonitor", the_repository, + "history/scoped-source-repair-required", 1); + if (!clean_status_capture_external_history_source_from_snapshot( + the_repository->index, &scoped_history_source)) + deferred_scoped_history = 0; + else { + rollback_lock_file(&index_lock); + fd = -1; + trace2_data_intmax("fsmonitor", the_repository, + "history/scoped-source-epoch-mismatch", 1); + } + } + } + if (0 <= fd && !deferred_scoped_history) { int external_restored = clean_status_external_history_was_restored( the_repository->index); @@ -1953,6 +2183,14 @@ struct repository *repo UNUSED) fd = -1; } } + if (0 <= fd && guarded_scoped_history_source && + !clean_status_index_snapshot_still_matches_proof_epoch( + &scoped_history_source, the_repository->index)) { + rollback_lock_file(&index_lock); + fd = -1; + trace2_data_intmax("fsmonitor", the_repository, + "history/scoped-source-epoch-mismatch", 1); + } if (0 <= fd) { repo_update_index_if_able(the_repository, &index_lock); if (save_history_after_write && @@ -1974,6 +2212,7 @@ struct repository *repo UNUSED) rollback_lock_file(&index_lock); } } + clean_status_index_snapshot_release(&scoped_history_source); if (s.relative_paths) s.prefix = prefix; diff --git a/clean-status-history.c b/clean-status-history.c index 073182c9ed8f27..f3d5041f16853c 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -407,6 +407,17 @@ static int current_proof_is_writable(const struct index_state *istate) clean_status_revalidated_token_matches(istate); } +int clean_status_has_current_full_fsmonitor_proof( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return current_proof_is_writable(istate) && + state->manifest.checked && + !state->manifest.current_invalidated && + !state->manifest.global_fallback; +} + void clean_status_advance_fsmonitor_config_token( struct index_state *istate, const char *next_token) { @@ -567,6 +578,62 @@ void clean_status_capture_external_history_source( clean_status_history_store_record_release(&record); } +int clean_status_capture_external_history_source_from_snapshot( + struct index_state *istate, + const struct clean_status_index_snapshot *snapshot) +{ +#ifdef __linux__ + struct index_state original = INDEX_STATE_INIT(istate->repo); + struct clean_status_state *state = istate->clean_status; + unsigned char hash[GIT_MAX_RAWSZ]; + int owned, captured = -1; + + /* + * The selected entry may already contain a legitimate stat repair. + * Recover its original logical source from the still-pinned physical + * descriptor instead of authenticating the mutated in-memory index. + */ + if (!state || + !clean_status_external_history_enabled(istate) || + getenv(INDEX_ENVIRONMENT) || istate != istate->repo->index || + snapshot->fd < 0 || + !clean_status_index_snapshot_still_matches_proof_epoch( + snapshot, istate)) + goto done; + if (state->source_logical_hash_valid) { + captured = 0; + goto done; + } + owned = fcntl(snapshot->fd, F_DUPFD_CLOEXEC, 0); + if (owned < 0 || + do_read_index_from_fd(&original, owned, + istate->repo->index_file) < 0 || + original.repo != istate->repo || + original.version != snapshot->version || + original.cache_nr != snapshot->cache_nr || + !oideq(&original.oid, &snapshot->checksum) || + original.split_index || original.sparse_index != INDEX_EXPANDED || + clean_status_index_logical_digest(&original, hash) || + !clean_status_index_snapshot_still_matches_proof_epoch( + snapshot, istate)) + goto done; + memcpy(state->source_logical_hash, hash, + istate->repo->hash_algo->rawsz); + state->source_logical_hash_valid = 1; + trace2_data_intmax("fsmonitor", istate->repo, + "history/scoped-original-source-restored", 1); + captured = 0; + +done: + release_index(&original); + return captured; +#else + (void)istate; + (void)snapshot; + return -1; +#endif +} + static struct clean_status_external_checkpoint * clean_status_prepare_external_history(struct index_state *istate) { diff --git a/clean-status.h b/clean-status.h index 8602e66f6213c0..5a9698a8f9accc 100644 --- a/clean-status.h +++ b/clean-status.h @@ -8,6 +8,7 @@ struct cache_entry; struct attr_source_snapshot; struct clean_status_progress; struct clean_status_proof_epoch; +struct clean_status_index_snapshot; struct lock_file; struct repository; struct stat; @@ -66,6 +67,8 @@ int clean_status_revalidated_token_matches( int clean_status_has_persistent_fsmonitor_semantic_history( const struct index_state *istate); +int clean_status_has_current_full_fsmonitor_proof( + const struct index_state *istate); int clean_status_has_worktree_manifest_history( const struct index_state *istate); int clean_status_fsmonitor_semantic_adoption_needed( @@ -143,6 +146,9 @@ int clean_status_external_history_owns_index( void clean_status_require_external_history_source(struct repository *repo); void clean_status_capture_external_history_source( struct index_state *istate); +int clean_status_capture_external_history_source_from_snapshot( + struct index_state *istate, + const struct clean_status_index_snapshot *snapshot); int clean_status_save_external_history(struct index_state *istate); void clean_status_copy_fsmonitor_history(struct index_state *dst, const struct index_state *src); diff --git a/read-cache-ll.h b/read-cache-ll.h index 242474ce9861f5..34405b6d5e18d3 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -317,6 +317,9 @@ void prefetch_cache_entries(const struct index_state *istate, struct lock_file; int do_read_index(struct index_state *istate, const char *path, int must_exist); /* for testting only! */ +/* Takes ownership of fd, including when the state is already initialized. */ +int do_read_index_from_fd(struct index_state *istate, int fd, + const char *path); int read_index_from(struct index_state *, const char *path, const char *gitdir); int is_index_unborn(struct index_state *); diff --git a/read-cache.c b/read-cache.c index 7e85e45dc75a94..cf13875574f23d 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2466,8 +2466,9 @@ static void set_new_index_sparsity(struct index_state *istate) istate->sparse_index = 1; } -/* remember to discard_cache() before reading a different cache! */ -int do_read_index(struct index_state *istate, const char *path, int must_exist) +/* A nonnegative source_fd is owned by this reader. */ +static int do_read_index_1(struct index_state *istate, const char *path, + int must_exist, int source_fd) { int fd; struct stat st; @@ -2481,12 +2482,15 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) struct index_entry_offset_table *ieot = NULL; clean_status_attach_config(istate); - if (istate->initialized) + if (istate->initialized) { + if (source_fd >= 0) + close(source_fd); return istate->cache_nr; + } istate->timestamp.sec = 0; istate->timestamp.nsec = 0; - fd = git_open_cloexec(path, O_RDONLY); + fd = source_fd >= 0 ? source_fd : git_open_cloexec(path, O_RDONLY); if (fd < 0) { if (!must_exist && errno == ENOENT) { set_new_index_sparsity(istate); @@ -2625,6 +2629,24 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) die(_("index file corrupt")); } +/* remember to discard_cache() before reading a different cache! */ +int do_read_index(struct index_state *istate, const char *path, int must_exist) +{ + return do_read_index_1(istate, path, must_exist, -1); +} + +int do_read_index_from_fd(struct index_state *istate, int fd, + const char *path) +{ + if (fd < 0) + return -1; + if (istate->initialized) { + close(fd); + return -1; + } + return do_read_index_1(istate, path, 1, fd); +} + /* * Signal that the shared index is used by updating its mtime. * diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 315b2ca3578372..ede5d51d630c0c 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -5628,4 +5628,567 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP ) ' +test_lazy_prereq LINUX_SCOPED_HISTORY ' + test "$uname_s" = Linux +' + +test_expect_success LINUX_SCOPED_HISTORY,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'bounded tracked-only status verifies zero-stat entries before omitting history' ' + test_when_finished "rm -rf nondurable-scoped-repair \ + nondurable-scoped-repair-linked" && + test_create_repo nondurable-scoped-repair && + ( + cd nondurable-scoped-repair && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines indexed >cached/tracked && + test_write_lines sibling >cached/sibling && + test_write_lines "* -filter" "*.asset text" >.gitattributes && + test_write_lines ignored >.gitignore && + git add cached/tracked cached/sibling \ + .gitattributes .gitignore && + git commit -qm base && + git worktree add --detach -q \ + ../nondurable-scoped-repair-linked HEAD && + git config index.skipHash true && + git config core.untrackedCache true && + git config core.fsmonitor true && + git config filter.scoped.clean cat && + git config filter.scoped.smudge cat && + git config filter.scoped.required true && + git config status.showUntrackedFiles no && + for worktree in "$PWD" "$PWD/../nondurable-scoped-repair-linked" + do + gitdir=$(git -C "$worktree" \ + rev-parse --absolute-git-dir) && + test_write_lines temporary \ + >"$worktree/cached/tracked" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + diff -- cached/tracked >"$gitdir/scoped.patch" && + test_write_lines indexed \ + >"$worktree/cached/tracked" && + test-tool chmtime -120 \ + "$worktree/cached/tracked" && + test-tool chmtime -240 \ + "$worktree/cached/sibling" \ + "$worktree/.gitattributes" \ + "$worktree/.gitignore" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=normal \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + test_fsmonitor_full_proof "$gitdir/index" paired && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/forward.trace" \ + git -C "$worktree" apply --cached \ + "$gitdir/scoped.patch" && + test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <"$gitdir/forward.trace" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/reverse.trace" \ + git -C "$worktree" apply --cached --reverse \ + "$gitdir/scoped.patch" && + test_trace2_data fsmonitor \ + apply/untracked-replacement-preserved 1 \ + <"$gitdir/reverse.trace" && + test_fsmonitor_full_proof "$gitdir/index" paired && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + ls-files --debug -- cached/tracked \ + >"$gitdir/zero-stat" && + test_grep "ctime: 0:0" "$gitdir/zero-stat" && + test_grep "mtime: 0:0" "$gitdir/zero-stat" && + test_grep "size: 0" "$gitdir/zero-stat" && + ( + cd "$worktree" && + test-tool dump-fsmonitor + ) >"$gitdir/fsmonitor" && + test_grep "[-]$" "$gitdir/fsmonitor" && + ( + cd "$worktree" && + GIT_CONFIG_PARAMETERS="${SQ}core.fsmonitor=false${SQ}" \ + test-tool dump-untracked-cache + ) >"$gitdir/untracked" && + test_grep "^/ .* valid$" "$gitdir/untracked" && + test_grep "^/cached/ .* valid$" \ + "$gitdir/untracked" && + test_write_lines definitely-dirty \ + >"$worktree/cached/tracked" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 --untracked-files=no \ + -- cached/tracked >"$gitdir/dirty.expected" && + test_grep "^1 \\.M .* cached/tracked$" \ + "$gitdir/dirty.expected" && + + # Normal-untracked pathspecs must still publish global history. + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/publish.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=normal -- cached/tracked \ + >"$gitdir/publish.actual" && + test_cmp "$gitdir/dirty.expected" \ + "$gitdir/publish.actual" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/publish.trace" && + test_region fsmonitor history_logical_digest \ + "$gitdir/publish.trace" && + find "$gitdir" -maxdepth 1 -type f \ + -name "index.csh1.*" >"$gitdir/checkpoints" && + test_line_count = 1 "$gitdir/checkpoints" && + checkpoint=$(cat "$gitdir/checkpoints") && + cat >"$gitdir/retoken.pl" <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my $rawsz = $ARGV[0] eq "sha256" ? 32 : 20; + for my $name ("FSMN", "FSUC", "FSCF") { + my $at = index($index, $name); + die "missing $name" if $at < 0; + my $size = unpack("N", substr($index, $at + 4, 4)); + my $payload = substr($index, $at + 8, $size); + my $count = ($payload =~ + s/builtin:test:[0-9]/builtin:test:3/g); + die "unexpected $name token" unless $count == 1; + if ($name eq "FSCF") { + my $proof = substr($payload, 0, -$rawsz); + my $checksum = $rawsz == 32 ? + sha256($proof) : sha1($proof); + substr($payload, -$rawsz, $rawsz, + $checksum); + } + substr($index, $at + 8, $size, $payload); + } + my $payload = substr($index, 0, -$rawsz); + print $payload, "\0" x $rawsz; + EOF + perl "$gitdir/retoken.pl" "$(test_oid algo)" \ + <"$gitdir/index" >"$gitdir/index.retoken" && + mv "$gitdir/index.retoken" "$gitdir/index" && + test_fsmonitor_full_proof "$gitdir/index" paired \ + "builtin:test:3" && + test_trailing_hash "$gitdir/index" \ + >"$gitdir/initial-zero.hash" && + test_oid zero >"$gitdir/zero.expected" && + test_cmp "$gitdir/zero.expected" \ + "$gitdir/initial-zero.hash" && + cp "$gitdir/index" "$gitdir/zero.index" && + cp "$checkpoint" "$gitdir/checkpoint.snapshot" && + + for attempt in first second + do + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$attempt.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/$attempt.actual" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$attempt.trace" && + ! test_trace2_data fsmonitor config/invalid-extension 1 \ + <"$gitdir/$attempt.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$attempt.trace" && + test_cmp "$gitdir/dirty.expected" \ + "$gitdir/$attempt.actual" && + test_cmp_bin "$gitdir/zero.index" \ + "$gitdir/index" && + test_cmp_bin "$gitdir/checkpoint.snapshot" \ + "$checkpoint" && + test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/$attempt.trace" && + test_trace2_data fsmonitor \ + history/scoped-source-capture-skipped 1 \ + <"$gitdir/$attempt.trace" && + test_trace2_data fsmonitor config/token-advanced 1 \ + <"$gitdir/$attempt.trace" && + test_region ! fsmonitor history_logical_digest \ + "$gitdir/$attempt.trace" && + test_region ! index do_write_index \ + "$gitdir/$attempt.trace" || return 1 + done && + + # A zero-stat clean entry must still be physically repaired. + test_write_lines indexed \ + >"$worktree/cached/tracked" && + test-tool chmtime =-60 \ + "$worktree/cached/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/repair.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no -- cached/tracked \ + >"$gitdir/repair.actual" && + test_must_be_empty "$gitdir/repair.actual" && + test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/repair.trace" && + test_trace2_data fsmonitor \ + history/scoped-source-repair-required 1 \ + <"$gitdir/repair.trace" && + test_trace2_data fsmonitor \ + history/scoped-original-source-restored 1 \ + <"$gitdir/repair.trace" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/repair.trace" && + test_region fsmonitor history_logical_digest \ + "$gitdir/repair.trace" && + test_region index do_write_index \ + "$gitdir/repair.trace" && + ! test_cmp_bin "$gitdir/zero.index" \ + "$gitdir/index" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + ls-files --debug -- cached/tracked \ + >"$gitdir/repaired-stat" && + test_grep ! "size: 0" "$gitdir/repaired-stat" && + test_fsmonitor_full_proof "$gitdir/index" paired && + + # That repaired source still survives a subsequent foreign writer. + cp "$gitdir/index" "$gitdir/repaired.index" && + cp "$checkpoint" "$gitdir/repaired.checkpoint" && + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + update-index --force-write-index && + test_grep ! FSUC "$gitdir/index" && + cp "$gitdir/index" "$gitdir/foreign-stripped.index" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/follower.expected" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/follower.trace" \ + git --no-optional-locks -C "$worktree" \ + status --porcelain=v2 --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/follower.actual" && + test_cmp "$gitdir/follower.expected" \ + "$gitdir/follower.actual" && + test_cmp_bin "$gitdir/foreign-stripped.index" \ + "$gitdir/index" && + test_cmp_bin "$gitdir/repaired.checkpoint" \ + "$checkpoint" && + test_trace2_data fsmonitor history/external-restored 1 \ + <"$gitdir/follower.trace" && + test_region ! index do_write_index \ + "$gitdir/follower.trace" && + cp "$gitdir/repaired.index" "$gitdir/index" && + + # Neither a selected nor an unrelated racy CE may lose its write. + selected_mtime=$(test-tool chmtime --get \ + "$worktree/cached/tracked") && + test-tool chmtime =$selected_mtime "$gitdir/index" && + cp "$gitdir/index" "$gitdir/selected-racy.before" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/selected-racy.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no -- cached/tracked \ + >"$gitdir/selected-racy.actual" && + test_must_be_empty "$gitdir/selected-racy.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-skipped 1 \ + <"$gitdir/selected-racy.trace" && + test_trace2_data fsmonitor history/external-save-reject \ + racy-index \ + <"$gitdir/selected-racy.trace" && + test_region index do_write_index \ + "$gitdir/selected-racy.trace" && + test-tool chmtime =-30 \ + "$worktree/cached/sibling" && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/sibling \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=normal \ + >"$gitdir/sibling-refresh.actual" && + test_must_be_empty "$gitdir/sibling-refresh.actual" && + test_fsmonitor_full_proof "$gitdir/index" paired && + sibling_mtime=$(test-tool chmtime --get \ + "$worktree/cached/sibling") && + test-tool chmtime =$sibling_mtime "$gitdir/index" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/unselected-racy.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no -- cached/tracked \ + >"$gitdir/unselected-racy.actual" && + test_must_be_empty "$gitdir/unselected-racy.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-skipped 1 \ + <"$gitdir/unselected-racy.trace" && + test_trace2_data fsmonitor history/external-save-reject \ + racy-index \ + <"$gitdir/unselected-racy.trace" && + test_region index do_write_index \ + "$gitdir/unselected-racy.trace" && + + # Dirt that disappears during refresh must take the repair path. + if test_have_prereq PIPE + then + cp "$gitdir/zero.index" "$gitdir/index" && + test_write_lines race-dirty \ + >"$worktree/cached/tracked" && + ready="$gitdir/dirty-clean.ready" && + resume="$gitdir/dirty-clean.resume" && + fsmonitor_query_pid= && + trap cleanup_fsmonitor_query_barrier 0 && + mkfifo "$resume" && + { + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TEST_CLEAN_STATUS_SCOPED_HISTORY_BARRIER_READY="$ready" \ + GIT_TEST_CLEAN_STATUS_SCOPED_HISTORY_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$gitdir/dirty-clean.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/dirty-clean.actual" \ + 2>"$gitdir/dirty-clean.err" & + fsmonitor_query_pid=$! + } && + wait_for_fsmonitor_query_barrier \ + "$ready" "$fsmonitor_query_pid" && + test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/dirty-clean.trace" && + test_write_lines indexed \ + >"$worktree/cached/tracked" && + test-tool chmtime =-60 \ + "$worktree/cached/tracked" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/dirty-clean.expected" && + printf x >"$resume" && + wait "$fsmonitor_query_pid" && + fsmonitor_query_pid= && + trap - 0 && + test_cmp "$gitdir/dirty-clean.expected" \ + "$gitdir/dirty-clean.actual" && + test_trace2_data fsmonitor \ + history/scoped-source-repair-required 1 \ + <"$gitdir/dirty-clean.trace" && + test_trace2_data fsmonitor \ + history/scoped-original-source-restored 1 \ + <"$gitdir/dirty-clean.trace" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/dirty-clean.trace" && + test_region index do_write_index \ + "$gitdir/dirty-clean.trace" && + test_fsmonitor_full_proof "$gitdir/index" paired + else + : + fi && + + # Root equivalents and implicit -uno retain the original writer. + for scope in root root-dot root-magic wildcard implicit + do + case "$scope" in + root) + set -- --untracked-files=no + ;; + root-dot) + set -- --untracked-files=no -- . + ;; + root-magic) + set -- --untracked-files=no -- :/ + ;; + wildcard) + set -- --untracked-files=no -- "cached/*" + ;; + implicit) + set -- -- cached/tracked + ;; + esac && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$scope.trace" \ + git -C "$worktree" status --porcelain=v2 \ + "$@" >"$gitdir/$scope.actual" && + test_must_be_empty "$gitdir/$scope.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/$scope.trace" && + test_region fsmonitor history_logical_digest \ + "$gitdir/$scope.trace" || return 1 + done && + + # A bounded literal must not select an attribute or exclude source. + for source in .gitattributes .gitignore + do + case "$source" in + .gitattributes) + label=attributes + ;; + .gitignore) + label=ignore + ;; + esac && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$label.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no -- "$source" \ + >"$gitdir/$label.actual" && + test_must_be_empty "$gitdir/$label.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/$label.trace" || return 1 + done && + + # A mismatched config and an actual provider delta fail closed. + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/config.trace" \ + git -C "$worktree" -c core.autocrlf=true \ + status --porcelain=v2 \ + --untracked-files=no -- cached/tracked \ + >"$gitdir/config.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/config.trace" && + cp "$gitdir/zero.index" "$gitdir/index" && + test_write_lines provider-dirty \ + >"$worktree/cached/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/tracked \ + GIT_TRACE2_EVENT="$gitdir/provider.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no -- cached/tracked \ + >"$gitdir/provider.actual" && + test_grep "^1 \\.M .* cached/tracked$" \ + "$gitdir/provider.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-skipped 1 \ + <"$gitdir/provider.trace" && + + # An active clean filter cannot enter this lane. + test_write_lines "cached/tracked filter=scoped" \ + >"$worktree/.gitattributes" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + git -C "$worktree" add -- .gitattributes && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=normal \ + >"$gitdir/active-prime.actual" && + git --no-optional-locks -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/active.expected" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/active.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no -- cached/tracked \ + >"$gitdir/active.actual" && + test_cmp "$gitdir/active.expected" \ + "$gitdir/active.actual" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/active.trace" && + + # A subsequently failing required filter cannot be bypassed. + cp "$gitdir/index" "$gitdir/filter.before" && + test_must_fail env \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$gitdir/filter.trace" \ + git -C "$worktree" \ + -c filter.scoped.clean=false \ + -c filter.scoped.smudge=cat \ + -c filter.scoped.required=true \ + status --porcelain=v2 --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/filter.actual" \ + 2>"$gitdir/filter.err" && + test_grep "clean filter .scoped. failed" \ + "$gitdir/filter.err" && + test_cmp_bin "$gitdir/filter.before" "$gitdir/index" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-skipped 1 \ + <"$gitdir/filter.trace" && + + # A competing physical writer must never be overwritten. + if test_have_prereq PIPE + then + test_write_lines "* -filter" "*.asset text" \ + >"$worktree/.gitattributes" && + cp "$gitdir/zero.index" "$gitdir/index" && + test_write_lines race-dirty \ + >"$worktree/cached/tracked" && + ready="$gitdir/foreign.ready" && + resume="$gitdir/foreign.resume" && + fsmonitor_query_pid= && + trap cleanup_fsmonitor_query_barrier 0 && + mkfifo "$resume" && + { + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TEST_CLEAN_STATUS_SCOPED_HISTORY_BARRIER_READY="$ready" \ + GIT_TEST_CLEAN_STATUS_SCOPED_HISTORY_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$gitdir/foreign.trace" \ + git -C "$worktree" status --porcelain=v2 \ + --untracked-files=no \ + -- cached/tracked \ + >"$gitdir/foreign.actual" \ + 2>"$gitdir/foreign.err" & + fsmonitor_query_pid=$! + } && + wait_for_fsmonitor_query_barrier \ + "$ready" "$fsmonitor_query_pid" && + test_trace2_data fsmonitor \ + history/scoped-source-capture-deferred 1 \ + <"$gitdir/foreign.trace" && + test_path_is_missing "$gitdir/index.lock" && + test_write_lines competing \ + >"$worktree/cached/sibling" && + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + add cached/sibling && + cp "$gitdir/index" "$gitdir/foreign.index" && + test_trailing_hash "$gitdir/foreign.index" \ + >"$gitdir/foreign.zero" && + test_cmp "$gitdir/zero.expected" \ + "$gitdir/foreign.zero" && + printf x >"$resume" && + wait "$fsmonitor_query_pid" && + fsmonitor_query_pid= && + trap - 0 && + test_cmp_bin "$gitdir/foreign.index" \ + "$gitdir/index" && + ! test_trace2_data fsmonitor \ + history/scoped-source-capture-skipped 1 \ + <"$gitdir/foreign.trace" && + test_trace2_data fsmonitor \ + history/scoped-source-epoch-mismatch 1 \ + <"$gitdir/foreign.trace" && + ! test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/foreign.trace" && + test_region ! index do_write_index \ + "$gitdir/foreign.trace" + else + : + fi || return 1 + done + ) +' + test_done From aaa01b4ec19c0c269d0af240cdbe0a01d128b491 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 03:21:27 -0500 Subject: [PATCH 45/73] clean-status: release the raw fsmonitor shadow bitmap In cb6cbd4f5e (status: defer Linux scoped history hashing until a repair, 2026-08-17), the repair path began reading the pinned original index into a temporary index_state. This raw read deliberately skips post_read_index_from(), so it does not run tweak_fsmonitor(), which would normally consume and free the FSMN dirty bitmap. release_index() does not own that bitmap. Consequently, reading an FSMN extension into the temporary index leaks both the EWAH structure and its buffer. The Linux LeakSanitizer jobs report 56 bytes per capture from the existing zero-stat repair test in t7519. Free and clear the temporary bitmap at the common cleanup label before releasing the index, as the other parsed-index shadows in this file already do. This covers failed parses and successful captures without changing the proof, pinned descriptor, or index-write decisions. --- clean-status-history.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clean-status-history.c b/clean-status-history.c index f3d5041f16853c..8dcd0760909065 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -625,6 +625,9 @@ int clean_status_capture_external_history_source_from_snapshot( captured = 0; done: + if (original.fsmonitor_dirty) + ewah_free(original.fsmonitor_dirty); + original.fsmonitor_dirty = NULL; release_index(&original); return captured; #else From d313b5f240ee3f224899ad77bb38ab4c19e70982 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 11:21:01 -0500 Subject: [PATCH 46/73] commit: restore external proofs before preparing the index A foreign index writer can preserve FSMN and UNTR while removing the paired FSUC and FSCF extensions. Commit reads that index before it has enabled external history, so an authenticated checkpoint cannot prevent another worktree-manifest scan. Partial commits also discard the proof when constructing their temporary base index. Enable external history before the first index read for an ordinary worktree using reliable stat data and builtin fsmonitor. Carry that history into a partial commit's base index only when its provider token has already been revalidated. Alternate indexes retain their existing behavior. Exercise all, include, amend, and only commits in main and linked worktrees after removing the paired physical extensions. Require the external proof to survive without another manifest scan, and retain a changed-attributes control which must invalidate it. --- builtin/commit.c | 6 ++ t/t7519-status-fsmonitor.sh | 147 ++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/builtin/commit.c b/builtin/commit.c index 1d0f21138f8ca0..1de778261812d5 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -358,6 +358,9 @@ static void create_base_index(const struct commit *current_head) opts.head_idx = 1; opts.index_only = 1; opts.merge = 1; + opts.preserve_semantic_history = + clean_status_external_history_enabled(the_repository->index) && + clean_status_revalidated_token_matches(the_repository->index); opts.src_index = the_repository->index; opts.dst_index = the_repository->index; @@ -2368,6 +2371,9 @@ int cmd_commit(int argc, &s, git_commit_config, &clean_digest); clean_status_config_final(&clean_digest); clean_status_set_config_digest(the_repository, &clean_digest); + if (!getenv(INDEX_ENVIRONMENT) && fstat_is_reliable() && + fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC) + clean_status_enable_external_history(the_repository); s.commit_template = 1; status_format = STATUS_FORMAT_NONE; /* Ignore status.short */ s.colopts = 0; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index ede5d51d630c0c..a253bd28ce6302 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1569,6 +1569,153 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'commit modes restore external-only worktree proofs' ' + test_when_finished "rm -rf commit-external-only commit-external-linked" && + test_create_repo commit-external-only && + ( + cd commit-external-only && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_write_lines "tracked -text" >.gitattributes && + git add .gitattributes && + git commit -qm attributes && + git worktree add --detach ../commit-external-linked HEAD && + test-tool chmtime -120 tracked .gitattributes \ + ../commit-external-linked/tracked \ + ../commit-external-linked/.gitattributes && + git update-index --refresh && + git -C ../commit-external-linked update-index --refresh && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + cat >.git/remove-paired-proofs.pl <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my $rawsz = $ARGV[0] eq "sha256" ? 32 : 20; + for my $name ("FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + substr($index, $offset, 8 + $size, ""); + } + my $payload = substr($index, 0, -$rawsz); + print $payload, $rawsz == 32 ? sha256($payload) : sha1($payload); + EOF + for worktree in "$PWD" "$PWD/../commit-external-linked" + do + gitdir=$(git -C "$worktree" rev-parse --absolute-git-dir) && + for mode in all include amend only attributes + do + test-tool chmtime -120 "$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git -C "$worktree" update-index --refresh && + rm -f "$gitdir"/index.csh1.* "$gitdir"/index.cswi.* && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$mode.prime" && + test_must_be_empty "$gitdir/$mode.prime" && + test_fsmonitor_full_proof "$gitdir/index" paired && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkpoint.trace" \ + git -C "$worktree" status --short \ + >"$gitdir/$mode.checkpoint" && + test_must_be_empty "$gitdir/$mode.checkpoint" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/$mode.checkpoint.trace" && + find "$gitdir" -maxdepth 1 -type f \ + -name "index.csh1.*" >"$gitdir/$mode.csh" && + test_line_count = 1 "$gitdir/$mode.csh" && + perl "$PWD/.git/remove-paired-proofs.pl" \ + "$(test_oid algo)" <"$gitdir/index" \ + >"$gitdir/index.foreign" && + mv "$gitdir/index.foreign" "$gitdir/index" && + test_grep FSMN "$gitdir/index" && + test_grep ! FSUC "$gitdir/index" && + test_grep ! FSCF "$gitdir/index" && + query=DDDDCCCCCCCCCCCC && + case "$mode" in + all) + test_write_lines all >"$worktree/tracked" && + path=tracked && + set -- -a -qm commit-all + ;; + include) + test_write_lines include >"$worktree/tracked" && + path=tracked && + set -- --include tracked -qm commit-include + ;; + amend) + test_write_lines amend >"$worktree/tracked" && + path=tracked && + set -- -a --amend --no-edit -q + ;; + only) + test_write_lines only >"$worktree/tracked" && + path=tracked && + query=DDDDDDCCCCCCCCCCCC && + set -- --only tracked -qm commit-only + ;; + attributes) + test_write_lines "tracked text" \ + >"$worktree/.gitattributes" && + path=.gitattributes && + set -- -a -qm changed-attributes + ;; + esac && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE="$query" \ + GIT_TEST_FSMONITOR_QUERY_PATH="$path" \ + GIT_TRACE2_EVENT="$gitdir/$mode.commit.trace" \ + git -C "$worktree" commit "$@" && + test_trace2_data fsmonitor history/external-restored 1 \ + <"$gitdir/$mode.commit.trace" && + if test "$mode" = attributes + then + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/$mode.commit.trace" && + test_trace2_data fsmonitor semantic/manifest-invalidated 1 \ + <"$gitdir/$mode.commit.trace" && + test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/$mode.commit.trace" + else + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/$mode.commit.trace" && + ! test_trace2_data fsmonitor untracked/proof-missing 1 \ + <"$gitdir/$mode.commit.trace" && + if test "$mode" = only + then + test_trace2_data fsmonitor history/untracked-paired-transfer 1 \ + <"$gitdir/$mode.commit.trace" || return 1 + fi && + test_fsmonitor_full_proof "$gitdir/index" paired + fi && + cp "$gitdir/index" "$gitdir/$mode.before-status" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$mode.status" && + test_must_be_empty "$gitdir/$mode.status" && + test_cmp_bin "$gitdir/$mode.before-status" "$gitdir/index" && + if test "$mode" != attributes + then + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$mode.status.trace" && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <"$gitdir/$mode.status.trace" || return 1 + fi || return 1 + done || return 1 + done + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'write-tree preserves authenticated primary and linked index proofs' ' test_when_finished "rm -rf write-tree-bound-proof write-tree-linked" && From 88f1686b27819d8bc70f13fabb22f612f4694292 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 11:29:19 -0500 Subject: [PATCH 47/73] fsmonitor: recognize equivalent index-only reader requests d079f66b83 (fsmonitor: avoid repository-wide bootstrap for bounded readers, 2026-08-16) lets selected readers use ordinary stat checks when the index has only a partial semantic proof. Its argument checks miss equivalent check-attr and ls-files syntax, and the diff plumbing never opts in. These commands can rebuild the whole worktree manifest before answering an index-only or single-path request. Decide from the parsed operation instead. Pure index-name and stage output does not depend on fsmonitor validity bits, so ls-files can also accept deleted paths, globs, and an unrestricted index listing. Keep worktree, attribute, tag, debug, and other state-sensitive modes out. Use check-attr's parsed filename boundary and share porcelain diff's existing regular-path restriction with diff-files and diff-index. Extend the partial-proof regression with equivalent spellings, index-only and plumbing forms, and unsupported-mode controls. Both formats reproduce the old manifest scan and pass with the expanded admission. --- builtin/check-attr.c | 29 ++++++------- builtin/diff-files.c | 8 ++++ builtin/diff-index.c | 15 ++++++- builtin/diff.c | 25 ----------- builtin/ls-files.c | 37 ++++++---------- diff-lib.c | 24 +++++++++++ diff.h | 1 + t/t7534-status-scoped-readers.sh | 72 ++++++++++++++++++++++++++++++++ 8 files changed, 144 insertions(+), 67 deletions(-) diff --git a/builtin/check-attr.c b/builtin/check-attr.c index f7083585fbfa95..d000e8d0f221dc 100644 --- a/builtin/check-attr.c +++ b/builtin/check-attr.c @@ -129,23 +129,6 @@ int cmd_check_attr(int argc, prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; - if (!stdin_paths && !source) { - for (i = 0; i < argc && strcmp(argv[i], "--"); i++) - ; - scoped_bootstrap = i < argc && - argc - i - 1 > 0 && argc - i - 1 <= 64; - } - if (scoped_bootstrap) - fsmonitor_begin_scoped_bootstrap(the_repository->index); - if (repo_read_index(the_repository) < 0) { - die("invalid cache"); - } - if (scoped_bootstrap) - fsmonitor_end_scoped_bootstrap(the_repository->index); - - if (cached_attrs) - git_attr_set_direction(GIT_ATTR_INDEX); - doubledash = -1; for (i = 0; doubledash < 0 && i < argc; i++) { if (!strcmp(argv[i], "--")) @@ -188,6 +171,18 @@ int cmd_check_attr(int argc, error_with_usage("No file specified"); } + scoped_bootstrap = !stdin_paths && !source && + argc - filei > 0 && argc - filei <= 64; + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(the_repository->index); + if (repo_read_index(the_repository) < 0) + die("invalid cache"); + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(the_repository->index); + + if (cached_attrs) + git_attr_set_direction(GIT_ATTR_INDEX); + check = attr_check_alloc(); if (!all_attrs) { for (i = 0; i < cnt; i++) { diff --git a/builtin/diff-files.c b/builtin/diff-files.c index 0de2094ca2a62d..267bbe3a41e6b8 100644 --- a/builtin/diff-files.c +++ b/builtin/diff-files.c @@ -12,6 +12,7 @@ #include "diff.h" #include "diff-merges.h" #include "commit.h" +#include "fsmonitor.h" #include "preload-index.h" #include "revision.h" @@ -27,6 +28,7 @@ int cmd_diff_files(int argc, { struct rev_info rev; int result; + int scoped_bootstrap; unsigned options = 0; show_usage_if_asked(argc, argv, diff_files_usage); @@ -85,8 +87,14 @@ int cmd_diff_files(int argc, diff_merges_set_dense_combined_if_unset(&rev); prepare_diff_external_history(the_repository); + scoped_bootstrap = + diff_has_bounded_regular_pathspec(&rev.diffopt.pathspec); + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(the_repository->index); if (repo_read_index_preload(the_repository, &rev.diffopt.pathspec, 0) < 0) die_errno("repo_read_index_preload"); + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(the_repository->index); run_diff_files(&rev, options); result = diff_result_code(&rev); release_revisions(&rev); diff --git a/builtin/diff-index.c b/builtin/diff-index.c index 880a12d34b258f..701e900f78086f 100644 --- a/builtin/diff-index.c +++ b/builtin/diff-index.c @@ -6,6 +6,7 @@ #include "diff.h" #include "diff-merges.h" #include "commit.h" +#include "fsmonitor.h" #include "preload-index.h" #include "revision.h" #include "setup.h" @@ -25,6 +26,7 @@ int cmd_diff_index(int argc, unsigned int option = 0; int i; int result; + int scoped_bootstrap; show_usage_if_asked(argc, argv, diff_cache_usage); @@ -69,16 +71,27 @@ int cmd_diff_index(int argc, rev.max_count != -1 || rev.min_age != -1 || rev.max_age != -1) usage(diff_cache_usage); prepare_diff_external_history(the_repository); - if (!(option & DIFF_INDEX_CACHED)) { + if (!(option & DIFF_INDEX_CACHED)) setup_work_tree(the_repository); + scoped_bootstrap = (option & DIFF_INDEX_CACHED) || + diff_has_bounded_regular_pathspec(&rev.diffopt.pathspec); + if (scoped_bootstrap) + fsmonitor_begin_scoped_bootstrap(the_repository->index); + if (!(option & DIFF_INDEX_CACHED)) { if (repo_read_index_preload(the_repository, &rev.diffopt.pathspec, 0) < 0) { + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(the_repository->index); perror("repo_read_index_preload"); return -1; } } else if (repo_read_index(the_repository) < 0) { + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(the_repository->index); perror("repo_read_index"); return -1; } + if (scoped_bootstrap) + fsmonitor_end_scoped_bootstrap(the_repository->index); run_diff_index(&rev, option); result = diff_result_code(&rev); release_revisions(&rev); diff --git a/builtin/diff.c b/builtin/diff.c index 643a4925e12fbd..4d983985e00bd6 100644 --- a/builtin/diff.c +++ b/builtin/diff.c @@ -33,7 +33,6 @@ #include "revision.h" #include "log-tree.h" #include "setup.h" -#include "symlinks.h" #include "thread-utils.h" #include "oid-array.h" #include "tree.h" @@ -55,30 +54,6 @@ COMMON_DIFF_OPTIONS_HELP; static int scoped_diff_bootstrap_used; -static int diff_has_bounded_regular_pathspec(const struct pathspec *pathspec) -{ - int i; - - if (pathspec->nr <= 0 || pathspec->nr > 64 || - pathspec->has_wildcard || - (pathspec->magic & - (PATHSPEC_GLOB | PATHSPEC_ICASE | - PATHSPEC_EXCLUDE | PATHSPEC_ATTR))) - return 0; - for (i = 0; i < pathspec->nr; i++) { - const struct pathspec_item *item = &pathspec->items[i]; - struct stat st; - - if (!item->match || item->len <= 0 || - item->match[item->len - 1] == '/' || - !strcmp(item->match, ".") || - has_symlink_leading_path(item->match, item->len) || - lstat(item->match, &st) || !S_ISREG(st.st_mode)) - return 0; - } - return 1; -} - static const char *blob_path(struct object_array_entry *entry) { return entry->path ? entry->path : entry->name; diff --git a/builtin/ls-files.c b/builtin/ls-files.c index 5c2656c43f5fe1..2d5b6bbade54e9 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -588,25 +588,17 @@ static int option_parse_exclude_standard(const struct option *opt, return 0; } -static int ls_files_has_bounded_stage_request(int argc, const char **argv) +static int ls_files_is_index_only(const struct dir_struct *dir, int show_tag) { - int i; - int stage = 0; - - for (i = 1; i < argc && strcmp(argv[i], "--"); i++) { - if (!strcmp(argv[i], "--stage") || !strcmp(argv[i], "-s")) - stage = 1; - else if (strcmp(argv[i], "-z")) - return 0; - } - if (!stage || i == argc || argc - i - 1 <= 0 || - argc - i - 1 > 64) + if (show_deleted || show_others || show_unmerged || + show_resolve_undo || show_modified || show_killed || + show_valid_bit || show_fsmonitor_bit || show_eol || + recurse_submodules || show_tag || debug_mode || + with_tree || format || exc_given || dir->exclude_per_dir || + (dir->flags & DIR_SHOW_IGNORED) || + (pathspec.magic & PATHSPEC_ATTR)) return 0; - for (i++; i < argc; i++) { - if (!*argv[i] || starts_with(argv[i], ":(") || - strpbrk(argv[i], "*?[")) - return 0; - } + return 1; } @@ -702,7 +694,10 @@ int cmd_ls_files(int argc, prefix_len = strlen(prefix); repo_config(repo, git_default_config, NULL); - scoped_bootstrap = ls_files_has_bounded_stage_request(argc, argv); + argc = parse_options(argc, argv, prefix, builtin_ls_files_options, + ls_files_usage, 0); + parse_pathspec(&pathspec, 0, PATHSPEC_PREFER_CWD, prefix, argv); + scoped_bootstrap = ls_files_is_index_only(&dir, show_tag); if (scoped_bootstrap) fsmonitor_begin_scoped_bootstrap(repo->index); if (repo_read_index(repo) < 0) @@ -710,8 +705,6 @@ int cmd_ls_files(int argc, if (scoped_bootstrap) fsmonitor_end_scoped_bootstrap(repo->index); - argc = parse_options(argc, argv, prefix, builtin_ls_files_options, - ls_files_usage, 0); pl = add_pattern_list(&dir, EXC_CMDL, "--exclude option"); for (i = 0; i < exclude_list.nr; i++) { add_pattern(exclude_list.items[i].string, "", 0, pl, --exclude_args); @@ -758,10 +751,6 @@ int cmd_ls_files(int argc, die("ls-files --recurse-submodules does not support " "--error-unmatch"); - parse_pathspec(&pathspec, 0, - PATHSPEC_PREFER_CWD, - prefix, argv); - /* * Find common prefix for all pathspec's * This is used as a performance optimization which unfortunately cannot diff --git a/diff-lib.c b/diff-lib.c index 8f8aba3038a8a6..c6175021c81f9f 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -30,6 +30,30 @@ * diff-files */ +int diff_has_bounded_regular_pathspec(const struct pathspec *pathspec) +{ + int i; + + if (pathspec->nr <= 0 || pathspec->nr > 64 || + pathspec->has_wildcard || + (pathspec->magic & + (PATHSPEC_GLOB | PATHSPEC_ICASE | + PATHSPEC_EXCLUDE | PATHSPEC_ATTR))) + return 0; + for (i = 0; i < pathspec->nr; i++) { + const struct pathspec_item *item = &pathspec->items[i]; + struct stat st; + + if (!item->match || item->len <= 0 || + item->match[item->len - 1] == '/' || + !strcmp(item->match, ".") || + has_symlink_leading_path(item->match, item->len) || + lstat(item->match, &st) || !S_ISREG(st.st_mode)) + return 0; + } + return 1; +} + /* * Has the work tree entity been removed? * diff --git a/diff.h b/diff.h index eb81289415f8f3..cc2b4e00ecc8d8 100644 --- a/diff.h +++ b/diff.h @@ -701,6 +701,7 @@ void diff_get_merge_base(const struct rev_info *revs, struct object_id *mb); /* update index stat data for content-checked entries */ #define DIFF_UPDATE_INDEX_STAT 04 void run_diff_files(struct rev_info *revs, unsigned int option); +int diff_has_bounded_regular_pathspec(const struct pathspec *pathspec); #define DIFF_INDEX_CACHED 01 #define DIFF_INDEX_MERGE_BASE 02 diff --git a/t/t7534-status-scoped-readers.sh b/t/t7534-status-scoped-readers.sh index d4451eba636c2d..5797a16674442c 100755 --- a/t/t7534-status-scoped-readers.sh +++ b/t/t7534-status-scoped-readers.sh @@ -105,6 +105,22 @@ assert_scoped_reader () { test_cmp_bin "$gitdir/checkpoint.pristine" "$scoped_checkpoint" } +assert_unscoped_reader () { + scoped_label=$1 && + shift && + cp "$gitdir/index" "$gitdir/$scoped_label.index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$scoped_label.trace" \ + git -C "$worktree" "$@" \ + >"$gitdir/$scoped_label.actual" && + ! test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <"$gitdir/$scoped_label.trace" && + test_cmp_bin "$gitdir/$scoped_label.index" "$gitdir/index" && + test_cmp_bin "$gitdir/checkpoint.pristine" "$scoped_checkpoint" +} + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ 'bounded physical-index readers reject incomplete history without a manifest' ' test_when_finished "rm -rf scoped-readers scoped-readers-linked" && @@ -165,6 +181,24 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP assert_scoped_reader worktree-default 1 \ diff --no-ext-diff --no-textconv -- tracked && test_must_be_empty "$gitdir/worktree-default.actual" && + assert_scoped_reader plumbing-files-readonly 0 \ + diff-files --no-ext-diff --no-textconv -- tracked && + assert_scoped_reader plumbing-files-default 1 \ + diff-files --no-ext-diff --no-textconv -- tracked && + assert_scoped_reader plumbing-index-readonly 0 \ + diff-index --no-ext-diff --no-textconv HEAD -- tracked && + assert_scoped_reader plumbing-index-default 1 \ + diff-index --no-ext-diff --no-textconv HEAD -- tracked && + assert_scoped_reader plumbing-cached-readonly 0 \ + diff-index --cached --name-only -z HEAD -- && + assert_scoped_reader plumbing-cached-default 1 \ + diff-index --cached --name-only -z HEAD -- && + assert_unscoped_reader plumbing-files-root \ + diff-files --no-ext-diff --no-textconv -- . && + assert_unscoped_reader plumbing-index-root \ + diff-index --no-ext-diff --no-textconv HEAD -- . && + assert_unscoped_reader plumbing-files-wildcard \ + diff-files --no-ext-diff --no-textconv -- "track*" && if test "$worktree" != "$PWD" then cp "$gitdir/index" "$gitdir/partial.saved" && @@ -189,12 +223,50 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP test_cmp_bin "$gitdir/private.index" "$gitdir/index" && assert_scoped_reader attributes 0 \ check-attr -a -- tracked sibling && + assert_scoped_reader implicit-all-attributes 0 \ + check-attr -a tracked sibling && + assert_scoped_reader named-attributes 0 \ + check-attr text tracked sibling && + assert_scoped_reader literal-pattern-attributes 0 \ + check-attr text "tracked*" && + assert_scoped_reader absent-path-attributes 0 \ + check-attr text missing && assert_scoped_reader cached-attributes 0 \ check-attr --cached -a -- tracked && + assert_scoped_reader cached-implicit-attributes 0 \ + check-attr --cached text tracked && + assert_unscoped_reader source-attributes \ + check-attr --source=HEAD -a -- tracked && assert_scoped_reader index-stage-readonly 0 \ ls-files --stage -- tracked staged && assert_scoped_reader index-stage-default 1 \ ls-files --stage -- tracked staged && + assert_scoped_reader index-stage-combined 0 \ + ls-files -sz tracked staged && + assert_scoped_reader index-stage-abbreviated 0 \ + ls-files --stage --abbrev=12 tracked staged && + assert_scoped_reader index-stage-literal 0 \ + ls-files --stage -- ":(literal)tracked" && + assert_scoped_reader index-stage-wildcard 0 \ + ls-files --stage -- "track*" && + assert_scoped_reader index-stage-root 0 \ + ls-files --stage -- . && + assert_scoped_reader index-cached 0 \ + ls-files --cached -- tracked staged && + assert_scoped_reader index-cached-all 0 \ + ls-files --cached && + assert_scoped_reader index-default-all 1 \ + ls-files && + rm "$worktree/staged" && + assert_scoped_reader index-deleted-stage 0 \ + ls-files --stage -- staged && + test_write_lines staged >"$worktree/staged" && + assert_unscoped_reader index-debug \ + ls-files --debug -- tracked && + assert_unscoped_reader index-format \ + ls-files --format="%(path)" -- tracked && + assert_unscoped_reader index-attribute-pathspec \ + ls-files -- ":(attr:text)tracked" && GIT_OPTIONAL_LOCKS=0 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$gitdir/fsmonitor-mode.trace" \ From 81eaddb26844301eb7728f8cb8b68659bcecd68d Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 11:40:31 -0500 Subject: [PATCH 48/73] exclude: tolerate sibling churn around a pinned regular source An external excludes proof pins its parent and source file, but compares the parent's complete stat data when it reopens the path. Creating an unrelated sibling changes the parent's timestamps or link count and rejects an otherwise unchanged regular file. A busy parent can therefore force another repository-wide metadata scan. For a regular source held open throughout verification, compare the parent using the existing directory-namespace identity instead. Recheck the held source and its anchored pathname after reopening the parent; the source's full identity and content checks remain unchanged. Keep the stricter parent comparison for absent and nonregular sources. In particular, an absent path appearing and disappearing between two lookups must not be mistaken for harmless sibling churn. Deterministic unit tests cover both benign regular-file races and hostile parent, target, permission, symlink, and absence transitions. --- exclude-source-proof.c | 31 ++- path-namespace.c | 12 + path-namespace.h | 2 + t/unit-tests/u-exclude-source-proof.c | 306 +++++++++++++++++++++++++- t/unit-tests/u-path-namespace.c | 46 ++++ 5 files changed, 386 insertions(+), 11 deletions(-) diff --git a/exclude-source-proof.c b/exclude-source-proof.c index 8dba2273540a4a..c46452b35c209a 100644 --- a/exclude-source-proof.c +++ b/exclude-source-proof.c @@ -113,25 +113,34 @@ static int open_source_at(int parent_fd, const char *relative, int nofollow, static int parent_identity_stable( struct exclude_source_proof *proof, const char *parent, - int held_fd, const struct stat *expected) + int held_fd, const struct stat *expected, int regular_source) { struct stat held, reopened; int fd = proof->open_parent(proof->open_data, parent); + /* + * A regular source has its own held descriptor and repeated target + * identity checks. Absence and nonregular sources cannot distinguish a + * transient target change from harmless parent-directory churn. + */ int stable = !fstat(held_fd, &held) && fd >= 0 && !fstat(fd, &reopened) && - path_namespace_stat_equal(expected, &held) && - path_namespace_stat_equal(expected, &reopened); + (regular_source ? + (path_namespace_directory_stat_equal(expected, &held) && + path_namespace_directory_stat_equal(expected, &reopened)) : + (path_namespace_stat_equal(expected, &held) && + path_namespace_stat_equal(expected, &reopened))); if (fd >= 0) close(fd); return stable; } -static int parent_stable(struct exclude_source_capture *capture) +static int parent_stable(struct exclude_source_capture *capture, + int regular_source) { return parent_identity_stable( capture->proof, capture->parent, capture->parent_fd, - &capture->parent_stat); + &capture->parent_stat, regular_source); } static void capture_free(struct exclude_source_capture *capture) @@ -334,7 +343,7 @@ void exclude_source_capture_record( if (!source_stat) { if (!exclude_source_capture_absent(capture) || - !parent_stable(capture) || + !parent_stable(capture, 0) || !exclude_source_capture_absent(capture)) { proof->invalid = 1; return; @@ -348,8 +357,10 @@ void exclude_source_capture_record( xsize_t(source_stat->st_size) != size || fstat(source_fd, &final) || !path_namespace_stat_equal(source_stat, &final) || - !source_matches(capture, &final) || - !parent_stable(capture)) { + !parent_stable(capture, S_ISREG(final.st_mode)) || + fstat(source_fd, &final) || + !path_namespace_stat_equal(source_stat, &final) || + !source_matches(capture, &final)) { proof->invalid = 1; return; } @@ -386,7 +397,7 @@ static int proof_entry_matches( goto done; if (!entry->exists) { ret = exclude_source_capture_absent(capture) && - parent_stable(capture) && + parent_stable(capture, 0) && exclude_source_capture_absent(capture); goto done; } @@ -406,7 +417,7 @@ static int proof_entry_matches( hash_object_file(proof->istate->repo->hash_algo, buf, size, OBJ_BLOB, &oid); if (!oideq(&oid, &entry->oid) || - !parent_stable(capture) || + !parent_stable(capture, S_ISREG(after.st_mode)) || fstat(fd, &final) || !path_namespace_stat_equal(&after, &final) || !source_matches(capture, &final)) diff --git a/path-namespace.c b/path-namespace.c index 533c8b53262899..3ff199ada72917 100644 --- a/path-namespace.c +++ b/path-namespace.c @@ -205,6 +205,18 @@ int path_namespace_stat_equal(const struct stat *a, const struct stat *b) return path_stat_identity_equal(&first, &second); } +int path_namespace_directory_stat_equal(const struct stat *a, + const struct stat *b) +{ + struct stat_fingerprint first, second; + + if (!S_ISDIR(a->st_mode) || !S_ISDIR(b->st_mode)) + return 0; + stat_fingerprint_init(&first, a); + stat_fingerprint_init(&second, b); + return stat_fingerprint_equal(&first, &second); +} + int path_namespace_reopen_component( int parent_fd, const char *component, int flags, path_namespace_open_fn open_fn, const struct stat *expected) diff --git a/path-namespace.h b/path-namespace.h index d702b2570aae73..23a155e3ad999c 100644 --- a/path-namespace.h +++ b/path-namespace.h @@ -28,6 +28,8 @@ void path_namespace_hash(struct git_hash_ctx *ctx, void path_namespace_hash_stat(struct git_hash_ctx *ctx, const struct stat *st); int path_namespace_stat_equal(const struct stat *a, const struct stat *b); +int path_namespace_directory_stat_equal(const struct stat *a, + const struct stat *b); int path_namespace_reopen_component( int parent_fd, const char *component, int flags, path_namespace_open_fn open_fn, const struct stat *expected); diff --git a/t/unit-tests/u-exclude-source-proof.c b/t/unit-tests/u-exclude-source-proof.c index 21e7cc33c5870c..d081549ab5fd63 100644 --- a/t/unit-tests/u-exclude-source-proof.c +++ b/t/unit-tests/u-exclude-source-proof.c @@ -16,6 +16,8 @@ static struct index_state istate = { }; static char *trash; static int fail_open_parent; +static void (*mutate_before_open_parent)(const char *parent); +static unsigned int mutate_before_open_parent_after; static int open_parent(void *data UNUSED, const char *path) { @@ -23,7 +25,14 @@ static int open_parent(void *data UNUSED, const char *path) errno = EACCES; return -1; } - return open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (mutate_before_open_parent && + !--mutate_before_open_parent_after) { + void (*mutate)(const char *) = mutate_before_open_parent; + + mutate_before_open_parent = NULL; + mutate(path); + } + return open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); } static struct exclude_source_proof *new_proof(void) @@ -40,6 +49,62 @@ static char *make_path(const char *name) return strbuf_detach(&path, NULL); } +static void create_sibling_entries(const char *parent) +{ + char *file = xstrfmt("%s/sibling", parent); + char *directory = xstrfmt("%s/sibling-directory", parent); + + write_file_buf(file, "noise", 5); + cl_must_pass(mkdir(directory, 0700)); + free(directory); + free(file); +} + +static void replace_parent_with_same_source(const char *parent) +{ + char *previous = xstrfmt("%s-old", parent); + char *source = xstrfmt("%s/source", parent); + + cl_must_pass(rename(parent, previous)); + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + free(source); + free(previous); +} + +static void replace_parent_with_symlink(const char *parent) +{ + char *previous = xstrfmt("%s-old", parent); + + cl_must_pass(rename(parent, previous)); + cl_must_pass(symlink(previous, parent)); + free(previous); +} + +static void create_and_remove_absent_source(const char *parent) +{ + char *source = xstrfmt("%s/missing", parent); + char *directory = xstrfmt("%s/sibling-directory", parent); + + write_file_buf(source, "briefly present", 15); + cl_must_pass(unlink(source)); + cl_must_pass(mkdir(directory, 0700)); + free(directory); + free(source); +} + +static void replace_source_during_parent_churn(const char *parent) +{ + char *source = xstrfmt("%s/source", parent); + char *replacement = xstrfmt("%s/replacement", parent); + + write_file_buf(replacement, "changed", 7); + cl_must_pass(rename(replacement, source)); + create_sibling_entries(parent); + free(replacement); + free(source); +} + static void record_file(struct exclude_source_proof *proof, const char *path) { struct exclude_source_capture *capture = @@ -83,6 +148,8 @@ void test_exclude_source_proof__initialize(void) char template[] = "/tmp/exclude-source-proof-XXXXXX"; fail_open_parent = 0; + mutate_before_open_parent = NULL; + mutate_before_open_parent_after = 0; cl_assert(mkdtemp(template) != NULL); trash = xstrdup(template); } @@ -117,6 +184,233 @@ void test_exclude_source_proof__accepts_same_content_replacement(void) free(parent); } +void test_exclude_source_proof__accepts_sibling_churn_during_regular_capture(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat source_stat; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &source_stat)); + create_sibling_entries(parent); + exclude_source_capture_record(capture, fd, &source_stat, "content", 7); + exclude_source_capture_release(capture); + cl_must_pass(close(fd)); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__accepts_sibling_churn_during_regular_validation(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + mutate_before_open_parent = create_sibling_entries; + mutate_before_open_parent_after = 2; + cl_assert(exclude_source_proof_validate(proof)); + cl_assert(mutate_before_open_parent == NULL); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_parent_replacement_during_regular_capture(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat source_stat; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &source_stat)); + replace_parent_with_same_source(parent); + exclude_source_capture_record(capture, fd, &source_stat, "content", 7); + exclude_source_capture_release(capture); + cl_must_pass(close(fd)); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_parent_replacement_during_regular_validation(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + mutate_before_open_parent = replace_parent_with_same_source; + mutate_before_open_parent_after = 2; + cl_assert(!exclude_source_proof_validate(proof)); + cl_assert(mutate_before_open_parent == NULL); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_symlinked_parent_during_regular_validation(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + mutate_before_open_parent = replace_parent_with_symlink; + mutate_before_open_parent_after = 2; + cl_assert(!exclude_source_proof_validate(proof)); + cl_assert(mutate_before_open_parent == NULL); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_target_replacement_during_parent_churn(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + mutate_before_open_parent = replace_source_during_parent_churn; + mutate_before_open_parent_after = 2; + cl_assert(!exclude_source_proof_validate(proof)); + cl_assert(mutate_before_open_parent == NULL); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_sibling_churn_during_absent_capture(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + char *parent = make_path("parent"); + char *source = make_path("parent/missing"); + + cl_must_pass(mkdir(parent, 0700)); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + cl_assert(exclude_source_capture_absent(capture)); + create_sibling_entries(parent); + exclude_source_capture_record(capture, -1, NULL, NULL, 0); + exclude_source_capture_release(capture); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_sibling_churn_during_fifo_capture(void) +{ + struct exclude_source_proof *proof = + exclude_source_proof_create(&istate, NULL, open_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); + struct exclude_source_capture *capture; + struct stat source_stat; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkfifo(source, 0600)); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &source_stat)); + cl_assert(S_ISFIFO(source_stat.st_mode)); + create_sibling_entries(parent); + exclude_source_capture_record(capture, fd, &source_stat, NULL, 0); + exclude_source_capture_release(capture); + cl_must_pass(close(fd)); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_transient_absent_source_during_validation(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/missing"); + + cl_must_pass(mkdir(parent, 0700)); + record_absence(proof, source); + mutate_before_open_parent = create_and_remove_absent_source; + mutate_before_open_parent_after = 2; + cl_assert(!exclude_source_proof_validate(proof)); + cl_assert(mutate_before_open_parent == NULL); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_parent_permission_change_during_capture(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat source_stat; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &source_stat)); + cl_must_pass(chmod(parent, 0500)); + exclude_source_capture_record(capture, fd, &source_stat, "content", 7); + exclude_source_capture_release(capture); + cl_must_pass(close(fd)); + cl_must_pass(chmod(parent, 0700)); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + void test_exclude_source_proof__rejects_different_content_replacement(void) { struct exclude_source_proof *proof = new_proof(); @@ -485,6 +779,16 @@ void test_exclude_source_proof__captures_fifo_without_blocking(void) EMPTY_TEST(test_exclude_source_proof__initialize) EMPTY_TEST(test_exclude_source_proof__cleanup) SKIP_TEST(test_exclude_source_proof__accepts_same_content_replacement) +SKIP_TEST(test_exclude_source_proof__accepts_sibling_churn_during_regular_capture) +SKIP_TEST(test_exclude_source_proof__accepts_sibling_churn_during_regular_validation) +SKIP_TEST(test_exclude_source_proof__rejects_parent_replacement_during_regular_capture) +SKIP_TEST(test_exclude_source_proof__rejects_parent_replacement_during_regular_validation) +SKIP_TEST(test_exclude_source_proof__rejects_symlinked_parent_during_regular_validation) +SKIP_TEST(test_exclude_source_proof__rejects_target_replacement_during_parent_churn) +SKIP_TEST(test_exclude_source_proof__rejects_sibling_churn_during_absent_capture) +SKIP_TEST(test_exclude_source_proof__rejects_sibling_churn_during_fifo_capture) +SKIP_TEST(test_exclude_source_proof__rejects_transient_absent_source_during_validation) +SKIP_TEST(test_exclude_source_proof__rejects_parent_permission_change_during_capture) SKIP_TEST(test_exclude_source_proof__rejects_different_content_replacement) SKIP_TEST(test_exclude_source_proof__accepts_repeated_observation) SKIP_TEST(test_exclude_source_proof__rejects_missing_source_buffer) diff --git a/t/unit-tests/u-path-namespace.c b/t/unit-tests/u-path-namespace.c index 3c80140a8bbf72..b3735df2c4a4a1 100644 --- a/t/unit-tests/u-path-namespace.c +++ b/t/unit-tests/u-path-namespace.c @@ -73,6 +73,52 @@ void test_path_namespace__stat_fields(void) #endif } +void test_path_namespace__directory_identity_ignores_unrelated_entries(void) +{ + struct stat original, changed; + + cl_must_pass(stat(".", &original)); + cl_assert(S_ISDIR(original.st_mode)); + cl_assert(path_namespace_directory_stat_equal(&original, &original)); + + changed = original; + changed.st_nlink++; + changed.st_size++; + changed.st_mtime++; + changed.st_ctime++; + cl_assert(!path_namespace_stat_equal(&original, &changed)); + cl_assert(path_namespace_directory_stat_equal(&original, &changed)); + + changed = original; + changed.st_dev++; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); + changed = original; + changed.st_ino++; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); + changed = original; + changed.st_mode ^= S_IXGRP; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); + changed = original; + changed.st_uid++; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); + changed = original; + changed.st_gid++; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); +#ifdef __APPLE__ + changed = original; + changed.st_birthtimespec.tv_sec++; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); + changed = original; + changed.st_gen++; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); +#endif + + changed = original; + changed.st_mode = S_IFREG | 0600; + cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); + cl_assert(!path_namespace_directory_stat_equal(&changed, &changed)); +} + static int source_fd = -1; static int reopen_source(int dirfd UNUSED, const char *path, int flags UNUSED) From 40798d47ec36593cd114ce7b8874ab242f5bfb74 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 11:43:55 -0500 Subject: [PATCH 49/73] fsmonitor: release raw index bitmaps with their owner aaa01b4ec1 (clean-status: release the raw fsmonitor shadow bitmap, 2026-08-17) releases the bitmap owned by one raw index reader. The two external-history witness readers and split-index bases can own the same kind of bitmap, but release_index() does not free it. Repeated bitmap generation can also overwrite an allocation left by an earlier write. Make release_index() release and clear the owned bitmap, and replace an existing bitmap safely when generating the next one. Existing transfers already clear the donor, and callers which consume the bitmap clear it after freeing it. Extend the FSMN parser test with raw-read/release and repeated-fill ownership checks. The old cleanup fails the focused test; the corrected cleanup passes it under both object formats. --- fsmonitor.c | 5 ++++- read-cache.c | 3 +++ t/helper/test-read-cache.c | 27 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/fsmonitor.c b/fsmonitor.c index b1c07929d98581..d075305a8fa1b6 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -374,7 +374,10 @@ static struct ewah_bitmap *fsmonitor_bitmap_from_index( void fill_fsmonitor_bitmap(struct index_state *istate) { - istate->fsmonitor_dirty = fsmonitor_bitmap_from_index(istate); + struct ewah_bitmap *bitmap = fsmonitor_bitmap_from_index(istate); + + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_dirty = bitmap; } static void serialize_fsmonitor_extension(struct strbuf *sb, diff --git a/read-cache.c b/read-cache.c index cf13875574f23d..bf0866a6e5540f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -26,6 +26,7 @@ #include "tree.h" #include "commit.h" #include "environment.h" +#include "ewah/ewok.h" #include "gettext.h" #include "mem-pool.h" #include "name-hash.h" @@ -2754,6 +2755,8 @@ void release_index(struct index_state *istate) free(istate->fsmonitor_last_update); free(istate->fsmonitor_last_update_pending); free(istate->fsmonitor_untracked_token); + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_dirty = NULL; clean_status_release(istate); free(istate->preload_bulk_tracked_state); free(istate->preload_bulk_stat_updates); diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 3009e38d3b0bb6..2f8ce51bf39a2f 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -191,6 +191,31 @@ static int check_invalid_fsmn(const struct strbuf *encoded, return 0; } +static int test_fsmn_bitmap_ownership(const struct strbuf *encoded) +{ + struct index_state parsed = INDEX_STATE_INIT(the_repository); + struct index_state regenerated = INDEX_STATE_INIT(the_repository); + + parsed.cache_nr = 1; + read_fsmonitor_extension(&parsed, encoded->buf, encoded->len); + if (!parsed.fsmonitor_token_valid || !parsed.fsmonitor_dirty) + return error("raw FSMN bitmap was not published"); + parsed.cache_nr = 0; + release_index(&parsed); + + fill_fsmonitor_bitmap(®enerated); + if (!regenerated.fsmonitor_dirty) + return error("initial FSMN bitmap was not published"); + fill_fsmonitor_bitmap(®enerated); + if (!regenerated.fsmonitor_dirty) + return error("regenerated FSMN bitmap did not replace its owner"); + release_index(®enerated); + + if (parsed.fsmonitor_dirty || regenerated.fsmonitor_dirty) + return error("released index retained its FSMN bitmap"); + return 0; +} + static int test_fsmn_parser(void) { struct index_state duplicate = INDEX_STATE_INIT(the_repository); @@ -244,6 +269,8 @@ static int test_fsmn_parser(void) make_raw_fsmn(&malformed, 1, words, 2, 1); if (check_invalid_fsmn(&malformed, "non-final RLW")) return 1; + if (test_fsmn_bitmap_ownership(&encoded)) + return 1; strbuf_release(&malformed); strbuf_release(&encoded); From 0a5f399b17c11e3bdfad29025158250c4b9a9a6e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 12:35:06 -0500 Subject: [PATCH 50/73] status: carry a failed sidecar probe into the index refresh 61f7705f80 (status: answer exact clean status before index deserialization, 2026-07-27) lets a clean sidecar answer status before reading the index. When its provider query reports a reset or an error, we fall back to reading the index, which makes another provider query. An empty second response must not revive the boundary rejected by the first one. Otherwise stale tracked and untracked proofs can hide a changed file or newly activated attributes. Carry that lost boundary to cmd_status(). After reading the index, discard the semantic manifest and strongly invalidate both tracked and untracked state before collecting or publishing status. Preserve an already authenticated stat baseline only when the reader also obtained its closing token from the provider. This avoids rebuilding the same manifest after two resets without trusting a later empty delta. Ordinary nonempty deltas retain their existing scoped handling. Exercise the reset-then-empty and error-then-empty sequences with a filter-induced modification and an independent strong-stat oracle. Also cover clean hits, consecutive resets, a second provider error, ordinary deltas, read-only index preservation, and writable status followed by another read-only query. --- builtin/commit.c | 23 +++++- clean-status-fast.c | 15 ++-- clean-status.h | 2 +- t/t7530-status-clean-sidecar.sh | 119 ++++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 6 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 1de778261812d5..e44f0f76ca81f6 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -23,6 +23,7 @@ #include "environment.h" #include "diff.h" #include "commit.h" +#include "fsmonitor.h" #include "fsmonitor-settings.h" #include "add-interactive.h" #include "gettext.h" @@ -1882,6 +1883,7 @@ struct repository *repo UNUSED) int normal_has_head; int reissue_clean_sidecar = 0; int repository_inputs_changed = 0; + int sidecar_provider_reset = 0; int reissue_after_write = 0; int save_history_after_write = 0; int deferred_scoped_history = 0; @@ -2003,7 +2005,8 @@ struct repository *repo UNUSED) s.certify_clean_status = exact_clean_query; if (reusable_clean_query && clean_status_try_sidecar(the_repository, &clean_digest, - &repository_inputs_changed)) { + &repository_inputs_changed, + &sidecar_provider_reset)) { if (exact_clean_query || print_clean_sidecar(&s, prefix)) { wt_status_collect_free_buffers(&s); @@ -2025,6 +2028,24 @@ struct repository *repo UNUSED) if (use_optional_locks()) clean_status_require_external_history_source(the_repository); repo_read_index(the_repository); + if (sidecar_provider_reset) { + /* + * The fast probe already lost the old provider boundary. Even + * if the index reader's second query is empty, it must not + * revive the tracked or untracked proof we just rejected. A + * provider-owned, authenticated stat baseline has already + * invalidated that proof and must retain its closing token. + */ + if (!clean_status_fsmonitor_semantic_baseline_pending( + the_repository->index) || + !fsmonitor_pending_token_from_provider(the_repository->index)) { + clean_status_invalidate_current_manifest(the_repository->index); + fsmonitor_invalidate_semantics(the_repository->index); + untracked_cache_invalidate_all(the_repository->index); + } + trace2_data_intmax("status", the_repository, + "clean-proof/provider-reset-carried", 1); + } if (use_optional_locks()) { deferred_scoped_history = clean_status_defer_scoped_history_capture( diff --git a/clean-status-fast.c b/clean-status-fast.c index e0e2661b2be726..53f05ce1df41a8 100644 --- a/clean-status-fast.c +++ b/clean-status-fast.c @@ -22,9 +22,10 @@ int clean_status_try_sidecar( struct repository *repo UNUSED, const struct clean_status_config_digest *config UNUSED, - int *repository_inputs_changed) + int *repository_inputs_changed, int *provider_reset) { *repository_inputs_changed = 0; + *provider_reset = 0; return 0; } @@ -199,7 +200,7 @@ static int current_worktree_is_main(struct repository *repo) int clean_status_try_sidecar( struct repository *repo, const struct clean_status_config_digest *config, - int *repository_inputs_changed) + int *repository_inputs_changed, int *provider_reset) { struct clean_status_sidecar_record record = CLEAN_STATUS_SIDECAR_RECORD_INIT; @@ -216,6 +217,7 @@ int clean_status_try_sidecar( int ret = 0; *repository_inputs_changed = 0; + *provider_reset = 0; if (!config->finalized || (config->filter_configured && config->normalized_filter_disable) || getenv(INDEX_ENVIRONMENT) || is_bare_repository(repo) || @@ -281,8 +283,13 @@ int clean_status_try_sidecar( query_token = xmemdupz( record.sidecar.token, record.sidecar.token_len); if (query_builtin_fsmonitor(query_token, &query) != - FSMONITOR_QUERY_DELTA || - query.paths.len) { + FSMONITOR_QUERY_DELTA) { + /* A later successful query cannot erase this lost boundary. */ + *provider_reset = 1; + trace_miss(repo, "fast-provider-changed"); + goto done; + } + if (query.paths.len) { trace_miss(repo, "fast-provider-changed"); goto done; } diff --git a/clean-status.h b/clean-status.h index 5a9698a8f9accc..24f07807d2ee99 100644 --- a/clean-status.h +++ b/clean-status.h @@ -110,7 +110,7 @@ int clean_status_issue_sidecar( int clean_status_try_sidecar( struct repository *repo, const struct clean_status_config_digest *config, - int *repository_inputs_changed); + int *repository_inputs_changed, int *provider_reset); int clean_status_read_fsmonitor_config(struct index_state *istate, const void *data, unsigned long size); diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 6de465a64bda35..4fb103c2dcff72 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -2612,4 +2612,123 @@ test_expect_success DURABLE_FSMONITOR \ done ' +test_expect_success PERL_TEST_HELPERS \ + 'a trivial fast probe survives a later empty provider delta' ' + test_when_finished "rm -rf sidecar-trivial-probe" && + test_create_repo sidecar-trivial-probe && + ( + cd sidecar-trivial-probe && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test-tool chmtime -120 tracked && + git update-index --refresh && + git config core.autocrlf false && + git config core.untrackedCache true && + git config filter.sidecar.clean "sed s/base/converted/" && + git config filter.sidecar.required true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --fsmonitor && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/issue && + test_must_be_empty .git/issue && + test_path_is_file .git/index.csts && + cp .git/index .git/index.before && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status --porcelain=v2 >.git/clean && + test_must_be_empty .git/clean && + test_trace2_data status clean-proof/hit 1 <.git/clean.trace && + test_region ! index do_read_index .git/clean.trace && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TTCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/clean-reset.trace" \ + git status --porcelain=v2 >.git/clean-reset && + test_must_be_empty .git/clean-reset && + test_trace2_data status clean-proof/provider-reset-carried 1 \ + <.git/clean-reset.trace && + test_trace2_data fsmonitor semantic/token-reset-stat-baseline 1 \ + <.git/clean-reset.trace && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <.git/clean-reset.trace >.git/clean-reset.scans && + test_line_count = 1 .git/clean-reset.scans && + ! test_trace2_data fsmonitor semantic/manifest-scan-count 2 \ + <.git/clean-reset.trace && + test_cmp_bin .git/index.before .git/index && + test_write_lines "tracked filter=sidecar" >.gitattributes && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false -c core.untrackedCache=false \ + -c core.trustctime=true -c core.checkStat=default \ + status --porcelain=v2 >.git/expect && + test_grep "^1 \\.M .* tracked$" .git/expect && + test_grep "^? \\.gitattributes$" .git/expect && + for outcome in T E TT TE + do + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE="$outcome"CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/$outcome.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_trace2_data status clean-proof/miss fast-provider-changed \ + <".git/$outcome.trace" && + test_trace2_data status clean-proof/provider-reset-carried 1 \ + <".git/$outcome.trace" && + test_grep ! "\"key\":\"clean-proof/hit\"" \ + ".git/$outcome.trace" && + test_cmp_bin .git/index.before .git/index && + case "$outcome" in + TT) + test_trace2_data fsmonitor \ + semantic/token-reset-stat-baseline 1 \ + <.git/TT.trace && + test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <.git/TT.trace >.git/TT.scans && + test_line_count = 1 .git/TT.scans && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 2 \ + <.git/TT.trace + ;; + TE) + ! test_trace2_data fsmonitor \ + semantic/token-reset-stat-baseline 1 \ + <.git/TE.trace && + test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <.git/TE.trace && + test_trace2_data fsmonitor \ + semantic/manifest-scan-count 2 \ + <.git/TE.trace + ;; + esac || return 1 + done && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$PWD/.git/delta.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + ! test_trace2_data status clean-proof/provider-reset-carried 1 \ + <.git/delta.trace && + test_cmp_bin .git/index.before .git/index && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/writable.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_trace2_data status clean-proof/provider-reset-carried 1 \ + <.git/writable.trace && + ! test_trace2_data status clean-proof/sidecar 1 \ + <.git/writable.trace && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/follower && + test_cmp .git/expect .git/follower + ) +' + test_done From 2444e3525032a654f9e718f661b24d459ffac515 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 12:36:03 -0500 Subject: [PATCH 51/73] fsmonitor: retire the provider token after a failed cookie 56cef9cb1a (fsmonitor: use pthread_cond_timedwait for cookie wait, 2026-04-15) bounds the wait for a filesystem event. A timeout or cookie creation error means the daemon cannot prove that it has drained the event queue. Returning a trivial response with the same token leaves that token usable by another request. A later successful cookie can therefore make an already rejected boundary look complete. Force the existing resynchronization path while holding the main lock when a cookie returns FCIR_ERROR. This rotates the token and aborts other waiters before replying. Do not resynchronize again for FCIR_ABORT: that waiter already belongs to a listener-initiated reset. Force cookie creation to fail by temporarily removing its directory. Require a different token in the trivial response and reject the old token after restoring the directory. Check successful replay separately when the filesystem actually delivers the recovery cookie. --- builtin/fsmonitor--daemon.c | 8 +++++ t/meson.build | 1 + t/t7535-fsmonitor-cookie-reset.sh | 56 +++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100755 t/t7535-fsmonitor-cookie-reset.sh diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index d96d7e8fb9d0e5..b52421f672c930 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -851,6 +851,14 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, error(_("fsmonitor: cookie_result '%d' != SEEN"), cookie_result); do_trivial = 1; + /* + * This boundary could not be synchronized. Retire it so + * a later successful cookie cannot make an old client's + * token appear complete again. An aborted cookie already + * belongs to a listener-initiated reset. + */ + if (cookie_result == FCIR_ERROR) + do_flush = 1; } } diff --git a/t/meson.build b/t/meson.build index e486e988bc9aaa..d97af43eda3ac1 100644 --- a/t/meson.build +++ b/t/meson.build @@ -967,6 +967,7 @@ integration_tests = [ 't7532-preload-index-linux.sh', 't7533-status-scoped-stash.sh', 't7534-status-scoped-readers.sh', + 't7535-fsmonitor-cookie-reset.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t7535-fsmonitor-cookie-reset.sh b/t/t7535-fsmonitor-cookie-reset.sh new file mode 100755 index 00000000000000..d83013928c80a2 --- /dev/null +++ b/t/t7535-fsmonitor-cookie-reset.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +test_description='failed fsmonitor cookies retire the provider boundary' + +. ./test-lib.sh + +if ! test_have_prereq FSMONITOR_DAEMON +then + skip_all='fsmonitor--daemon is not supported on this platform' + test_done +fi + +test_expect_success 'a failed cookie permanently invalidates the old token' ' + test_when_finished "git -C cookie-reset fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo cookie-reset && + GIT_TRACE2_EVENT="$PWD/cookie-daemon.trace" \ + git -C cookie-reset fsmonitor--daemon start --start-timeout=10 && + mv cookie-reset/.git/fsmonitor--daemon/cookies \ + cookie-reset/.git/fsmonitor--daemon/cookies.saved && + test_when_finished "test ! -d cookie-reset/.git/fsmonitor--daemon/cookies.saved || + mv cookie-reset/.git/fsmonitor--daemon/cookies.saved \ + cookie-reset/.git/fsmonitor--daemon/cookies" && + test-tool -C cookie-reset fsmonitor-client flush >before && + nul_to_q before.q && + test_grep "^builtin:.*:0Q/Q$" before.q && + old_token=$(sed "s/Q.*//" before.q) && + test-tool -C cookie-reset fsmonitor-client query \ + --token "$old_token" >failed && + nul_to_q failed.q && + test_grep "^builtin:.*:0Q/Q$" failed.q && + new_token=$(sed "s/Q.*//" failed.q) && + test "$old_token" != "$new_token" && + mv cookie-reset/.git/fsmonitor--daemon/cookies.saved \ + cookie-reset/.git/fsmonitor--daemon/cookies && + test-tool -C cookie-reset fsmonitor-client query \ + --token "$old_token" >recovered && + nul_to_q recovered.q && + test_grep "^builtin:.*Q/Q$" recovered.q && + recovered_token=$(sed "s/Q.*//" recovered.q) && + test "$recovered_token" != "$old_token" && + if test "$recovered_token" = "$new_token" && + test_trace2_data fsmonitor response/token different \ + Date: Mon, 17 Aug 2026 12:36:49 -0500 Subject: [PATCH 52/73] fsmonitor: encode Darwin worktree-root events without overreading 65723b305a (compat/fsmonitor/fsm-listen-darwin: implement FSEvent listener on MacOS, 2022-03-25) converts absolute events to worktree-relative paths. The classifier accepts the watched root itself, but the listener skips the root and a presumed slash. That reads past the terminating NUL for an exact-root event and can queue arbitrary trailing bytes or the reserved trivial-response marker. Handle the root before advancing to a relative pathname. Encode it as the existing global-invalidation path, leaving ordinary file, directory, and merged file/directory events unchanged. Keep alias resolution and metadata-only event filtering at their existing boundaries. Exercise the formatter through the builtin response parser, including stale bytes after a root's NUL. A root event must remain a global DELTA, not a TRIVIAL response. --- compat/fsmonitor/fsm-listen-darwin.c | 31 ++++++++----------- fsmonitor.c | 29 ++++++++++++++++++ fsmonitor.h | 9 ++++++ t/unit-tests/u-fsmonitor-response.c | 45 ++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 19 deletions(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 57f27faefa3eb2..f25d7cdd907af9 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -24,7 +24,7 @@ #endif #include "git-compat-util.h" -#include "fsmonitor-ll.h" +#include "fsmonitor.h" #include "fsm-listen.h" #include "fsmonitor--daemon.h" #include "fsmonitor-path-utils.h" @@ -420,26 +420,19 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, * know how much to invalidate/refresh. */ - if (event_flags[k] & (kFSEventStreamEventFlagItemIsFile | kFSEventStreamEventFlagItemIsSymlink)) { - const char *rel = path_k + - state->path_worktree_watch.len + 1; - + fsmonitor_format_worktree_paths( + &tmp, path_k, state->path_worktree_watch.len, + !!(event_flags[k] & + (kFSEventStreamEventFlagItemIsFile | + kFSEventStreamEventFlagItemIsSymlink)), + !!(event_flags[k] & + kFSEventStreamEventFlagItemIsDir)); + for (const char *relative = tmp.buf; + relative < tmp.buf + tmp.len; + relative += strlen(relative) + 1) { if (!batch) batch = fsmonitor_batch__new(); - my_add_path(batch, rel); - } - - if (event_flags[k] & kFSEventStreamEventFlagItemIsDir) { - const char *rel = path_k + - state->path_worktree_watch.len + 1; - - strbuf_reset(&tmp); - strbuf_addstr(&tmp, rel); - strbuf_addch(&tmp, '/'); - - if (!batch) - batch = fsmonitor_batch__new(); - my_add_path(batch, tmp.buf); + my_add_path(batch, relative); } break; diff --git a/fsmonitor.c b/fsmonitor.c index d075305a8fa1b6..a8dbf21b17a6f9 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -896,6 +896,35 @@ void fsmonitor_query_result_release(struct fsmonitor_query_result *result) strbuf_release(&result->paths); } +void fsmonitor_format_worktree_paths( + struct strbuf *paths, const char *path, size_t worktree_len, + int is_file, int is_directory) +{ + const char *relative = path + worktree_len; + + strbuf_reset(paths); + if (!is_file && !is_directory) + return; + + /* The root has no relative pathname; never read past its NUL. */ + if (!*relative || (*relative == '/' && !relative[1])) { + strbuf_addstr(paths, FSMONITOR_PATH_GLOBAL_INVALIDATE); + strbuf_addch(paths, '\0'); + return; + } + + relative++; + if (is_file) { + strbuf_addstr(paths, relative); + strbuf_addch(paths, '\0'); + } + if (is_directory) { + strbuf_addstr(paths, relative); + strbuf_addch(paths, '/'); + strbuf_addch(paths, '\0'); + } +} + static int fsmonitor_valid_worktree_path(const char *path, size_t len) { struct strbuf copy = STRBUF_INIT; diff --git a/fsmonitor.h b/fsmonitor.h index be8136767a9bbe..7ba51d6bd05961 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -36,6 +36,15 @@ struct fsmonitor_query_result { } void fsmonitor_query_result_release(struct fsmonitor_query_result *result); + +/* + * Encode an already classified and alias-resolved worktree event. The caller + * must have verified that worktree_len names the path's worktree prefix. + */ +void fsmonitor_format_worktree_paths( + struct strbuf *paths, const char *path, size_t worktree_len, + int is_file, int is_directory); + enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( const struct strbuf *raw, struct fsmonitor_query_result *result); enum fsmonitor_query_outcome query_builtin_fsmonitor( diff --git a/t/unit-tests/u-fsmonitor-response.c b/t/unit-tests/u-fsmonitor-response.c index 770b8ecea17487..1b024d72f6dea5 100644 --- a/t/unit-tests/u-fsmonitor-response.c +++ b/t/unit-tests/u-fsmonitor-response.c @@ -27,6 +27,29 @@ static void check_malformed(const void *data, size_t len) check_response(data, len, FSMONITOR_QUERY_ERROR, "", NULL, 0); } +static void check_worktree_event( + const char *path, size_t worktree_len, + int is_file, int is_directory, + const void *expected, size_t expected_len) +{ + struct strbuf paths = STRBUF_INIT; + struct strbuf response = STRBUF_INIT; + + fsmonitor_format_worktree_paths( + &paths, path, worktree_len, is_file, is_directory); + cl_assert_equal_i(paths.len, expected_len); + cl_assert(!expected_len || !memcmp(paths.buf, expected, expected_len)); + + strbuf_addstr(&response, "builtin:worktree"); + strbuf_addch(&response, '\0'); + strbuf_addbuf(&response, &paths); + check_response(response.buf, response.len, FSMONITOR_QUERY_DELTA, + "builtin:worktree", expected, expected_len); + + strbuf_release(&response); + strbuf_release(&paths); +} + void test_fsmonitor_response__rejects_malformed_framing(void) { static const char missing_nul[] = "builtin:1"; @@ -74,6 +97,12 @@ void test_fsmonitor_response__accepts_valid_builtin_responses(void) static const char delta[] = "builtin:2\0a\0dir/file\0dir/\0"; static const char global[] = "builtin:3\0//\0"; static const char trivial[] = "builtin:4\0/\0"; + static const char stale_root[] = "/repo\0stale/path"; + static const char global_path[] = "//\0"; + static const char file_path[] = "tracked\0"; + static const char directory_path[] = "nested/\0"; + static const char both_paths[] = "merged\0merged/\0"; + static const char case_path[] = "Tracked\0"; check_response(delta, sizeof(delta) - 1, FSMONITOR_QUERY_DELTA, "builtin:2", delta + sizeof("builtin:2"), @@ -83,6 +112,22 @@ void test_fsmonitor_response__accepts_valid_builtin_responses(void) sizeof(global) - 1 - sizeof("builtin:3")); check_response(trivial, sizeof(trivial) - 1, FSMONITOR_QUERY_TRIVIAL, "builtin:4", NULL, 0); + + check_worktree_event(stale_root, strlen("/repo"), 0, 1, + global_path, sizeof(global_path) - 1); + check_worktree_event("/repo/", strlen("/repo"), 0, 1, + global_path, sizeof(global_path) - 1); + check_worktree_event("/repo", strlen("/repo"), 1, 1, + global_path, sizeof(global_path) - 1); + check_worktree_event("/repo/tracked", strlen("/repo"), 1, 0, + file_path, sizeof(file_path) - 1); + check_worktree_event("/repo/nested", strlen("/repo"), 0, 1, + directory_path, sizeof(directory_path) - 1); + check_worktree_event("/repo/merged", strlen("/repo"), 1, 1, + both_paths, sizeof(both_paths) - 1); + check_worktree_event("/REPO/Tracked", strlen("/repo"), 1, 0, + case_path, sizeof(case_path) - 1); + check_worktree_event("/repo", strlen("/repo"), 0, 0, NULL, 0); } void test_fsmonitor_response__validates_hardlink_inode_markers(void) From ae6012c6352a0caa573141d33a1f0e05996924e0 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 12:39:24 -0500 Subject: [PATCH 53/73] fsmonitor: open watch-limit backoff markers without blocking 60077b92ac (status: preserve semantic history across scoped and index changes, 2026-08-11) records a temporary watch-limit failure in the git directory. The reader checks that this optional marker is a small, private regular file, but opens it before checking its type. A FIFO with no writer therefore blocks every command which reads fsmonitor settings, instead of being rejected as an invalid marker. Open the marker with O_NONBLOCK. Keep the existing no-follow, owner, link-count, mode, size, age, worktree-identity, and watch-limit checks. The marker implementation is already limited to Linux and macOS. Cover a no-writer FIFO with a bounded alarm, and retain oversized and malformed-marker controls. Compare status against an independent strong-stat oracle while the scripted provider reports the real changes. --- fsmonitor-ipc.c | 2 +- t/meson.build | 1 + t/t7536-fsmonitor-watch-limit-backoff.sh | 95 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100755 t/t7536-fsmonitor-watch-limit-backoff.sh diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 867a0c975f3dc3..799db433cf86f6 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -230,7 +230,7 @@ int fsmonitor_ipc__watch_limit_backoff(struct repository *r) if (!watch_limit_backoff_enabled()) return 0; path = repo_git_path(r, FSMONITOR_WATCH_LIMIT_MARKER); - fd = open(path, O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + fd = open(path, O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK); if (fd < 0) goto done; if (fstat(fd, &st) || !S_ISREG(st.st_mode) || diff --git a/t/meson.build b/t/meson.build index d97af43eda3ac1..7b10e2b568475d 100644 --- a/t/meson.build +++ b/t/meson.build @@ -968,6 +968,7 @@ integration_tests = [ 't7533-status-scoped-stash.sh', 't7534-status-scoped-readers.sh', 't7535-fsmonitor-cookie-reset.sh', + 't7536-fsmonitor-watch-limit-backoff.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t7536-fsmonitor-watch-limit-backoff.sh b/t/t7536-fsmonitor-watch-limit-backoff.sh new file mode 100755 index 00000000000000..c80a4002809a61 --- /dev/null +++ b/t/t7536-fsmonitor-watch-limit-backoff.sh @@ -0,0 +1,95 @@ +#!/bin/sh + +test_description='fsmonitor watch-limit backoff authenticates optional markers' + +. ./test-lib.sh + +if ! test_have_prereq FSMONITOR_DAEMON +then + skip_all='fsmonitor--daemon is not supported on this platform' + test_done +fi + +case "$uname_s" in +Linux | Darwin) + ;; +*) + skip_all='inotify watch-limit markers are not supported on this platform' + test_done + ;; +esac + +sane_unset GIT_TEST_SPLIT_INDEX GIT_TEST_FSMONITOR + +setup_backoff_marker_fixture () { + test_create_repo "$1" && + ( + cd "$1" && + test_commit base tracked && + git config core.preloadIndex false && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_write_lines changed >tracked && + test_write_lines visible >visible && + git -c core.fsmonitor=false -c core.untrackedCache=false \ + --no-optional-locks status --porcelain=v2 >.git/expect && + test_grep "^1 \\.M .* tracked$" .git/expect && + test_grep "^? visible$" .git/expect + ) +} + +check_rejected_backoff_marker () { + ( + cd "$1" && + GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=// \ + GIT_TRACE2_EVENT="$PWD/.git/$2.trace" \ + perl -e "alarm 5; exec @ARGV" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + ! test_trace2_data fsm_client \ + settings/inotify-watch-limit-backoff 1 \ + <".git/$2.trace" && + test_grep ! \ + "\"event\":\"child_start\".*\"fsmonitor--daemon\"" \ + ".git/$2.trace" + ) +} + +test_expect_success PIPE,PERL_TEST_HELPERS \ + 'a FIFO watch-limit marker never blocks ordinary status' ' + setup_backoff_marker_fixture marker-fifo && + marker=marker-fifo/.git/fsmonitor--daemon.inotify-limit && + mkfifo "$marker" && + test_when_finished "rm -f $marker" && + check_rejected_backoff_marker marker-fifo fifo && + test -p "$marker" +' + +test_expect_success PERL_TEST_HELPERS \ + 'an oversized watch-limit marker cannot disable fsmonitor' ' + setup_backoff_marker_fixture marker-oversized && + marker=marker-oversized/.git/fsmonitor--daemon.inotify-limit && + printf "%0257d\\n" 0 >"$marker" && + chmod 600 "$marker" && + test "$(wc -c <"$marker")" -gt 256 && + check_rejected_backoff_marker marker-oversized oversized && + test_path_is_file "$marker" +' + +test_expect_success PERL_TEST_HELPERS \ + 'a malformed watch-limit marker cannot disable fsmonitor' ' + setup_backoff_marker_fixture marker-malformed && + marker=marker-malformed/.git/fsmonitor--daemon.inotify-limit && + printf "inotify-limit-v1\\ninvalid-identity\\nnot-a-limit\\n" \ + >"$marker" && + chmod 600 "$marker" && + check_rejected_backoff_marker marker-malformed malformed && + test_path_is_file "$marker" +' + +test_done From 0e380d274c8169d4f7696aa076c83c98393b682d Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 12:41:39 -0500 Subject: [PATCH 54/73] fsmonitor: reuse authenticated manifests for wider directory deltas 7c19bf467b (fsmonitor: retain checked manifests across scoped directory deltas, 2026-08-15) bounds the local attribute check to 64 candidate paths. A closing event for a directory with 64 child directories crosses that limit even when no attributes changed, invalidating the proof and building the whole-worktree manifest a second time. Collect each indexed directory once, sort the candidate paths, and walk them together with one authenticated manifest cursor. This removes the fixed cutoff, repeated sorted insertions, and repeated manifest searches. Keep the existing descriptor-anchored source checks, negative lookups, and before/after repository, index, attribute, and provider fences. Require the existing high-fanout case to finish with one manifest scan and reuse its checked subtree. Also race an attribute creation in the last child directory and require the ordinary fail-closed result. --- clean-status-manifest.c | 87 +++++++++++++++++++++++++++---------- t/t7519-status-fsmonitor.sh | 56 +++++++++++++++++++----- 2 files changed, 108 insertions(+), 35 deletions(-) diff --git a/clean-status-manifest.c b/clean-status-manifest.c index fe881e44e38edd..9f1628a6840e30 100644 --- a/clean-status-manifest.c +++ b/clean-status-manifest.c @@ -180,18 +180,29 @@ int clean_status_manifest_end_directory_delta(struct index_state *istate) } #if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int directory_manifest_entry_path_compare( + const struct attr_manifest_entry *entry, const char *name) +{ + size_t len = strlen(name); + size_t common = entry->path_len < len ? entry->path_len : len; + int cmp = memcmp(entry->path, name, common); + + if (cmp) + return cmp; + return entry->path_len < len ? -1 : entry->path_len > len; +} + static int directory_attribute_source_matches( struct index_state *istate, struct semantic_verify_path *path, - const char *name, size_t position) + const char *name, const struct attr_manifest_entry *entry, + size_t position) { - struct clean_status_state *state = istate->clean_status; - struct attr_manifest_entry entry; const struct cache_entry *indexed; const struct git_hash_algo *algo = istate->repo->hash_algo; const char *basename; unsigned char observed[GIT_MAX_RAWSZ]; struct stat st; - int parent_fd, found, present, pos; + int parent_fd, found, pos; if (semantic_verify_resolve_parent(path, name, position, &parent_fd, &basename)) @@ -205,24 +216,22 @@ static int directory_attribute_source_matches( if (worktree_attr_source_read(path, name, position, algo, observed, &found)) return 0; - present = !find_manifest_entry(&state->manifest.current, - name, algo, &entry); if (found) - return present && entry.source == ATTR_MANIFEST_WORKTREE && - !memcmp(entry.hash, observed, algo->rawsz); + return entry && entry->source == ATTR_MANIFEST_WORKTREE && + !memcmp(entry->hash, observed, algo->rawsz); if (!fstatat(parent_fd, basename, &st, AT_SYMLINK_NOFOLLOW) || errno != ENOENT) return 0; pos = index_name_pos(istate, name, strlen(name)); if (pos < 0) - return !present; + return !entry; indexed = istate->cache[pos]; if (!S_ISREG(indexed->ce_mode) || ce_stage(indexed) || ce_skip_worktree(indexed) || ce_intent_to_add(indexed) || (indexed->ce_flags & CE_VALID)) return 0; - return present && entry.source == ATTR_MANIFEST_INDEX && - !memcmp(entry.hash, indexed->oid.hash, algo->rawsz); + return entry && entry->source == ATTR_MANIFEST_INDEX && + !memcmp(entry->hash, indexed->oid.hash, algo->rawsz); } #endif @@ -236,12 +245,15 @@ int clean_status_manifest_directory_unchanged( struct clean_status_index_snapshot snapshot; struct clean_status_config_digest config; struct attr_fingerprint attrs; + struct attr_manifest_cursor manifest_cursor; + struct attr_manifest_entry manifest_entry; struct string_list candidates = STRING_LIST_INIT_DUP; struct strbuf candidate = STRBUF_INIT; const struct git_hash_algo *algo = istate->repo->hash_algo; + const char *previous = NULL; unsigned int first, namespace_unstable = 0; - size_t len; - int pos, pinned = 0, safe = 0; + size_t len, previous_len = 0; + int pos, manifest_ret, pinned = 0, safe = 0; uint32_t required = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | FSMONITOR_CLEAN_PROOF_FULL_INDEX; @@ -302,8 +314,7 @@ int clean_status_manifest_directory_unchanged( strbuf_addstr(&candidate, directory); strbuf_addstr(&candidate, GITATTRIBUTES_FILE); - string_list_insert(&candidates, candidate.buf); - /* Bound attribute-source I/O, not the affected in-memory entries. */ + string_list_append(&candidates, candidate.buf); for (unsigned int i = first; i < istate->cache_nr && starts_with(istate->cache[i]->name, directory); i++) { const struct cache_entry *ce = istate->cache[i]; @@ -314,21 +325,49 @@ int clean_status_manifest_directory_unchanged( S_ISSPARSEDIR(ce->ce_mode)) goto done; while ((slash = strchr(slash, '/')) != NULL) { - strbuf_reset(&candidate); - strbuf_add(&candidate, ce->name, - slash - ce->name + 1); - strbuf_addstr(&candidate, GITATTRIBUTES_FILE); - string_list_insert(&candidates, candidate.buf); - if (candidates.nr > 64) - goto done; + size_t parent_len = slash - ce->name; + + if (!previous || previous_len <= parent_len || + previous[parent_len] != '/' || + memcmp(previous, ce->name, parent_len)) { + strbuf_reset(&candidate); + strbuf_add(&candidate, ce->name, parent_len + 1); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + string_list_append(&candidates, candidate.buf); + } slash++; } + previous = ce->name; + previous_len = ce_namelen(ce); } - for (size_t i = 0; i < candidates.nr; i++) + string_list_sort(&candidates); + string_list_remove_duplicates(&candidates, 0); + if (attr_manifest_cursor_init(&manifest_cursor, + state->manifest.current.buf, + state->manifest.current.len, algo)) + goto done; + manifest_ret = attr_manifest_cursor_next(&manifest_cursor, + &manifest_entry); + for (size_t i = 0; i < candidates.nr; i++) { + const char *name = candidates.items[i].string; + const struct attr_manifest_entry *entry = NULL; + + while (manifest_ret > 0 && + directory_manifest_entry_path_compare( + &manifest_entry, name) < 0) + manifest_ret = attr_manifest_cursor_next( + &manifest_cursor, &manifest_entry); + if (manifest_ret < 0) + goto done; + if (manifest_ret > 0 && + !directory_manifest_entry_path_compare( + &manifest_entry, name)) + entry = &manifest_entry; if (!directory_attribute_source_matches( - istate, path, candidates.items[i].string, + istate, path, name, entry, first + i)) goto done; + } semantic_verify_path_free(path, &namespace_unstable, NULL); path = NULL; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index a253bd28ce6302..7412ef7e55682b 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -4204,21 +4204,28 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ' test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ - 'directory closure rejects too many distinct attribute candidates' ' + 'directory closure reuses more than 64 authenticated attribute sources' ' test_when_finished "rm -rf directory-many-attribute-candidates" && test_create_repo directory-many-attribute-candidates && ( cd directory-many-attribute-candidates && sane_unset GIT_TEST_SPLIT_INDEX && - mkdir cached sibling && + mkdir before cached sibling && for descendant in $(test_seq 1 64) do mkdir "cached/child-$descendant" && printf "aaaa\n" \ >"cached/child-$descendant/tracked" || return 1 done && + test_write_lines "# unchanged before" \ + >before/.gitattributes && + test_write_lines "# unchanged inside" \ + >cached/child-32/.gitattributes && + test_write_lines "# unchanged after" \ + >sibling/.gitattributes && + test_write_lines retained >before/tracked && test_write_lines retained >sibling/tracked && - git add cached sibling && + git add before cached sibling && git commit -qm base && git config core.trustctime false && git config core.checkStat minimal && @@ -4247,11 +4254,11 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_cmp .git/expect .git/actual && test_grep "^1 \\.M .* cached/child-1/tracked$" .git/actual && test_grep "^? sibling/visible$" .git/actual && - test_trace2_data fsmonitor semantic/manifest-scan-count 2 \ + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ <.git/status.trace && - ! test_trace2_data fsmonitor semantic/manifest-directory-reused 1 \ + test_trace2_data fsmonitor semantic/manifest-directory-reused 1 \ <.git/status.trace && - ! test_trace2_data status \ + test_trace2_data status \ fsmonitor_token/reused-semantic-subtrees 1 \ <.git/status.trace && test_trace2_data fsmonitor token_closure/accepted 1 \ @@ -4262,8 +4269,9 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'directory closure rejects raced attributes and rechecks raced excludes' ' - test_when_finished "rm -rf directory-race-attributes directory-race-ignore" && - for mutation in attributes ignore + test_when_finished "rm -rf directory-race-attributes \ + directory-race-nested-attributes directory-race-ignore" && + for mutation in attributes nested-attributes ignore do test_create_repo "directory-race-$mutation" && ( @@ -4281,6 +4289,16 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_write_lines "$descendant" \ >"cached/retained-$descendant" || return 1 done && + if test "$mutation" = nested-attributes + then + for descendant in $(test_seq 1 64) + do + mkdir "cached/child-$descendant" && + test_write_lines "$descendant" \ + >"cached/child-$descendant/tracked" || + return 1 + done + fi && for sibling in $(test_seq 1 8) do mkdir "sibling-$sibling" && @@ -4289,6 +4307,10 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ done && git add .gitignore cached/.gitignore cached/tracked \ cached/retained-* sibling-* && + if test "$mutation" = nested-attributes + then + git add cached/child-* + fi && git commit -qm base && git config core.trustctime false && git config core.checkStat minimal && @@ -4327,6 +4349,10 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ then test_write_lines "tracked text eol=crlf" \ >cached/.gitattributes + elif test "$mutation" = nested-attributes + then + test_write_lines "tracked text eol=crlf" \ + >cached/child-64/.gitattributes else test_write_lines "!junk.ignored" >cached/.gitignore fi && @@ -4343,10 +4369,18 @@ test_expect_success PIPE,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_cmp .git/expect .git/actual && test_grep "^1 \\.M .* cached/tracked$" .git/actual && test_grep "^? sibling-1/visible$" .git/actual && - if test "$mutation" = attributes + if test "$mutation" = attributes || + test "$mutation" = nested-attributes then - test_grep "^? cached/\\.gitattributes$" \ - .git/actual && + if test "$mutation" = attributes + then + test_grep "^? cached/\\.gitattributes$" \ + .git/actual + else + test_grep \ + "^? cached/child-64/\\.gitattributes$" \ + .git/actual + fi && test_trace2_data fsmonitor \ semantic/manifest-scan-count 2 \ <.git/status.trace && From d01b4d613053bce5efabaae99a03e0bed976f0a8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 12:43:57 -0500 Subject: [PATCH 55/73] fsmonitor: bound nested attribute invalidation to its index cone 4d1d52613d (fsmonitor: invalidate conversion state for attribute-file events, 2026-07-24) checks every cache-entry name for each changed .gitattributes file. Only entries below that file's directory are invalidated, but several nested sources still require several complete in-memory index walks. Find the first matching entry with the sparse-aware binary search and stop at the end of the contiguous prefix. Do not expand a sparse index merely to locate the range. Keep the full walk for root attributes, case-insensitive matching, and Windows or Cygwin paths whose separators cannot be searched in the byte-sorted index. Cover middle and final ranges, absent prefixes, a collapsed sparse directory, case-folding collisions, and Windows-style separators. The change bounds name comparisons; it does not weaken attribute or content invalidation. --- fsmonitor.c | 22 ++++- t/unit-tests/u-fsmonitor-attributes.c | 119 ++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index a8dbf21b17a6f9..a217b42baf27e4 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -492,7 +492,8 @@ int fsmonitor_invalidate_attributes_path(struct index_state *istate, { size_t len = strlen(name), base, attr_len = strlen(GITATTRIBUTES_FILE); size_t invalidated = 0; - unsigned int i; + unsigned int first = 0, i; + int bounded = 0; while (len && is_dir_sep(name[len - 1])) len--; @@ -504,12 +505,27 @@ int fsmonitor_invalidate_attributes_path(struct index_state *istate, return 0; git_attr_invalidate_all(); - for (i = 0; i < istate->cache_nr; i++) { + if (base && !repo_ignore_case(the_repository)) { +#if defined(GIT_WINDOWS_NATIVE) || defined(__CYGWIN__) + bounded = !memchr(name, '\\', base); +#else + bounded = 1; +#endif + } + if (bounded) { + int pos = index_name_pos_sparse(istate, name, base); + + first = pos < 0 ? -pos - 1 : pos; + } + for (i = first; i < istate->cache_nr; i++) { struct cache_entry *ce = istate->cache[i]; if (base && (ce->ce_namelen < base || - fspathncmp(ce->name, name, base))) + fspathncmp(ce->name, name, base))) { + if (bounded) + break; continue; + } fsmonitor_invalidate_cache_entry(ce); invalidated++; } diff --git a/t/unit-tests/u-fsmonitor-attributes.c b/t/unit-tests/u-fsmonitor-attributes.c index 3eedefca7aad0b..a6784273925694 100644 --- a/t/unit-tests/u-fsmonitor-attributes.c +++ b/t/unit-tests/u-fsmonitor-attributes.c @@ -1,3 +1,5 @@ +#define USE_THE_REPOSITORY_VARIABLE + #include "unit-test.h" #include "fsmonitor.h" #include "fsmonitor-ll.h" @@ -73,6 +75,123 @@ void test_fsmonitor_attributes__root_source_invalidates_every_entry(void) release_index(&istate); } +void test_fsmonitor_attributes__bounds_middle_and_final_nested_cones(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 6); + istate.cache_alloc = istate.cache_nr = 6; + add_entry(&istate, 0, "before/tracked"); + add_entry(&istate, 1, "middle/first"); + add_entry(&istate, 2, "middle/nested/tracked"); + add_entry(&istate, 3, "middle/second"); + add_entry(&istate, 4, "middle0-sibling/tracked"); + add_entry(&istate, 5, "zzz/tracked"); + + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "middle/nested/.gitattributes")); + cl_assert(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(!(istate.cache[2]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[3]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(istate.cache[4]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(istate.cache[5]->ce_flags & CE_FSMONITOR_VALID); + + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "zzz/.gitattributes")); + cl_assert(!(istate.cache[5]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[4]->ce_flags & CE_FSMONITOR_VALID); + release_index(&istate); +} + +void test_fsmonitor_attributes__missing_cone_preserves_every_entry(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_entry(&istate, 0, "before/tracked"); + add_entry(&istate, 1, "later/tracked"); + + cl_assert(!fsmonitor_invalidate_attributes_path( + &istate, "between/.gitattributes")); + cl_assert(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + release_index(&istate); +} + +void test_fsmonitor_attributes__does_not_expand_sparse_directory(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + istate.sparse_index = INDEX_COLLAPSED; + add_entry(&istate, 0, "cone/"); + istate.cache[0]->ce_mode = S_IFDIR; + add_entry(&istate, 1, "outside/tracked"); + + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "cone/.gitattributes")); + cl_assert_equal_i(istate.sparse_index, INDEX_COLLAPSED); + cl_assert_equal_i(istate.cache_nr, 2); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + release_index(&istate); +} + +void test_fsmonitor_attributes__casefolded_cones_keep_full_fallback(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + int previous_ignore_case = + the_repository->config_values_private_.ignore_case; + int previously_initialized = the_repository->initialized; + + CALLOC_ARRAY(istate.cache, 3); + istate.cache_alloc = istate.cache_nr = 3; + add_entry(&istate, 0, "A/first"); + add_entry(&istate, 1, "M/untouched"); + add_entry(&istate, 2, "a/second"); + the_repository->initialized = 1; + the_repository->config_values_private_.ignore_case = 1; + + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "a/.gitattributes")); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(!(istate.cache[2]->ce_flags & CE_FSMONITOR_VALID)); + + the_repository->config_values_private_.ignore_case = + previous_ignore_case; + the_repository->initialized = previously_initialized; + release_index(&istate); +} + +void test_fsmonitor_attributes__windows_separator_keeps_full_fallback(void) +{ +#if !defined(GIT_WINDOWS_NATIVE) && !defined(__CYGWIN__) + cl_skip(); +#else + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_entry(&istate, 0, "alpha/tracked"); + add_entry(&istate, 1, "beta/tracked"); + + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "alpha\\.gitattributes")); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + release_index(&istate); +#endif +} + void test_fsmonitor_attributes__disabled_provider_preserves_skipped_stat(void) { struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; From 53b813d8775806d2895fc27c2dae53f3cc4c7380 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 13:04:47 -0500 Subject: [PATCH 56/73] status: preserve index proofs during temporary fsmonitor backoff 60077b92ac (status: preserve semantic history across scoped and index changes, 2026-08-11) disables the builtin provider temporarily after an inotify watch-limit failure. Reading the index in that mode correctly discards its fsmonitor-valid bits, but an ordinary status can then write that temporary state back. A short outage consequently removes the durable tracked and untracked proofs and makes later commands rebuild them. Remember why the provider was disabled, without treating it as usable. During authenticated watch-limit backoff, let status perform its normal strong checks but skip optional index, history, and sidecar publication. Explicitly disabling fsmonitor and commands that must write the index retain their existing behavior. Each explicit settings transition clears the temporary reason. Exercise the real marker format in main and linked worktrees. Repeated status calls must agree with a provider-disabled oracle while preserving the physical index and checkpoint byte for byte. Cover recovery after the marker is removed, mandatory writes, and explicit disablement too. --- builtin/commit.c | 13 +- fsmonitor-settings.c | 15 ++ fsmonitor-settings.h | 1 + t/helper/test-fsmonitor-client.c | 37 +++++ t/t7536-fsmonitor-watch-limit-backoff.sh | 202 +++++++++++++++++++++++ 5 files changed, 263 insertions(+), 5 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index e44f0f76ca81f6..363797652eef2f 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1888,6 +1888,7 @@ struct repository *repo UNUSED) int save_history_after_write = 0; int deferred_scoped_history = 0; int guarded_scoped_history_source = 0; + int optional_status_writes; struct clean_status_index_snapshot scoped_history_source = { .fd = -1, }; @@ -1962,6 +1963,8 @@ struct repository *repo UNUSED) builtin_status_usage, 0); finalize_colopts(&s.colopts, -1); finalize_deferred_config(&s); + optional_status_writes = use_optional_locks() && + !fsm_settings__is_watch_limit_backoff(the_repository); handle_untracked_files_arg(&s); handle_ignored_arg(&s); @@ -2013,7 +2016,7 @@ struct repository *repo UNUSED) return 0; } } - if (normal_clean_query && use_optional_locks() && + if (normal_clean_query && optional_status_writes && clean_status_identity_is_durable()) reissue_clean_sidecar = clean_status_sidecar_needs_reissue( @@ -2025,7 +2028,7 @@ struct repository *repo UNUSED) if (isatty(2)) clean_status_enable_progress(the_repository); } - if (use_optional_locks()) + if (optional_status_writes) clean_status_require_external_history_source(the_repository); repo_read_index(the_repository); if (sidecar_provider_reset) { @@ -2046,7 +2049,7 @@ struct repository *repo UNUSED) trace2_data_intmax("status", the_repository, "clean-proof/provider-reset-carried", 1); } - if (use_optional_locks()) { + if (optional_status_writes) { deferred_scoped_history = clean_status_defer_scoped_history_capture( &s, &scoped_history_source); @@ -2062,7 +2065,7 @@ struct repository *repo UNUSED) clean_status_capture_external_history_source( the_repository->index); } - if (normal_clean_query && use_optional_locks() && + if (normal_clean_query && optional_status_writes && clean_status_identity_is_durable() && (reissue_clean_sidecar || clean_status_external_history_was_restored( @@ -2075,7 +2078,7 @@ struct repository *repo UNUSED) s.show_untracked_files != SHOW_NO_UNTRACKED_FILES && !s.show_ignored_mode); - if (use_optional_locks()) + if (optional_status_writes) fd = repo_hold_locked_index(the_repository, &index_lock, 0); else fd = -1; diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index a0c12533413013..0ae8a8c8da956a 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -15,6 +15,7 @@ struct fsmonitor_settings { enum fsmonitor_mode mode; enum fsmonitor_reason reason; char *hook_path; + unsigned watch_limit_backoff : 1; }; /* @@ -124,6 +125,7 @@ static void lookup_fsmonitor_settings(struct repository *r) trace2_data_intmax("fsm_client", r, "settings/inotify-watch-limit-backoff", 1); fsm_settings__set_disabled(r); + r->settings.fsmonitor->watch_limit_backoff = 1; } else if (bool_value) fsm_settings__set_ipc(r); else @@ -159,6 +161,15 @@ enum fsmonitor_mode fsm_settings__get_mode(struct repository *r) return r->settings.fsmonitor->mode; } +int fsm_settings__is_watch_limit_backoff(struct repository *r) +{ + if (!r->settings.fsmonitor) + lookup_fsmonitor_settings(r); + + return r->settings.fsmonitor->mode == FSMONITOR_MODE_DISABLED && + r->settings.fsmonitor->watch_limit_backoff; +} + const char *fsm_settings__get_hook_path(struct repository *r) { if (!r->settings.fsmonitor) @@ -185,6 +196,7 @@ void fsm_settings__set_ipc(struct repository *r) r->settings.fsmonitor->mode = FSMONITOR_MODE_IPC; r->settings.fsmonitor->reason = reason; + r->settings.fsmonitor->watch_limit_backoff = 0; FREE_AND_NULL(r->settings.fsmonitor->hook_path); } @@ -206,6 +218,7 @@ void fsm_settings__set_hook(struct repository *r, const char *path) r->settings.fsmonitor->mode = FSMONITOR_MODE_HOOK; r->settings.fsmonitor->reason = reason; + r->settings.fsmonitor->watch_limit_backoff = 0; FREE_AND_NULL(r->settings.fsmonitor->hook_path); r->settings.fsmonitor->hook_path = strdup(path); } @@ -217,6 +230,7 @@ void fsm_settings__set_disabled(struct repository *r) r->settings.fsmonitor->mode = FSMONITOR_MODE_DISABLED; r->settings.fsmonitor->reason = FSMONITOR_REASON_OK; + r->settings.fsmonitor->watch_limit_backoff = 0; FREE_AND_NULL(r->settings.fsmonitor->hook_path); } @@ -228,6 +242,7 @@ void fsm_settings__set_incompatible(struct repository *r, r->settings.fsmonitor->mode = FSMONITOR_MODE_INCOMPATIBLE; r->settings.fsmonitor->reason = reason; + r->settings.fsmonitor->watch_limit_backoff = 0; FREE_AND_NULL(r->settings.fsmonitor->hook_path); } diff --git a/fsmonitor-settings.h b/fsmonitor-settings.h index ab02e3995ee8f4..07e6081a5d878a 100644 --- a/fsmonitor-settings.h +++ b/fsmonitor-settings.h @@ -30,6 +30,7 @@ void fsm_settings__set_incompatible(struct repository *r, enum fsmonitor_reason reason); enum fsmonitor_mode fsm_settings__get_mode(struct repository *r); +int fsm_settings__is_watch_limit_backoff(struct repository *r); const char *fsm_settings__get_hook_path(struct repository *r); enum fsmonitor_reason fsm_settings__get_reason(struct repository *r); diff --git a/t/helper/test-fsmonitor-client.c b/t/helper/test-fsmonitor-client.c index b5e428a0730a61..653d09455382bb 100644 --- a/t/helper/test-fsmonitor-client.c +++ b/t/helper/test-fsmonitor-client.c @@ -8,6 +8,7 @@ #include "test-tool.h" #include "parse-options.h" #include "fsmonitor-ipc.h" +#include "path.h" #include "read-cache-ll.h" #include "repository.h" #include "setup.h" @@ -85,6 +86,38 @@ static int do_send_flush(void) return 0; } +static int do_record_watch_limit(void) +{ +#if defined(__linux__) || defined(__APPLE__) + struct strbuf identity = STRBUF_INIT; + struct stat st; + char *path = NULL; + int ret = 1; + + if (fsmonitor_ipc__get_worktree_identity(the_repository, &identity)) { + error("could not identify the fsmonitor worktree"); + goto done; + } + fsmonitor_ipc__record_watch_limit_failure(identity.buf); + path = repo_git_path(the_repository, + "fsmonitor--daemon.inotify-limit"); + if (lstat(path, &st) || !S_ISREG(st.st_mode) || + st.st_uid != geteuid() || st.st_nlink != 1 || + (st.st_mode & 077)) { + error("could not record an owned fsmonitor watch-limit marker"); + goto done; + } + ret = 0; + +done: + free(path); + strbuf_release(&identity); + return ret; +#else + return error("watch-limit markers are not supported on this platform"); +#endif +} + struct hammer_thread_data { pthread_t pthread_id; @@ -189,6 +222,7 @@ int cmd__fsmonitor_client(int argc, const char **argv) const char * const fsmonitor_client_usage[] = { "test-tool fsmonitor-client query []", "test-tool fsmonitor-client flush", + "test-tool fsmonitor-client record-watch-limit", "test-tool fsmonitor-client hammer [] [] []", NULL, }; @@ -218,6 +252,9 @@ int cmd__fsmonitor_client(int argc, const char **argv) if (!strcmp(subcmd, "flush")) return !!do_send_flush(); + if (!strcmp(subcmd, "record-watch-limit")) + return !!do_record_watch_limit(); + if (!strcmp(subcmd, "hammer")) return !!do_hammer(token, nr_threads, nr_requests); diff --git a/t/t7536-fsmonitor-watch-limit-backoff.sh b/t/t7536-fsmonitor-watch-limit-backoff.sh index c80a4002809a61..0f3eb532cbb5fa 100755 --- a/t/t7536-fsmonitor-watch-limit-backoff.sh +++ b/t/t7536-fsmonitor-watch-limit-backoff.sh @@ -3,6 +3,7 @@ test_description='fsmonitor watch-limit backoff authenticates optional markers' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh if ! test_have_prereq FSMONITOR_DAEMON then @@ -21,6 +22,11 @@ esac sane_unset GIT_TEST_SPLIT_INDEX GIT_TEST_FSMONITOR +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + setup_backoff_marker_fixture () { test_create_repo "$1" && ( @@ -60,6 +66,51 @@ check_rejected_backoff_marker () { ) } +assert_backoff_full_proof () { + perl - "$1" <<-\EOF + binmode STDIN; + open my $input, "<", $ARGV[0] or die "cannot read index: $!\n"; + binmode $input; + local $/; + my $index = <$input>; + my %tokens; + for my $name ("FSMN", "FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + my $payload = substr($index, $offset + 8, $size); + if ($name eq "FSCF") { + my $flags = unpack("N", substr($payload, 8, 4)); + die "unbound FSCF flags $flags\n" if $flags != 15; + my $length = unpack("N", substr($payload, 12, 4)); + $tokens{$name} = substr($payload, 20, $length); + } else { + my $end = index($payload, "\0", 4); + die "invalid $name token\n" if $end < 0; + $tokens{$name} = substr($payload, 4, $end - 4); + } + } + die "missing builtin provider token\n" unless + $tokens{"FSMN"} =~ /\Abuiltin:/; + die "mismatched tracked provider token\n" unless + $tokens{"FSMN"} eq $tokens{"FSCF"}; + die "mismatched untracked provider token\n" unless + $tokens{"FSMN"} eq $tokens{"FSUC"}; + EOF +} + +assert_backoff_history_unchanged () { + test_cmp_bin "$1/index.before-backoff" "$1/index" && + test_cmp_bin "$1/checkpoint.before-backoff" "$2" && + if test -f "$1/sidecar.before-backoff" + then + test_cmp_bin "$1/sidecar.before-backoff" "$1/index.csts" + else + test_path_is_missing "$1/index.csts" + fi && + test_path_is_missing "$1/index.lock" +} + test_expect_success PIPE,PERL_TEST_HELPERS \ 'a FIFO watch-limit marker never blocks ordinary status' ' setup_backoff_marker_fixture marker-fifo && @@ -92,4 +143,155 @@ test_expect_success PERL_TEST_HELPERS \ test_path_is_file "$marker" ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'temporary backoff preserves main and linked index and history proofs' ' + test_create_repo watch-backoff-main && + test_when_finished "git -C watch-backoff-main -c core.fsmonitor=false \ + worktree remove --force ../watch-backoff-linked \ + >/dev/null 2>&1 || :" && + ( + cd watch-backoff-main && + test_commit base tracked && + git worktree add --detach ../watch-backoff-linked HEAD && + git config core.autocrlf false && + git config core.preloadIndex false && + git config core.untrackedCache true && + git config core.fsmonitor true && + for worktree in "$PWD" "$PWD/../watch-backoff-linked" + do + gitdir=$(git -C "$worktree" rev-parse --absolute-git-dir) && + test-tool chmtime -120 "$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" update-index --refresh && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/prime" && + test_must_be_empty "$gitdir/prime" && + assert_backoff_full_proof "$gitdir/index" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/checkpoint.trace" \ + git -C "$worktree" status --short \ + >"$gitdir/checkpoint.status" && + test_must_be_empty "$gitdir/checkpoint.status" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/checkpoint.trace" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status \ + >"$gitdir/sidecar.status" && + find "$gitdir" -maxdepth 1 -type f \ + -name "index.csh1.*" >"$gitdir/checkpoints" && + test_line_count = 1 "$gitdir/checkpoints" && + checkpoint=$(cat "$gitdir/checkpoints") && + assert_backoff_full_proof "$gitdir/index" && + cp "$gitdir/index" "$gitdir/index.before-backoff" && + cp "$checkpoint" "$gitdir/checkpoint.before-backoff" && + if test -f "$gitdir/index.csts" + then + cp "$gitdir/index.csts" \ + "$gitdir/sidecar.before-backoff" + fi && + GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + test-tool -C "$worktree" fsmonitor-client \ + record-watch-limit && + marker="$gitdir/fsmonitor--daemon.inotify-limit" && + test_path_is_file "$marker" && + test_line_count = 3 "$marker" && + test_grep "^inotify-limit-v1$" "$marker" && + test_write_lines changed >"$worktree/tracked" && + test_write_lines visible >"$worktree/visible" && + git -C "$worktree" -c core.fsmonitor=false \ + -c core.untrackedCache=false --no-optional-locks \ + status --porcelain=v2 >"$gitdir/expected" && + test_grep "^1 \\.M .* tracked$" "$gitdir/expected" && + test_grep "^? visible$" "$gitdir/expected" && + assert_backoff_history_unchanged "$gitdir" "$checkpoint" && + for attempt in first second + do + GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=// \ + GIT_TRACE2_EVENT="$gitdir/$attempt.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$attempt.actual" && + test_cmp "$gitdir/expected" \ + "$gitdir/$attempt.actual" && + test_trace2_data fsm_client \ + settings/inotify-watch-limit-backoff 1 \ + <"$gitdir/$attempt.trace" && + ! test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/$attempt.trace" && + test_region ! fsmonitor history_logical_digest \ + "$gitdir/$attempt.trace" && + test_region ! index do_write_index \ + "$gitdir/$attempt.trace" && + test_grep ! \ + "\"event\":\"child_start\".*\"fsmonitor--daemon\"" \ + "$gitdir/$attempt.trace" && + test_path_is_file "$marker" && + assert_backoff_history_unchanged \ + "$gitdir" "$checkpoint" || return 1 + done && + rm "$marker" && + GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=// \ + GIT_TRACE2_EVENT="$gitdir/recovery.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/recovery.actual" && + test_cmp "$gitdir/expected" "$gitdir/recovery.actual" && + ! test_trace2_data fsm_client \ + settings/inotify-watch-limit-backoff 1 \ + <"$gitdir/recovery.trace" && + test_path_is_missing "$marker" && + assert_backoff_full_proof "$gitdir/index" || return 1 + done + ) +' + +test_expect_success PERL_TEST_HELPERS \ + 'mandatory writers still update the index during temporary backoff' ' + setup_backoff_marker_fixture watch-backoff-mandatory && + ( + cd watch-backoff-mandatory && + cp .git/index .git/index.before && + GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + test-tool fsmonitor-client record-watch-limit && + GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + GIT_TRACE2_EVENT="$PWD/.git/mandatory.trace" \ + git add tracked && + test_trace2_data fsm_client \ + settings/inotify-watch-limit-backoff 1 \ + <.git/mandatory.trace && + test_region index do_write_index .git/mandatory.trace && + ! cmp .git/index.before .git/index && + test_grep ! FSMN .git/index && + test_grep ! FSUC .git/index && + git -c core.fsmonitor=false diff --cached --name-only \ + >.git/staged && + test_grep "^tracked$" .git/staged + ) +' + +test_expect_success PERL_TEST_HELPERS \ + 'an explicitly disabled fsmonitor still permits optional index writes' ' + setup_backoff_marker_fixture watch-backoff-explicit-disable && + ( + cd watch-backoff-explicit-disable && + cp .git/index .git/index.before && + GIT_TRACE2_EVENT="$PWD/.git/disabled.trace" \ + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/disabled.actual && + test_cmp .git/expect .git/disabled.actual && + test_region index do_write_index .git/disabled.trace && + ! cmp .git/index.before .git/index && + test_grep ! FSMN .git/index && + ! test_trace2_data fsm_client \ + settings/inotify-watch-limit-backoff 1 \ + <.git/disabled.trace + ) +' + test_done From a03fa3b0631b7d2c939cd16ccf6fd3ba269cc6f1 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 13:15:47 -0500 Subject: [PATCH 57/73] checkout-index: restore external proofs before updating the index 60077b92ac (status: preserve semantic history across scoped and index changes, 2026-08-11) lets external history recover proofs removed by a legacy index writer. checkout-index computes the configuration digest, but never enables that recovery before reading the index. A subsequent stat refresh can therefore persist an index without the paired proofs and leave the next status to rebuild them. Parse the checkout options before reading the index. Enable external history only for an ordinary canonical-index update with reliable stat data and the builtin provider. Keep temporary files, prefixes, alternate indexes, and non-default stages outside that admission. The existing no-op checkout still skips its index write; restoring a proof alone is not a reason to rewrite the index. Cover main and linked worktrees with an external-only checkpoint. A real dirty checkout must restore the proof, update the index, and leave a clean no-scan follower. A clean checkout must remain byte-identical. Also replay a changed attributes file honestly and verify the resulting CRLF conversion, downgraded proof, and independent status oracle. --- builtin/checkout-index.c | 15 +- t/t7519-status-fsmonitor.sh | 278 ++++++++++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+), 5 deletions(-) diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c index ac17acea58233e..f474bf524da0f6 100644 --- a/builtin/checkout-index.c +++ b/builtin/checkout-index.c @@ -12,6 +12,7 @@ #include "clean-status-config.h" #include "config.h" #include "environment.h" +#include "fsmonitor-settings.h" #include "gettext.h" #include "hook.h" #include "lockfile.h" @@ -277,13 +278,8 @@ int cmd_checkout_index(int argc, prepare_repo_settings(repo); repo->settings.command_requires_full_index = 0; - if (repo_read_index(repo) < 0) { - die("invalid cache"); - } - argc = parse_options(argc, argv, prefix, builtin_checkout_index_options, builtin_checkout_index_usage, 0); - state.istate = repo->index; state.force = force; state.quiet = quiet; state.not_new = not_new; @@ -298,6 +294,15 @@ int cmd_checkout_index(int argc, die(_("options '%s' and '%s' cannot be used together"), "--stage=all", "--no-temp"); + if (index_opt && !state.base_dir_len && !to_tempfile && + !checkout_stage && !getenv(INDEX_ENVIRONMENT) && + fstat_is_reliable() && + fsm_settings__get_mode(repo) == FSMONITOR_MODE_IPC) + clean_status_enable_external_history(repo); + if (repo_read_index(repo) < 0) + die("invalid cache"); + state.istate = repo->index; + /* * when --prefix is specified we do not want to update cache. */ diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 7412ef7e55682b..c56393794a5b28 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -1716,6 +1716,284 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELP ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'checkout-index updates restore external-only worktree proofs' ' + test_when_finished "rm -rf checkout-external-only checkout-external-linked" && + test_create_repo checkout-external-only && + ( + cd checkout-external-only && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit sibling sibling && + git worktree add --detach ../checkout-external-linked HEAD && + test-tool chmtime -120 tracked sibling \ + ../checkout-external-linked/tracked \ + ../checkout-external-linked/sibling && + git update-index --refresh && + git -C ../checkout-external-linked update-index --refresh && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + cat >.git/remove-checkout-proofs.pl <<-\EOF && + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + local $/; + my $index = ; + my $rawsz = $ARGV[0] eq "sha256" ? 32 : 20; + for my $name ("FSUC", "FSCF") { + my $offset = index($index, $name); + die "missing $name extension\n" if $offset < 0; + my $size = unpack("N", substr($index, $offset + 4, 4)); + substr($index, $offset, 8 + $size, ""); + } + my $payload = substr($index, 0, -$rawsz); + print $payload, $rawsz == 32 ? sha256($payload) : sha1($payload); + EOF + strip_checkout_proofs="$PWD/.git/remove-checkout-proofs.pl" && + for worktree in "$PWD" "$PWD/../checkout-external-linked" + do + gitdir=$(git -C "$worktree" rev-parse --absolute-git-dir) && + for mode in noop ordinary temp prefix alternate attributes + do + test-tool chmtime -120 "$worktree/tracked" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git -C "$worktree" update-index --refresh && + rm -f "$worktree/.gitattributes" \ + "$gitdir"/index.csh1.* \ + "$gitdir"/index.cswi.* \ + "$gitdir/index.csts" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git -C "$worktree" update-index --fsmonitor && + GIT_INDEX_FILE="$gitdir/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$mode.prime" && + test_must_be_empty "$gitdir/$mode.prime" && + test_fsmonitor_full_proof "$gitdir/index" paired && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkpoint.trace" \ + git -C "$worktree" status --short \ + >"$gitdir/$mode.checkpoint" && + test_must_be_empty "$gitdir/$mode.checkpoint" && + test_trace2_data fsmonitor history/external-stored 1 \ + <"$gitdir/$mode.checkpoint.trace" && + find "$gitdir" -maxdepth 1 -type f \ + -name "index.csh1.*" >"$gitdir/$mode.csh" && + test_line_count = 1 "$gitdir/$mode.csh" && + checkpoint=$(cat "$gitdir/$mode.csh") && + cp "$checkpoint" "$gitdir/$mode.checkpoint.before" && + perl "$strip_checkout_proofs" "$(test_oid algo)" \ + <"$gitdir/index" >"$gitdir/index.foreign" && + mv "$gitdir/index.foreign" "$gitdir/index" && + test_grep FSMN "$gitdir/index" && + test_grep UNTR "$gitdir/index" && + test_grep ! FSUC "$gitdir/index" && + test_grep ! FSCF "$gitdir/index" && + cp "$gitdir/index" "$gitdir/$mode.stripped.index" && + + case "$mode" in + noop) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkout.trace" \ + git -C "$worktree" checkout-index \ + -f -u tracked && + test_trace2_data fsmonitor \ + history/external-restored 1 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data index \ + extension/fsmn/read/token builtin:test:3 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data index \ + extension/fsmn/read/token builtin:test:1 \ + <"$gitdir/$mode.checkout.trace" && + test_region ! index do_write_index \ + "$gitdir/$mode.checkout.trace" && + test_cmp_bin "$gitdir/$mode.stripped.index" \ + "$gitdir/index" + ;; + ordinary) + test_write_lines modified >"$worktree/tracked" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 \ + >"$gitdir/$mode.expected" && + test_line_count = 1 "$gitdir/$mode.expected" && + test_grep "^1 \\.M .* tracked$" \ + "$gitdir/$mode.expected" && + test_cmp_bin "$gitdir/$mode.stripped.index" \ + "$gitdir/index" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkout.trace" \ + git -C "$worktree" checkout-index \ + -f -u tracked && + test_trace2_data fsmonitor \ + history/external-restored 1 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data fsmonitor apply_count 1 \ + <"$gitdir/$mode.checkout.trace" && + test_region index do_write_index \ + "$gitdir/$mode.checkout.trace" && + test_fsmonitor_full_proof "$gitdir/index" \ + paired && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$mode.checkout.trace" && + cp "$gitdir/index" \ + "$gitdir/$mode.before-status" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$mode.status" && + test_must_be_empty "$gitdir/$mode.status" && + test_cmp_bin "$gitdir/$mode.before-status" \ + "$gitdir/index" && + test_trace2_data fsmonitor config/coherent 1 \ + <"$gitdir/$mode.status.trace" && + ! test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$mode.status.trace" + ;; + temp) + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkout.trace" \ + git -C "$worktree" checkout-index \ + -u --temp tracked \ + >"$gitdir/$mode.output" && + temp_path=$(cut -f1 "$gitdir/$mode.output") && + test_path_is_file "$worktree/$temp_path" && + rm "$worktree/$temp_path" && + ! test_trace2_data fsmonitor \ + history/external-restored 1 \ + <"$gitdir/$mode.checkout.trace" && + test_cmp_bin "$gitdir/$mode.stripped.index" \ + "$gitdir/index" + ;; + prefix) + mkdir "$gitdir/checkout-prefix" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkout.trace" \ + git -C "$worktree" checkout-index \ + -f -u \ + --prefix="$gitdir/checkout-prefix/" \ + tracked && + test_path_is_file \ + "$gitdir/checkout-prefix/tracked" && + ! test_trace2_data fsmonitor \ + history/external-restored 1 \ + <"$gitdir/$mode.checkout.trace" && + test_cmp_bin "$gitdir/$mode.stripped.index" \ + "$gitdir/index" + ;; + alternate) + cp "$gitdir/index" "$gitdir/alternate.index" && + GIT_INDEX_FILE="$gitdir/alternate.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkout.trace" \ + git -C "$worktree" checkout-index \ + -f -u tracked && + ! test_trace2_data fsmonitor \ + history/external-restored 1 \ + <"$gitdir/$mode.checkout.trace" && + test_cmp_bin "$gitdir/$mode.stripped.index" \ + "$gitdir/index" + ;; + attributes) + test_write_lines "tracked text eol=crlf" \ + >"$worktree/.gitattributes" && + test_write_lines modified >"$worktree/tracked" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 \ + >"$gitdir/$mode.dirty" && + test_line_count = 2 "$gitdir/$mode.dirty" && + test_grep "^1 \\.M .* tracked$" \ + "$gitdir/$mode.dirty" && + test_grep "^? \\.gitattributes$" \ + "$gitdir/$mode.dirty" && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$gitdir/$mode.checkout.trace" \ + git -C "$worktree" checkout-index \ + -f -u tracked && + test_trace2_data fsmonitor \ + history/external-restored 1 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data fsmonitor apply_count 1 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data fsmonitor \ + semantic/attributes-scope 0 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data fsmonitor \ + semantic/manifest-scan-count 1 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data fsmonitor \ + semantic/manifest-invalidated 1 \ + <"$gitdir/$mode.checkout.trace" && + test_trace2_data fsmonitor \ + untracked/proof-missing 1 \ + <"$gitdir/$mode.checkout.trace" && + test_region index do_write_index \ + "$gitdir/$mode.checkout.trace" && + test_grep ! FSUC "$gitdir/index" && + perl - "$gitdir/index" <<-\EOF && + binmode STDIN; + open my $input, "<", $ARGV[0] or + die "cannot read index: $!\n"; + binmode $input; + local $/; + my $index = <$input>; + my $offset = index($index, "FSCF"); + die "missing FSCF extension\n" if $offset < 0; + my $flags = unpack("N", substr($index, $offset + 16, 4)); + die "unexpected FSCF flags $flags\n" if $flags != 9; + EOF + printf "base\r\n" >"$gitdir/$mode.converted" && + test_cmp_bin "$gitdir/$mode.converted" \ + "$worktree/tracked" && + cp "$gitdir/index" "$gitdir/$mode.before-status" && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$worktree" \ + -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status --porcelain=v2 \ + >"$gitdir/$mode.expected" && + test_line_count = 1 "$gitdir/$mode.expected" && + test_grep "^? \\.gitattributes$" \ + "$gitdir/$mode.expected" && + test_cmp_bin "$gitdir/$mode.before-status" \ + "$gitdir/index" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$gitdir/$mode.status.trace" \ + git -C "$worktree" status --porcelain=v2 \ + >"$gitdir/$mode.status" && + test_cmp "$gitdir/$mode.expected" \ + "$gitdir/$mode.status" && + test_cmp_bin "$gitdir/$mode.before-status" \ + "$gitdir/index" + ;; + esac && + test_cmp_bin "$gitdir/$mode.checkpoint.before" \ + "$checkpoint" || return 1 + done || return 1 + done + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'write-tree preserves authenticated primary and linked index proofs' ' test_when_finished "rm -rf write-tree-bound-proof write-tree-linked" && From 968fd3287b2ba578ffb5b838ac210b8edbdc7391 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 13:31:26 -0500 Subject: [PATCH 58/73] t7530: reject failed cookie synchronization in the prerequisite ef3af6c978 (t7530: probe fsmonitor without requiring an index write, 2026-08-16) accepts a builtin token as evidence that the native provider can support the clean-status sidecar tests. A failed cookie wait also returns a builtin token, together with a trivial response. The fixture can therefore proceed without establishing the proof that its later sidecar assertions require. Keep the daemon trace and require a seen cookie with no timed-out wait during the prerequisite. This follows the existing native fsmonitor test's treatment of an unavailable synchronization service. It does not turn a successful cookie into a stronger filesystem-ordering guarantee. --- t/t7530-status-clean-sidecar.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 4fb103c2dcff72..14c80d19d4d373 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -23,11 +23,14 @@ test_lazy_prereq DURABLE_FSMONITOR ' test-tool chmtime =-120 tracked && git -c core.fsmonitor=false update-index --refresh && git config core.fsmonitor true && - git fsmonitor--daemon start --start-timeout=10 && + GIT_TRACE_FSMONITOR="$PWD/.git/daemon.trace" \ + git fsmonitor--daemon start --start-timeout=10 && git status --porcelain=v2 >/dev/null && test-tool fsmonitor-client query --token 0 >token && nul_to_q token.filtered && - grep "^builtin:" token.filtered + grep "^builtin:" token.filtered && + grep "cookie-seen:" .git/daemon.trace && + ! grep "cookie_wait timed out" .git/daemon.trace result=$? git fsmonitor--daemon stop >/dev/null 2>&1 || : exit $result From 8b9f22cbfd0b1110714a6e48705905f929f17f4f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 13:58:22 -0500 Subject: [PATCH 59/73] read-cache: treat damaged optional index witnesses as misses 60077b92ac (status: preserve semantic history across scoped and index changes, 2026-08-11) uses a saved index to recover external worktree proofs. The recovery code opens and authenticates that witness, then reopens its pathname with the fatal main-index reader. Pruning or replacing the optional file between those opens can abort a command whose real index is valid. A FIFO can block before the file-type check. Read the entries from the already-pinned descriptor instead. Share a bounds-checked entry decoder with the main reader, but return a miss for malformed or unsupported witnesses. The optional reader verifies nonzero checksums, rejects split/sparse and resolve-undo state, and installs no acceleration extensions. Its caller still authenticates the checkpoint, logical index, provider replay, and named file identity. Read into owned storage so a concurrent truncate cannot cause an mmap fault. Open optional witnesses and snapshot probes without blocking where the platform supports it. Keep ordinary main-index corruption fatal. Cover both hash formats, index versions, malformed entries and extensions, borrowed-descriptor ownership, and recovery with a valid main index. --- clean-status-history.c | 23 +- clean-status-index.c | 8 +- read-cache-ll.h | 11 + read-cache.c | 383 ++++++++++++--- t/helper/test-read-cache.c | 152 ++++++ t/meson.build | 1 + t/t1602-index-witness.sh | 946 +++++++++++++++++++++++++++++++++++++ 7 files changed, 1463 insertions(+), 61 deletions(-) create mode 100755 t/t1602-index-witness.sh diff --git a/clean-status-history.c b/clean-status-history.c index 8dcd0760909065..cd2aa992e91b60 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -1198,6 +1198,19 @@ static void restore_external_untracked_history( } #endif +#if defined(__APPLE__) || SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int open_external_history_witness(const char *path) +{ +#ifdef O_NONBLOCK + return open_nofollow(path, O_RDONLY | O_CLOEXEC | O_NONBLOCK); +#else + (void)path; + errno = ENOSYS; + return -1; +#endif +} +#endif + static int restore_external_semantic_history( struct index_state *istate, const struct clean_status_history_checkpoint *checkpoint, @@ -1247,14 +1260,15 @@ static int restore_external_semantic_history( path = clean_status_history_store_witness_path( istate->repo->index_file, proof_namespace, istate->repo->hash_algo); - fd = open_nofollow(path, O_RDONLY | O_CLOEXEC); + fd = open_external_history_witness(path); if (fd < 0 || fstat(fd, &before) || !S_ISREG(before.st_mode) || before.st_nlink != 1 || before.st_uid != geteuid() || clean_status_identity_from_stat(&before_identity, &before) || lstat(path, &after) || before.st_dev != after.st_dev || before.st_ino != after.st_ino) goto done; - do_read_index(&witness, path, 1); + if (read_index_entries_from_fd(&witness, fd)) + goto done; if (fstat(fd, &after) || after.st_nlink != 1 || after.st_uid != geteuid() || clean_status_identity_from_stat(&after_identity, &after) || @@ -1430,14 +1444,15 @@ static int restore_external_bootstrap_manifest( path = clean_status_history_store_witness_path( istate->repo->index_file, proof_namespace, istate->repo->hash_algo); - fd = open_nofollow(path, O_RDONLY | O_CLOEXEC); + fd = open_external_history_witness(path); if (fd < 0 || fstat(fd, &before) || !S_ISREG(before.st_mode) || before.st_nlink != 1 || before.st_uid != geteuid() || clean_status_identity_from_stat(&before_identity, &before) || lstat(path, &after) || before.st_dev != after.st_dev || before.st_ino != after.st_ino) goto done; - do_read_index(&witness, path, 1); + if (read_index_entries_from_fd(&witness, fd)) + goto done; if (fstat(fd, &after) || after.st_nlink != 1 || after.st_uid != geteuid() || clean_status_identity_from_stat(&after_identity, &after) || diff --git a/clean-status-index.c b/clean-status-index.c index 34dc23d6c0abfb..50609cf323d693 100644 --- a/clean-status-index.c +++ b/clean-status-index.c @@ -58,13 +58,17 @@ static int snapshot_open( { struct clean_status_identity named; struct stat fd_st, named_st; - int fd; + int fd, flags = O_RDONLY | O_CLOEXEC; memset(snapshot, 0, sizeof(*snapshot)); snapshot->fd = -1; - fd = open_nofollow(path, O_RDONLY); +#ifdef O_NONBLOCK + flags |= O_NONBLOCK; +#endif + fd = open_nofollow(path, flags); if (fd < 0 || fstat(fd, &fd_st) || + !S_ISREG(fd_st.st_mode) || lstat(path, &named_st) || clean_status_identity_from_stat(&snapshot->identity, &fd_st) || clean_status_identity_from_stat(&named, &named_st) || diff --git a/read-cache-ll.h b/read-cache-ll.h index 34405b6d5e18d3..a6899217158916 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -320,6 +320,17 @@ int do_read_index(struct index_state *istate, const char *path, /* Takes ownership of fd, including when the state is already initialized. */ int do_read_index_from_fd(struct index_state *istate, int fd, const char *path); +/* + * Read only the entries of a full index into a fresh index_state. Optional + * extensions are ignored; split/sparse indexes, resolve-undo, malformed data, + * and unknown mandatory extensions are rejected. Nonzero checksums are always + * verified. A zero skipHash trailer requires separate authentication by the + * caller before using these entries as a proof. + * + * The caller owns fd. Its offset is unchanged, and failure leaves istate + * unchanged. Return 0 on success or -1 for a missing/unsupported witness. + */ +int read_index_entries_from_fd(struct index_state *istate, int fd); int read_index_from(struct index_state *, const char *path, const char *gitdir); int is_index_unborn(struct index_state *); diff --git a/read-cache.c b/read-cache.c index bf0866a6e5540f..923e8748bd45ea 100644 --- a/read-cache.c +++ b/read-cache.c @@ -32,6 +32,7 @@ #include "name-hash.h" #include "object-name.h" #include "path.h" +#include "path-namespace.h" #include "preload-index.h" #include "read-cache.h" #include "replace-object.h" @@ -1828,14 +1829,10 @@ struct ondisk_cache_entry { char name[FLEX_ARRAY]; }; -/* These are only used for v3 or lower */ +/* Index v2/v3 entries are padded to a multiple of eight bytes. */ #define align_padding_size(size, len) ((size + (len) + 8) & ~7) - (size + len) -#define align_flex_name(STRUCT,len) ((offsetof(struct STRUCT,data) + (len) + 8) & ~7) -#define ondisk_cache_entry_size(len) align_flex_name(ondisk_cache_entry,len) #define ondisk_data_size(flags, len) (the_hash_algo->rawsz + \ ((flags & CE_EXTENDED) ? 2 : 1) * sizeof(uint16_t) + len) -#define ondisk_data_size_max(len) (ondisk_data_size(CE_EXTENDED, len)) -#define ondisk_ce_size(ce) (ondisk_cache_entry_size(ondisk_data_size((ce)->ce_flags, ce_namelen(ce)))) /* Allow fsck to force verification of the index checksum. */ int verify_index_checksum; @@ -1918,30 +1915,76 @@ static int read_index_extension(struct index_state *istate, return 0; } +enum index_entry_decode_error { + INDEX_ENTRY_DECODE_OK, + INDEX_ENTRY_DECODE_CORRUPT, + INDEX_ENTRY_DECODE_FLAGS, + INDEX_ENTRY_DECODE_NAME, +}; + +enum index_entry_decode_flags { + INDEX_ENTRY_ALLOW_NAME_RESTART = 1 << 0, + INDEX_ENTRY_VERIFY_FORMAT = 1 << 1, +}; + +struct decoded_index_entry { + struct cache_entry *ce; + size_t size; + unsigned int bad_flags; +}; + +static int decode_index_entry_varint(const unsigned char **cursor, + const unsigned char *end, + uint64_t *result) +{ + const unsigned char *p = *cursor; + unsigned char c; + uint64_t value; + + if (p == end) + return -1; + c = *p++; + value = c & 127; + while (c & 128) { + value++; + if (!value || MSB(value, 7) || p == end) + return -1; + c = *p++; + value = (value << 7) + (c & 127); + } + *cursor = p; + *result = value; + return 0; +} + /* - * Parses the contents of the cache entry contained within the 'ondisk' buffer - * into a new incore 'cache_entry'. + * Decode one entry without reading beyond available bytes or reporting a + * fatal error. The main-index reader supplies its usual fatal wrapper below; + * optional index witnesses use the same decoder and treat errors as misses. * - * Note that 'char *ondisk' may not be aligned to a 4-byte address interval in - * index v4, so we cannot cast it to 'struct ondisk_cache_entry *' and access - * its members. Instead, we use the byte offsets of members within the struct to - * identify where 'get_be16()', 'get_be32()', and 'oidread()' (which can all - * read from an unaligned memory buffer) should read from the 'ondisk' buffer - * into the corresponding incore 'cache_entry' members. + * A v4 IEOT block starts with a complete name, but its strip count still + * describes the preceding block's last name. Preserve the main reader's + * treatment of a missing previous_ce as a name restart. The optional reader + * starts at the first entry and also requests the stricter format checks. + * + * V4 entries need not be aligned. Load fixed fields by their byte offsets, + * using get_be16(), get_be32(), and oidread() rather than a struct cast. */ -static struct cache_entry *create_from_disk(struct mem_pool *ce_mem_pool, - unsigned int version, - const char *ondisk, - unsigned long *ent_size, - const struct cache_entry *previous_ce) +static enum index_entry_decode_error decode_index_entry( + struct mem_pool *ce_mem_pool, const struct git_hash_algo *algo, + unsigned int version, const char *ondisk, size_t available, + const struct cache_entry *previous_ce, unsigned int options, + struct decoded_index_entry *decoded) { struct cache_entry *ce; - size_t len; - const char *name; - const unsigned hashsz = the_hash_algo->rawsz; - const char *flagsp = ondisk + offsetof(struct ondisk_cache_entry, data) + hashsz; + size_t len, suffix_len, consumed; + size_t fixed_size = offsetof(struct ondisk_cache_entry, data) + + algo->rawsz + sizeof(uint16_t); + const char *name, *end = ondisk + available; + const char *flagsp; unsigned int flags; size_t copy_len = 0; + int verify_format = options & INDEX_ENTRY_VERIFY_FORMAT; /* * Adjacent cache entries tend to share the leading paths, so it makes * sense to only store the differences in later entries. In the v4 @@ -1951,42 +1994,85 @@ static struct cache_entry *create_from_disk(struct mem_pool *ce_mem_pool, */ int expand_name_field = version == 4; + memset(decoded, 0, sizeof(*decoded)); + if (available < fixed_size) + return INDEX_ENTRY_DECODE_CORRUPT; + flagsp = ondisk + fixed_size - sizeof(uint16_t); + /* On-disk flags are just 16 bits */ flags = get_be16(flagsp); len = flags & CE_NAMEMASK; if (flags & CE_EXTENDED) { - int extended_flags; - extended_flags = get_be16(flagsp + sizeof(uint16_t)) << 16; + unsigned int extended_flags; + + if (available - fixed_size < sizeof(uint16_t)) + return INDEX_ENTRY_DECODE_CORRUPT; + extended_flags = + (unsigned int)get_be16(flagsp + sizeof(uint16_t)) << 16; /* We do not yet understand any bit out of CE_EXTENDED_FLAGS */ - if (extended_flags & ~CE_EXTENDED_FLAGS) - die(_("unknown index entry format 0x%08x"), extended_flags); + if (extended_flags & ~CE_EXTENDED_FLAGS) { + decoded->bad_flags = extended_flags; + return INDEX_ENTRY_DECODE_FLAGS; + } flags |= extended_flags; - name = (const char *)(flagsp + 2 * sizeof(uint16_t)); + fixed_size += sizeof(uint16_t); } - else - name = (const char *)(flagsp + sizeof(uint16_t)); + name = ondisk + fixed_size; if (expand_name_field) { const unsigned char *cp = (const unsigned char *)name; - uint64_t strip_len, previous_len; + uint64_t strip_len; - /* If we're at the beginning of a block, ignore the previous name */ - strip_len = decode_varint(&cp); + if (decode_index_entry_varint( + &cp, (const unsigned char *)end, &strip_len)) + return INDEX_ENTRY_DECODE_CORRUPT; if (previous_ce) { - previous_len = previous_ce->ce_namelen; - if (previous_len < strip_len) - die(_("malformed name field in the index, near path '%s'"), - previous_ce->name); - copy_len = previous_len - strip_len; - } + if (previous_ce->ce_namelen < strip_len) + return INDEX_ENTRY_DECODE_NAME; + copy_len = previous_ce->ce_namelen - strip_len; + } else if (strip_len && + !(options & INDEX_ENTRY_ALLOW_NAME_RESTART)) + return INDEX_ENTRY_DECODE_NAME; name = (const char *)cp; } if (len == CE_NAMEMASK) { - len = strlen(name); - if (expand_name_field) - len += copy_len; + const char *nul = memchr(name, '\0', end - name); + + if (!nul || copy_len > INT_MAX || + (size_t)(nul - name) > INT_MAX - copy_len) + return INDEX_ENTRY_DECODE_CORRUPT; + suffix_len = nul - name; + len = copy_len + suffix_len; + if (verify_format && len < CE_NAMEMASK) + return INDEX_ENTRY_DECODE_CORRUPT; + } else { + if (len < copy_len) + return INDEX_ENTRY_DECODE_CORRUPT; + suffix_len = len - copy_len; + if (suffix_len >= (size_t)(end - name) || name[suffix_len] || + (verify_format && memchr(name, '\0', suffix_len))) + return INDEX_ENTRY_DECODE_CORRUPT; + } + if (len > INT_MAX || + len > SIZE_MAX - offsetof(struct cache_entry, name) - 1) + return INDEX_ENTRY_DECODE_CORRUPT; + + consumed = (name - ondisk) + suffix_len + 1; + if (!expand_name_field) { + size_t padded; + + if (consumed > SIZE_MAX - 7) + return INDEX_ENTRY_DECODE_CORRUPT; + padded = (consumed + 7) & ~(size_t)7; + if (padded > available) + return INDEX_ENTRY_DECODE_CORRUPT; + if (verify_format) + for (size_t i = consumed; i < padded; i++) + if (ondisk[i]) + return INDEX_ENTRY_DECODE_CORRUPT; + consumed = padded; } ce = mem_pool__ce_alloc(ce_mem_pool, len); @@ -2017,18 +2103,189 @@ static struct cache_entry *create_from_disk(struct mem_pool *ce_mem_pool, ce->ce_namelen = len; ce->index = 0; oidread(&ce->oid, (const unsigned char *)ondisk + offsetof(struct ondisk_cache_entry, data), - the_repository->hash_algo); + algo); + + if (copy_len) + memcpy(ce->name, previous_ce->name, copy_len); + memcpy(ce->name + copy_len, name, suffix_len + 1); + decoded->ce = ce; + decoded->size = consumed; + return INDEX_ENTRY_DECODE_OK; +} + +static struct cache_entry *create_from_disk( + struct mem_pool *ce_mem_pool, unsigned int version, + const char *ondisk, size_t available, unsigned long *ent_size, + const struct cache_entry *previous_ce) +{ + struct decoded_index_entry decoded; + enum index_entry_decode_error err = decode_index_entry( + ce_mem_pool, the_hash_algo, version, ondisk, available, + previous_ce, INDEX_ENTRY_ALLOW_NAME_RESTART, &decoded); + + if (err == INDEX_ENTRY_DECODE_FLAGS) + die(_("unknown index entry format 0x%08x"), decoded.bad_flags); + if (err == INDEX_ENTRY_DECODE_NAME && previous_ce) + die(_("malformed name field in the index, near path '%s'"), + previous_ce->name); + if (err || decoded.size > ULONG_MAX) + die(_("index file corrupt")); + *ent_size = decoded.size; + return decoded.ce; +} - if (expand_name_field) { - if (copy_len) - memcpy(ce->name, previous_ce->name, copy_len); - memcpy(ce->name + copy_len, name, len + 1 - copy_len); - *ent_size = (name - ((char *)ondisk)) + len + 1 - copy_len; - } else { - memcpy(ce->name, name, len + 1); - *ent_size = ondisk_ce_size(ce); +/* Format-level checks only: a witness must not consult worktree config. */ +static int index_witness_entry_is_valid( + const struct cache_entry *ce, unsigned int version, + const struct cache_entry *previous) +{ + const char *component = ce->name; + + switch (ce->ce_mode) { + case 0100644: + case 0100755: + case 0120000: + case 0160000: + break; + default: + return 0; } - return ce; + if (!ce_namelen(ce) || + (version == 2 && (ce->ce_flags & CE_EXTENDED))) + return 0; + for (;;) { + const char *slash = strchr(component, '/'); + size_t len = slash ? (size_t)(slash - component) : + strlen(component); + + if (!len || (len == 1 && component[0] == '.') || + (len == 2 && !memcmp(component, "..", 2)) || + (len == 4 && component[0] == '.' && + (component[1] == 'g' || component[1] == 'G') && + (component[2] == 'i' || component[2] == 'I') && + (component[3] == 't' || component[3] == 'T'))) + return 0; + if (!slash) + break; + component = slash + 1; + } + if (previous) { + int cmp = strcmp(previous->name, ce->name); + + if (cmp > 0 || + (!cmp && (!ce_stage(previous) || + ce_stage(previous) >= ce_stage(ce)))) + return 0; + } + return 1; +} + +int read_index_entries_from_fd(struct index_state *istate, int fd) +{ + struct index_state parsed = INDEX_STATE_INIT(istate->repo); + const struct git_hash_algo *algo; + struct stat before, after; + unsigned char header[sizeof(struct cache_header)]; + char *data = NULL; + size_t size, end, offset, minimum_entry_size; + uint32_t nr; + int ret = -1; + + if (!istate->repo || !istate->repo->hash_algo || fd < 0 || + istate->initialized || istate->cache || istate->cache_nr || + istate->ce_mem_pool) + return -1; + algo = istate->repo->hash_algo; + trace2_region_enter("index", "read_index_entries", istate->repo); + if (fstat(fd, &before) || !S_ISREG(before.st_mode) || + before.st_size < 0 || + (uintmax_t)before.st_size > SIZE_MAX || + (uintmax_t)before.st_size > + (uintmax_t)maximum_signed_value_of_type(ssize_t)) + goto done; + size = (size_t)before.st_size; + if (size < sizeof(header) + algo->rawsz || + (size_t)pread_in_full(fd, header, sizeof(header), 0) != + sizeof(header) || memcmp(header, "DIRC", 4)) + goto done; + parsed.version = get_be32(header + 4); + if (parsed.version < INDEX_FORMAT_LB || + parsed.version > INDEX_FORMAT_UB) + goto done; + end = size - algo->rawsz; + nr = get_be32(header + 8); + offset = sizeof(header); + minimum_entry_size = offsetof(struct ondisk_cache_entry, data) + + algo->rawsz + sizeof(uint16_t) + 1 + (parsed.version == 4); + if (nr > INT_MAX || nr > (end - offset) / minimum_entry_size || + unsigned_mult_overflows((size_t)nr, sizeof(*parsed.cache))) + goto done; + /* A concurrent truncate must be a short read, not an mmap SIGBUS. */ + data = malloc(size); + if (!data || (size_t)pread_in_full(fd, data, size, 0) != size || + memcmp(data, header, sizeof(header))) + goto done; + oidread(&parsed.oid, (const unsigned char *)data + end, algo); + if (!is_null_oid(&parsed.oid) && + !hashfile_checksum_valid(algo, (const unsigned char *)data, size)) + goto done; + if (nr) { + parsed.cache = calloc(nr, sizeof(*parsed.cache)); + if (!parsed.cache) + goto done; + parsed.ce_mem_pool = malloc(sizeof(*parsed.ce_mem_pool)); + if (!parsed.ce_mem_pool) + goto done; + mem_pool_init(parsed.ce_mem_pool, 0); + } + parsed.cache_alloc = nr; + parsed.initialized = 1; + parsed.timestamp.sec = before.st_mtime; + parsed.timestamp.nsec = ST_MTIME_NSEC(before); + while (parsed.cache_nr < nr) { + struct decoded_index_entry decoded; + const struct cache_entry *previous = parsed.cache_nr ? + parsed.cache[parsed.cache_nr - 1] : NULL; + + if (decode_index_entry(parsed.ce_mem_pool, algo, parsed.version, + data + offset, end - offset, previous, + INDEX_ENTRY_VERIFY_FORMAT, &decoded) || + !index_witness_entry_is_valid(decoded.ce, parsed.version, + previous)) + goto done; + parsed.cache[parsed.cache_nr++] = decoded.ce; + offset += decoded.size; + } + while (offset < end) { + const char *ext = data + offset; + uint32_t ext_size; + + if (end - offset < 8) + goto done; + ext_size = get_be32(ext + 4); + if (ext_size > end - offset - 8 || + ext[0] < 'A' || ext[0] > 'Z' || + !memcmp(ext, "REUC", 4)) + goto done; + /* Optional acceleration extensions are deliberately not installed. */ + offset += 8; + offset += ext_size; + } + if (fstat(fd, &after) || !path_namespace_stat_equal(&before, &after)) + goto done; + + trace2_data_intmax("index", istate->repo, "read/entries-only", + parsed.cache_nr); + release_index(istate); + *istate = parsed; + index_state_init(&parsed, istate->repo); + ret = 0; + +done: + free(data); + release_index(&parsed); + trace2_region_leave("index", "read_index_entries", istate->repo); + return ret; } static void check_ce_order(struct index_state *istate) @@ -2302,20 +2559,32 @@ static void *load_index_extensions(void *_data) */ static unsigned long load_cache_entry_block(struct index_state *istate, struct mem_pool *ce_mem_pool, int offset, int nr, const char *mmap, - unsigned long start_offset, const struct cache_entry *previous_ce) + size_t mmap_size, unsigned long start_offset, + const struct cache_entry *previous_ce) { int i; unsigned long src_offset = start_offset; + size_t end; + + if (mmap_size < the_hash_algo->rawsz || offset < 0 || nr < 0 || + nr > INT_MAX - offset || (unsigned int)offset > istate->cache_nr || + (unsigned int)nr > istate->cache_nr - offset) + die(_("index file corrupt")); + end = mmap_size - the_hash_algo->rawsz; for (i = offset; i < offset + nr; i++) { struct cache_entry *ce; unsigned long consumed; + if (src_offset > end) + die(_("index file corrupt")); ce = create_from_disk(ce_mem_pool, istate->version, - mmap + src_offset, + mmap + src_offset, end - src_offset, &consumed, previous_ce); set_index_entry(istate, i, ce); + if (consumed > ULONG_MAX - src_offset) + die(_("index file corrupt")); src_offset += consumed; previous_ce = ce; } @@ -2337,7 +2606,8 @@ static unsigned long load_all_cache_entries(struct index_state *istate, } consumed = load_cache_entry_block(istate, istate->ce_mem_pool, - 0, istate->cache_nr, mmap, src_offset, NULL); + 0, istate->cache_nr, mmap, mmap_size, + src_offset, NULL); return consumed; } @@ -2357,6 +2627,7 @@ struct load_cache_entries_thread_data struct mem_pool *ce_mem_pool; int offset; const char *mmap; + size_t mmap_size; struct index_entry_offset_table *ieot; int ieot_start; /* starting index into the ieot array */ int ieot_blocks; /* count of ieot entries to process */ @@ -2375,7 +2646,8 @@ static void *load_cache_entries_thread(void *_data) /* iterate across all ieot blocks assigned to this thread */ for (i = p->ieot_start; i < p->ieot_start + p->ieot_blocks; i++) { p->consumed += load_cache_entry_block(p->istate, p->ce_mem_pool, - p->offset, p->ieot->entries[i].nr, p->mmap, p->ieot->entries[i].offset, NULL); + p->offset, p->ieot->entries[i].nr, p->mmap, + p->mmap_size, p->ieot->entries[i].offset, NULL); p->offset += p->ieot->entries[i].nr; } return NULL; @@ -2412,6 +2684,7 @@ static unsigned long load_cache_entries_threaded(struct index_state *istate, con p->istate = istate; p->offset = offset; p->mmap = mmap; + p->mmap_size = mmap_size; p->ieot = ieot; p->ieot_start = ieot_start; p->ieot_blocks = ieot_blocks; diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 2f8ce51bf39a2f..1a759163d12dfc 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -5,6 +5,7 @@ #include "attr-fingerprint.h" #include "attr-manifest.h" #include "clean-status.h" +#include "clean-status-index.h" #include "clean-status-internal.h" #include "config.h" #include "dir.h" @@ -20,6 +21,147 @@ #include "setup.h" #include "strbuf.h" +static int witness_has_only_entries(const struct index_state *istate) +{ + const unsigned int disk_flags = + CE_STAGEMASK | CE_EXTENDED | CE_VALID | CE_EXTENDED_FLAGS; + + if (!istate->initialized || istate->cache_changed || + istate->name_hash_initialized || istate->cache_tree || + istate->resolve_undo || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || istate->untracked || + istate->clean_status || istate->fsmonitor_dirty || + istate->fsmonitor_last_update || + istate->fsmonitor_last_update_pending || + istate->fsmonitor_untracked_token || + istate->fsmonitor_token_valid || istate->fsmonitor_extension_seen || + istate->fsmonitor_untracked_extension_seen || + istate->fsmonitor_untracked_valid) + return 0; + for (size_t i = 0; i < istate->cache_nr; i++) + if (istate->cache[i]->ce_flags & ~disk_flags) + return 0; + return 1; +} + +static int compare_witness_entries(const struct index_state *witness, + const struct index_state *full) +{ + const unsigned int disk_flags = + CE_STAGEMASK | CE_EXTENDED | CE_VALID | CE_EXTENDED_FLAGS; + + if (witness->version != full->version || + witness->cache_nr != full->cache_nr || + !oideq(&witness->oid, &full->oid) || + witness->timestamp.sec != full->timestamp.sec || + witness->timestamp.nsec != full->timestamp.nsec) + return error("witness index header differs from the full reader"); + for (size_t i = 0; i < witness->cache_nr; i++) { + const struct cache_entry *a = witness->cache[i]; + const struct cache_entry *b = full->cache[i]; + + if (memcmp(&a->ce_stat_data, &b->ce_stat_data, + sizeof(a->ce_stat_data)) || + a->ce_mode != b->ce_mode || + ((a->ce_flags ^ b->ce_flags) & disk_flags) || + !oideq(&a->oid, &b->oid) || + ce_namelen(a) != ce_namelen(b) || + memcmp(a->name, b->name, ce_namelen(a) + 1)) + return error("witness entry %"PRIuMAX" differs from the full reader", + (uintmax_t)i); + } + return 0; +} + +static int test_read_index_witness(const char *path, int compare, + int unlink_after_open, int expect_miss) +{ + struct index_state witness = INDEX_STATE_INIT(the_repository); + struct index_state full = INDEX_STATE_INIT(the_repository); + struct clean_status_index_snapshot snapshot = { .fd = -1 }; + struct stat st; + int flags = O_RDONLY | O_CLOEXEC; + int fd = -1, ret = 1, read_result; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); +#ifdef O_NONBLOCK + flags |= O_NONBLOCK; +#else + /* The parser is still useful for regular-file fixtures on this platform. */ + if (lstat(path, &st) || !S_ISREG(st.st_mode)) { + ret = !expect_miss; + goto done; + } +#endif + fd = open_nofollow(path, flags); + if (fd < 0 || fstat(fd, &st) || !S_ISREG(st.st_mode)) { + ret = !expect_miss; + goto done; + } + if (lseek(fd, 1, SEEK_SET) != 1) + goto done; + if (unlink_after_open && + (clean_status_index_snapshot_open_allow_null_checksum( + &snapshot, path, the_repository->hash_algo) || + unlink(path))) + goto done; + read_result = read_index_entries_from_fd(&witness, fd); + if (fstat(fd, &st) || lseek(fd, 0, SEEK_CUR) != 1) { + error("witness reader consumed its borrowed descriptor"); + goto done; + } + if (read_result) { + if (witness.initialized || witness.cache || witness.cache_nr || + witness.ce_mem_pool) { + error("failed witness read published partial state"); + goto done; + } + ret = !expect_miss; + goto done; + } + if (expect_miss) { + error("invalid witness was accepted"); + goto done; + } + if (!witness_has_only_entries(&witness)) { + error("witness reader installed non-entry state"); + goto done; + } + if (unlink_after_open && + clean_status_index_snapshot_still_matches_path( + &snapshot, path, the_repository->hash_algo)) { + error("unlinked witness retained its named snapshot"); + goto done; + } + if (compare) { + do_read_index(&full, path, 1); + if (compare_witness_entries(&witness, &full)) + goto done; + } + ret = 0; + +done: + if (fd >= 0) + close(fd); + clean_status_index_snapshot_release(&snapshot); + release_index(&full); + release_index(&witness); + return ret; +} + +static int test_index_witness_snapshot(const char *path) +{ + struct clean_status_index_snapshot snapshot = { .fd = -1 }; + int ret; + + setup_git_directory(the_repository); + ret = clean_status_index_snapshot_open_allow_null_checksum( + &snapshot, path, the_repository->hash_algo); + clean_status_index_snapshot_release(&snapshot); + return !!ret; +} + static int test_fsmonitor_content_recovery(const char *path) { struct index_state *istate; @@ -429,6 +571,16 @@ int cmd__read_cache(int argc, const char **argv) int i, cnt = 1; const char *name = NULL; + if (argc == 3 && !strcmp(argv[1], "--read-index-witness")) + return test_read_index_witness(argv[2], 0, 0, 0); + if (argc == 3 && !strcmp(argv[1], "--expect-index-witness-miss")) + return test_read_index_witness(argv[2], 0, 0, 1); + if (argc == 3 && !strcmp(argv[1], "--compare-index-witness")) + return test_read_index_witness(argv[2], 1, 0, 0); + if (argc == 3 && !strcmp(argv[1], "--read-index-witness-unlink")) + return test_read_index_witness(argv[2], 0, 1, 0); + if (argc == 3 && !strcmp(argv[1], "--index-witness-snapshot")) + return test_index_witness_snapshot(argv[2]); if (argc == 3 && !strcmp(argv[1], "--test-fsmonitor-content-recovery")) return test_fsmonitor_content_recovery(argv[2]); diff --git a/t/meson.build b/t/meson.build index 7b10e2b568475d..a0e542915d4d32 100644 --- a/t/meson.build +++ b/t/meson.build @@ -266,6 +266,7 @@ integration_tests = [ 't1517-outside-repo.sh', 't1600-index.sh', 't1601-index-bogus.sh', + 't1602-index-witness.sh', 't1700-split-index.sh', 't1701-racy-split-index.sh', 't1800-hook.sh', diff --git a/t/t1602-index-witness.sh b/t/t1602-index-witness.sh new file mode 100755 index 00000000000000..37b20663ff928b --- /dev/null +++ b/t/t1602-index-witness.sh @@ -0,0 +1,946 @@ +#!/bin/sh + +test_description='gentle entries-only reads of optional index witnesses' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh + +test_lazy_prereq UNTRACKED_CACHE ' + { git update-index --test-untracked-cache; ret=$?; } && + test $ret -ne 1 +' + +sane_unset GIT_TEST_SPLIT_INDEX GIT_TEST_INDEX_VERSION \ + GIT_TEST_INDEX_THREADS GIT_TEST_FSMONITOR + +test_expect_success PERL_TEST_HELPERS 'write index fixture generator' ' + cat >make-index.pl <<-\EOF + use strict; + use warnings; + use Digest::SHA qw(sha1 sha256); + binmode STDIN; + binmode STDOUT; + my ($algo, $case) = @ARGV; + my $rawsz = $algo eq "sha256" ? 32 : 20; + my $fixed = 40 + $rawsz + 2; + sub digest { return $rawsz == 32 ? sha256($_[0]) : sha1($_[0]); } + sub varint { + my ($n) = @_; + my @bytes = ($n & 127); + while ($n >>= 7) { unshift @bytes, 128 | (--$n & 127); } + return pack("C*", @bytes); + } + sub entry { + my %o = @_; + my $name = $o{name} // "alpha"; + my $version = $o{version} // 2; + my $len = length($name); + my $flags = ($o{flags} // 0) | + ($o{namelen} // ($len < 0xfff ? $len : 0xfff)); + my $data = pack("N10", 11, 12, 13, 14, 15, 16, + $o{mode} // 0100644, 17, 18, 19) . + ("\x11" x $rawsz) . pack("n", $flags); + $data .= pack("n", $o{extended} // 0) if $flags & 0x4000; + if ($version == 4) { + $data .= varint($o{strip} // 0) . + ($o{suffix} // $name) . "\0"; + } else { + $data .= $name . "\0"; + $data .= "\0" x ((8 - length($data) % 8) % 8); + } + return $data; + } + if ($case eq "strip-proofs" || $case eq "unbind-proof") { + local $/; + my $data = ; + my ($version, $nr) = unpack("NN", substr($data, 4, 8)); + die "expected an uncompressed index\n" if $version < 2 || $version > 3; + my $offset = 12; + for (1 .. $nr) { + my $flags = unpack("n", substr($data, $offset + 40 + $rawsz, 2)); + my $header = $fixed + (($flags & 0x4000) ? 2 : 0); + my $len = $flags & 0xfff; + $len = index($data, "\0", $offset + $header) - $offset - $header + if $len == 0xfff; + die "invalid name\n" if $len < 0; + $offset += ($header + $len + 8) & ~7; + } + my $out = substr($data, 0, $offset); + my $end = length($data) - $rawsz; + my $proof_seen = 0; + while ($offset < $end) { + die "short extension\n" if $end - $offset < 8; + my ($name, $size) = unpack("a4N", substr($data, $offset, 8)); + die "long extension\n" if $size > $end - $offset - 8; + my $body = substr($data, $offset + 8, $size); + $offset += 8 + $size; + next if $name eq "FSUC"; + next if $case eq "strip-proofs" && $name eq "FSCF"; + next if $case eq "unbind-proof" && $name eq "FSMN"; + if ($case eq "unbind-proof" && $name eq "FSCF") { + $proof_seen++; + my $flags = unpack("N", substr($body, 8, 4)); + substr($body, 8, 4, pack("N", $flags & ~6)); + $body = substr($body, 0, -$rawsz); + $body .= digest($body); + } + $out .= pack("a4N", $name, length($body)) . $body; + } + die "missing FSCF extension\n" if $case eq "unbind-proof" && !$proof_seen; + print $out, digest($out); + exit; + } + my $version = 2; + my @entries = (entry()); + my $extra = ""; + my $signature = "DIRC"; + my $count; + if ($case eq "empty") { @entries = (); } + elsif ($case eq "stages") { + @entries = map { entry(flags => $_ << 12) } 1 .. 3; + } + elsif ($case eq "extended") { + $version = 3; + @entries = (entry(version => 3, flags => 0xc000, extended => 0x6000)); + } + elsif ($case eq "compressed") { + $version = 4; + @entries = (entry(version => 4), + entry(version => 4, name => "alphabet", suffix => "bet"), + entry(version => 4, name => "beta", strip => 8)); + } + elsif ($case eq "long-compressed") { + $version = 4; + my $prefix = "long/" . ("a/" x 2100); + @entries = (entry(version => 4, name => $prefix . "one"), + entry(version => 4, name => $prefix . "two", strip => 3, + suffix => "two")); + } + elsif ($case eq "optional-extensions") { + $extra .= pack("a4N", $_, 4) . "junk" + for qw(TREE UNTR FSMN FSCF FSUC IEOT EOIE ZZZZ); + } + elsif ($case eq "bad-signature") { $signature = "NOPE"; } + elsif ($case eq "bad-version") { $version = 5; } + elsif ($case eq "bad-count") { $count = 0xffffffff; } + elsif ($case eq "truncated-header") { + my $data = "DIRC"; + print $data, digest($data); + exit; + } + elsif ($case eq "truncated-fixed") { $entries[0] = substr($entries[0], 0, $fixed - 1); } + elsif ($case eq "truncated-flags") { + $version = 3; + @entries = (substr(entry(version => 3, flags => 0x4000, + extended => 0x4000), 0, $fixed + 1)); + } + elsif ($case eq "unknown-flags") { + $version = 3; + @entries = (entry(version => 3, flags => 0x4000, extended => 1)); + } + elsif ($case eq "v2-extended") { + @entries = (entry(flags => 0x4000, extended => 0x4000)); + } + elsif ($case eq "missing-nul") { + $version = 4; + @entries = (substr(entry(version => 4), 0, -1)); + } + elsif ($case eq "embedded-nul") { @entries = (entry(name => "al\0ha")); } + elsif ($case eq "bad-padding") { + @entries = (entry(name => "ab")); + substr($entries[0], -1, 1, "\1"); + } + elsif ($case eq "truncated-varint" || $case eq "overflow-varint") { + $version = 4; + @entries = (substr(entry(version => 4), 0, $fixed) . + ($case eq "truncated-varint" ? "\x80\x80" : ("\x80" x 10) . "\0")); + } + elsif ($case eq "first-strip") { + $version = 4; + @entries = (entry(version => 4, strip => 1)); + } + elsif ($case eq "excessive-strip") { + $version = 4; + @entries = (entry(version => 4), + entry(version => 4, name => "beta", strip => 6)); + } + elsif ($case eq "short-name") { + $version = 4; + @entries = (entry(version => 4), + entry(version => 4, name => "b", suffix => "b")); + } + elsif ($case eq "short-long-name") { + $version = 4; + @entries = (entry(version => 4, namelen => 0xfff)); + } + elsif ($case eq "unordered") { @entries = (entry(name => "beta"), entry()); } + elsif ($case eq "duplicate-stage") { @entries = (entry(flags => 0x1000)) x 2; } + elsif ($case eq "mixed-stages") { @entries = (entry(), entry(flags => 0x1000)); } + elsif ($case eq "bad-mode") { @entries = (entry(mode => 0100664)); } + elsif ($case eq "empty-name") { @entries = (entry(name => "")); } + elsif ($case eq "absolute-name") { @entries = (entry(name => "/alpha")); } + elsif ($case eq "dotdot-name") { @entries = (entry(name => "a/../b")); } + elsif ($case eq "dotgit-name") { @entries = (entry(name => "a/.GiT/b")); } + elsif ($case eq "sparse-entry") { @entries = (entry(name => "dir/", mode => 0040000)); } + elsif ($case eq "resolve-undo" || $case eq "split-index" || + $case eq "sparse-index" || $case eq "mandatory-extension") { + my %names = ("resolve-undo" => "REUC", "split-index" => "link", + "sparse-index" => "sdir", "mandatory-extension" => "zzzz"); + $extra = pack("a4N", $names{$case}, 0); + } + elsif ($case eq "truncated-extension") { $extra = "FSMN"; } + elsif ($case eq "oversized-extension") { $extra = pack("a4N", "FSMN", 10) . "x"; } + elsif ($case ne "valid" && $case ne "skiphash" && + $case ne "bad-checksum" && $case ne "truncated-trailer") { + die "unknown fixture $case\n"; + } + my $data = $signature . pack("NN", $version, $count // scalar(@entries)) . + join("", @entries) . $extra; + my $checksum = $case eq "skiphash" ? "\0" x $rawsz : digest($data); + substr($checksum, 0, 1, chr(ord(substr($checksum, 0, 1)) ^ 1)) + if $case eq "bad-checksum"; + $checksum = substr($checksum, 0, -1) if $case eq "truncated-trailer"; + print $data, $checksum; + EOF +' + +for algo in sha1 sha256 +do + test_expect_success "$algo writer-produced v2/v3/v4 and skipHash witnesses" ' + git init --object-format="$algo" "$algo" && + git -C "$algo" config core.fsmonitor false && + git -C "$algo" config core.untrackedCache false && + git -C "$algo" config index.threads 1 && + mkdir "$algo/dir" && + test_write_lines alpha >"$algo/dir/alpha" && + test_write_lines alphabet >"$algo/dir/alphabet" && + test_write_lines beta >"$algo/dir/beta" && + git -C "$algo" add dir && + for version in 2 3 4 + do + if test "$version" = 2 + then + git -C "$algo" update-index --no-skip-worktree dir/alpha + else + git -C "$algo" update-index --skip-worktree dir/alpha + fi && + for skip in false true + do + git -C "$algo" -c index.skipHash="$skip" update-index \ + --index-version="$version" --force-write-index && + test "$version" = "$(git -C "$algo" update-index --show-index-version)" && + cp "$algo/.git/index" "$algo/.git/witness" && + test-tool -C "$algo" read-cache \ + --compare-index-witness .git/witness || return 1 + done || return 1 + done + ' + + test_expect_success PTHREADS "$algo v4 IEOT block restarts use the shared decoder" ' + git -C "$algo" config index.threads 3 && + git -C "$algo" -c index.skipHash=false update-index \ + --index-version=4 --force-write-index && + cp "$algo/.git/index" "$algo/.git/witness" && + test_grep IEOT "$algo/.git/witness" && + test_grep EOIE "$algo/.git/witness" && + test-tool -C "$algo" read-cache --compare-index-witness .git/witness && + git -C "$algo" config index.threads 1 + ' + + test_expect_success PERL_TEST_HELPERS "$algo exact entry fields and long compressed names" ' + for kind in valid empty stages extended compressed long-compressed skiphash + do + perl make-index.pl "$algo" "$kind" >"$algo/.git/witness" && + test-tool -C "$algo" read-cache \ + --compare-index-witness .git/witness || return 1 + done + ' + + test_expect_success PERL_TEST_HELPERS "$algo optional extensions are not decoded or installed" ' + perl make-index.pl "$algo" optional-extensions >"$algo/.git/witness" && + test-tool -C "$algo" read-cache --read-index-witness .git/witness + ' + + test_expect_success PERL_TEST_HELPERS "$algo malformed and unsupported witnesses are clean misses" ' + for kind in bad-signature bad-version bad-count bad-checksum \ + truncated-header truncated-fixed truncated-flags unknown-flags \ + v2-extended missing-nul embedded-nul bad-padding \ + truncated-varint overflow-varint first-strip excessive-strip \ + short-name short-long-name unordered duplicate-stage mixed-stages \ + bad-mode empty-name absolute-name dotdot-name dotgit-name \ + sparse-entry resolve-undo split-index sparse-index \ + mandatory-extension truncated-extension oversized-extension \ + truncated-trailer + do + perl make-index.pl "$algo" "$kind" >"$algo/.git/witness" && + test-tool -C "$algo" read-cache \ + --expect-index-witness-miss .git/witness || return 1 + done + ' + + test_expect_success PERL_TEST_HELPERS "$algo real-index corruption remains fatal" ' + for kind in truncated-header unknown-flags excessive-strip + do + perl make-index.pl "$algo" "$kind" >"$algo/.git/witness" && + test_must_fail env GIT_INDEX_FILE="$PWD/$algo/.git/witness" \ + git -C "$algo" ls-files >out 2>err && + test_grep "^fatal:" err || return 1 + done + ' + + test_expect_success PERL_TEST_HELPERS "$algo pinned reader never reopens a pruned pathname" ' + perl make-index.pl "$algo" valid >"$algo/.git/witness" && + test-tool -C "$algo" read-cache \ + --read-index-witness-unlink .git/witness && + test_path_is_missing "$algo/.git/witness" && + test-tool -C "$algo" read-cache \ + --expect-index-witness-miss .git/witness + ' + + test_expect_success PIPE "$algo witness and installer snapshot reject a FIFO" ' + rm -f "$algo/.git/witness" && + mkfifo "$algo/.git/witness" && + test_when_finished "rm -f $algo/.git/witness" && + test-tool -C "$algo" read-cache \ + --expect-index-witness-miss .git/witness && + test_must_fail test-tool -C "$algo" read-cache \ + --index-witness-snapshot .git/witness + ' +done + +test_lazy_prereq INDEX_WITNESS_APFS ' + test_have_prereq MACOS && + /bin/df -t apfs "$TRASH_DIRECTORY" >/dev/null +' + +# Inspect the framed extensions, not an incidental "FSCF" string in the index. +# These fixtures deliberately write v2 indexes with real checksums. Without +# an explicit expected token, require the real Darwin provider used below. +test_index_witness_full_proof () { + perl - "$1" "$(git rev-parse --show-object-format)" "${2-}" <<-\EOF + use strict; + use warnings; + use Digest::SHA qw(sha1 sha256); + my ($path, $algo, $expected_token) = @ARGV; + my $rawsz = $algo eq "sha256" ? 32 : 20; + sub digest { return $rawsz == 32 ? sha256($_[0]) : sha1($_[0]); } + open my $input, "<", $path or die "cannot read $path: $!\n"; + binmode $input; + local $/; + my $index = <$input>; + my $end = length($index) - $rawsz; + die "bad index checksum in $path\n" if $end < 12 || + digest(substr($index, 0, $end)) ne substr($index, $end); + my ($signature, $version, $nr) = unpack("a4NN", substr($index, 0, 12)); + die "expected an uncompressed index in $path\n" + if $signature ne "DIRC" || $version < 2 || $version > 3; + my $offset = 12; + for (1 .. $nr) { + my $fixed = 40 + $rawsz + 2; + die "short index entry in $path\n" if $end - $offset < $fixed; + my $flags = unpack("n", substr($index, $offset + $fixed - 2, 2)); + my $header = $fixed + (($flags & 0x4000) ? 2 : 0); + die "short entry flags in $path\n" if $end - $offset < $header; + my $nul = index($index, "\0", $offset + $header); + my $len = $flags & 0xfff; + die "bad index name in $path\n" if $nul < 0 || $nul >= $end || + ($len != 0xfff && $nul != $offset + $header + $len); + $len = $nul - $offset - $header; + $offset += ($header + $len + 8) & ~7; + die "short index padding in $path\n" if $offset > $end; + } + my %ext; + while ($offset < $end) { + die "short extension in $path\n" if $end - $offset < 8; + my ($name, $size) = unpack("a4N", substr($index, $offset, 8)); + die "bad extension $name in $path\n" + if $size > $end - $offset - 8 || exists $ext{$name}; + $ext{$name} = substr($index, $offset + 8, $size); + $offset += 8 + $size; + } + my $proof = $ext{FSCF} // die "missing FSCF in $path\n"; + die "short FSCF in $path\n" if length($proof) < 20; + my ($pv, $magic, $flags, $token_len, $manifest_len) = + unpack("N5", substr($proof, 0, 20)); + die "incomplete FSCF in $path (version $pv, flags $flags)\n" + if ($pv != 1 && $pv != 2) || $magic != 0x46534331 || + $flags != 15 || !$token_len || + length($proof) != 20 + $token_len + $manifest_len + + ($pv == 2 ? 5 : 4) * $rawsz || + digest(substr($proof, 0, -$rawsz)) ne substr($proof, -$rawsz); + my $token = substr($proof, 20, $token_len); + if (length($expected_token)) { + die "unexpected provider token in $path\n" + if $token ne $expected_token; + } else { + die "not a real builtin token in $path\n" + if $token !~ /^builtin:dirmeta-v1\.inode-v1\./; + } + for my $name (qw(FSMN FSUC)) { + my $body = $ext{$name} // die "missing $name in $path\n"; + my $want_version = $name eq "FSMN" ? 2 : 1; + my $nul = index($body, "\0", 4); + die "unbound $name in $path\n" + if length($body) < 5 || unpack("N", substr($body, 0, 4)) != + $want_version || $nul < 4 || + substr($body, 4, $nul - 4) ne $token; + } + die "missing UNTR in $path\n" if !exists $ext{UNTR}; + print "FSCF version $pv flags $flags token $token\n"; + EOF +} + +test_index_witness_cookie_health () ( + witness_cookie_label=$1 && + GIT_TRACE2_EVENT="$PWD/.git/$witness_cookie_label.cookie-initial.trace" \ + test-tool fsmonitor-client query --token 0 \ + >".git/$witness_cookie_label.cookie-initial" && + nul_to_q <".git/$witness_cookie_label.cookie-initial" \ + >".git/$witness_cookie_label.cookie-initial.q" && + test_grep "^builtin:.*Q/Q$" \ + ".git/$witness_cookie_label.cookie-initial.q" && + witness_cookie_token=$(sed "s/Q.*//" \ + ".git/$witness_cookie_label.cookie-initial.q") && + # A failed startup cookie may already have retired an older epoch. + wc -c <.git/witness-daemon.trace \ + >".git/$witness_cookie_label.cookie-daemon.offset" && + witness_cookie_log_offset=$(cat \ + ".git/$witness_cookie_label.cookie-daemon.offset") && + GIT_TRACE2_EVENT="$PWD/.git/$witness_cookie_label.cookie-healthy.trace" \ + test-tool fsmonitor-client query --token "$witness_cookie_token" \ + >".git/$witness_cookie_label.cookie-healthy" && + tail -c "+$((witness_cookie_log_offset + 1))" .git/witness-daemon.trace \ + >".git/$witness_cookie_label.cookie-daemon.trace" && + nul_to_q <".git/$witness_cookie_label.cookie-healthy" \ + >".git/$witness_cookie_label.cookie-healthy.q" && + test_grep "^builtin:.*Q" \ + ".git/$witness_cookie_label.cookie-healthy.q" && + test_grep ! "Q/Q$" ".git/$witness_cookie_label.cookie-healthy.q" && + test_grep "cookie-seen:" ".git/$witness_cookie_label.cookie-daemon.trace" && + test_grep ! "cookie_wait timed out" \ + ".git/$witness_cookie_label.cookie-daemon.trace" +) + +test_index_witness_physical_prime () ( + witness_prime_label=$1 && + test_index_witness_cookie_health "$witness_prime_label" && + # The physical index must carry the full source proof before CSH issuance. + GIT_OPTIONAL_LOCKS=1 GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TRACE2_EVENT="$PWD/.git/$witness_prime_label.prime.trace" \ + git status --porcelain=v2 >".git/$witness_prime_label.prime" && + test_index_witness_full_proof .git/index \ + >".git/$witness_prime_label.proof" +) + +test_index_witness_native_baseline () { + sane_unset GIT_INDEX_FILE GIT_INDEX_VERSION \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE GIT_TEST_FSMONITOR_QUERY_PATH \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_AT \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_READY \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_RESUME GIT_TEST_FSMONITOR_TOKEN && + git config index.version 2 && + git config index.skipHash false && + git config core.autocrlf false && + git config core.trustctime true && + git config core.checkStat default && + git config core.untrackedCache true && + git config core.fsmonitor false && + test-tool chmtime -120 "$@" && + git update-index --refresh && + git update-index --index-version=2 --force-write-index && + git config core.fsmonitor true && + GIT_TRACE_FSMONITOR="$PWD/.git/witness-daemon.trace" \ + GIT_TRACE2_EVENT="$PWD/.git/witness-daemon.trace2" \ + git fsmonitor--daemon start --start-timeout=10 && + GIT_TRACE2_EVENT="$PWD/.git/baseline.enable.trace" \ + git update-index --fsmonitor && + test_index_witness_physical_prime baseline && + test_must_be_empty .git/baseline.prime +} + +test_index_witness_issue_history () { + witness_issue_label=$1 && + witness_issue_expected_token=${2-} && + GIT_OPTIONAL_LOCKS=1 \ + GIT_TRACE2_EVENT="$PWD/.git/$witness_issue_label.issue.trace" \ + git status --short >".git/$witness_issue_label.issue" && + test_trace2_data fsmonitor history/external-stored 1 \ + <".git/$witness_issue_label.issue.trace" && + find .git -maxdepth 1 -type f -name "index.csh1.*" >.git/checkpoints && + find .git -maxdepth 1 -type f -name "index.cswi.*" >.git/witnesses && + test_line_count = 1 .git/checkpoints && + test_line_count = 1 .git/witnesses && + checkpoint=$(cat .git/checkpoints) && + witness=$(cat .git/witnesses) && + test_index_witness_full_proof "$witness" "$witness_issue_expected_token" \ + >".git/$witness_issue_label.witness-proof" && + cp "$checkpoint" .git/checkpoint.good && + cp "$witness" .git/witness.good +} + +test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'corrupt external semantic witnesses fall back with a valid main index' ' + test_when_finished "git -C recovery fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo recovery && + ( + cd recovery && + test_commit base tracked && + test_index_witness_native_baseline tracked && + test_index_witness_issue_history recovery && + test_must_be_empty .git/recovery.issue && + test_write_lines changed >tracked && + git update-index --add tracked && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + strip-proofs <.git/index >.git/index.foreign && + cp .git/index.foreign .git/index && + cp .git/checkpoint.good "$checkpoint" && + cp .git/witness.good "$witness" && + git -c core.fsmonitor=false --no-optional-locks \ + status --porcelain=v2 >.git/expect && + GIT_TRACE2_EVENT="$PWD/.git/valid.trace" \ + git --no-optional-locks status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_trace2_data fsmonitor history/external-semantic-restored 1 \ + <.git/valid.trace && + for kind in truncated-header unknown-flags excessive-strip + do + cp .git/index.foreign .git/index && + cp .git/checkpoint.good "$checkpoint" && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + "$kind" >"$witness" && + GIT_TRACE2_EVENT="$PWD/.git/$kind.trace" \ + git --no-optional-locks status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_cmp_bin .git/index.foreign .git/index && + ! test_trace2_data fsmonitor history/external-semantic-restored 1 \ + <".git/$kind.trace" || return 1 + done && + if test_have_prereq PIPE + then + cp .git/index.foreign .git/index && + cp .git/checkpoint.good "$checkpoint" && + rm -f "$witness" && + mkfifo "$witness" && + GIT_TRACE2_EVENT="$PWD/.git/fifo.trace" \ + git --no-optional-locks status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_cmp_bin .git/index.foreign .git/index && + ! test_trace2_data fsmonitor history/external-semantic-restored 1 \ + <.git/fifo.trace && + rm -f "$witness" + fi + ) +' + +test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ + 'bootstrap manifest recovery treats a damaged witness as a miss' ' + test_when_finished "git -C bootstrap fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo bootstrap && + ( + cd bootstrap && + test_commit base tracked && + test_write_lines "tracked diff=old" >.gitattributes && + git add .gitattributes && + git commit -qm attributes && + test_index_witness_native_baseline tracked .gitattributes && + git -c core.fsmonitor=false --no-optional-locks \ + rev-parse :.gitattributes >.git/attributes.old-oid && + test_write_lines "tracked diff=new" >.gitattributes && + test-tool chmtime -120 .gitattributes && + test_index_witness_physical_prime bootstrap && + test_grep "^1 \\.M .* .gitattributes$" .git/bootstrap.prime && + git -c core.fsmonitor=false --no-optional-locks \ + rev-parse :.gitattributes >.git/attributes.still-staged && + test_cmp .git/attributes.old-oid .git/attributes.still-staged && + test_index_witness_issue_history bootstrap && + test_write_lines " M .gitattributes" >.git/bootstrap.expect && + test_cmp .git/bootstrap.expect .git/bootstrap.issue && + GIT_INDEX_FILE="$PWD/.git/witness.good" \ + git -c core.fsmonitor=false --no-optional-locks \ + rev-parse :.gitattributes >.git/attributes.witness-oid && + test_cmp .git/attributes.old-oid .git/attributes.witness-oid && + git -c core.fsmonitor=false update-index --add .gitattributes && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + unbind-proof <.git/index >.git/index.foreign && + cp .git/index.foreign .git/index && + cp .git/checkpoint.good "$checkpoint" && + cp .git/witness.good "$witness" && + git -c core.fsmonitor=false --no-optional-locks \ + status --porcelain=v2 >.git/expect && + GIT_TRACE2_EVENT="$PWD/.git/valid.trace" \ + git --no-optional-locks status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_trace2_data fsmonitor history/external-bootstrap-manifest 1 \ + <.git/valid.trace && + cp .git/index.foreign .git/index && + cp .git/checkpoint.good "$checkpoint" && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + truncated-header >"$witness" && + GIT_TRACE2_EVENT="$PWD/.git/corrupt.trace" \ + git --no-optional-locks status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_cmp_bin .git/index.foreign .git/index && + ! test_trace2_data fsmonitor history/external-bootstrap-manifest 1 \ + <.git/corrupt.trace + ) +' + +test_lazy_prereq INDEX_WITNESS_SCRIPTED_IPC ' + test-tool simple-ipc SUPPORTS_SIMPLE_IPC +' + +# This provider exists in the pre-fix runtime too. Its one stable token is +# truthful only while the worktree is unchanged: make every worktree edit +# before starting it, and keep all later fixture writes inside .git. +index_witness_scripted_token=builtin:test-capable:0 + +test_index_witness_scripted_prepare () { + sane_unset GIT_INDEX_FILE GIT_INDEX_VERSION \ + GIT_TEST_FSMONITOR GIT_TEST_FSMONITOR_QUERY_SEQUENCE \ + GIT_TEST_FSMONITOR_QUERY_PATH GIT_TEST_FSMONITOR_QUERY_BARRIER_AT \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_READY \ + GIT_TEST_FSMONITOR_QUERY_BARRIER_RESUME GIT_TEST_FSMONITOR_TOKEN && + git config index.version 2 && + git config index.skipHash false && + git config core.autocrlf false && + git config core.trustctime true && + git config core.checkStat default && + git config core.preloadIndex false && + git config core.untrackedCache true && + git config core.fsmonitor false && + test-tool chmtime -120 "$@" && + git update-index --refresh && + git update-index --index-version=2 --force-write-index && + git config core.fsmonitor true +} + +test_index_witness_scripted_start () { + witness_ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + GIT_TRACE2_EVENT="$PWD/.git/scripted-provider.trace" \ + test-tool simple-ipc start-daemon --name="$witness_ipc_path" \ + --threads=1 --fsmonitor-capability-superset && + printf "%s\000/\000" "$index_witness_scripted_token" \ + >.git/scripted-initial.expect && + printf "%s\000" "$index_witness_scripted_token" \ + >.git/scripted-clean.expect && + GIT_TRACE2_EVENT="$PWD/.git/scripted-initial.trace" \ + test-tool fsmonitor-client query --token 0 \ + >.git/scripted-initial.actual && + test_cmp_bin .git/scripted-initial.expect .git/scripted-initial.actual && + for witness_query in first repeated + do + GIT_TRACE2_EVENT="$PWD/.git/scripted-$witness_query.trace" \ + test-tool fsmonitor-client query \ + --token "$index_witness_scripted_token" \ + >".git/scripted-$witness_query.actual" && + test_cmp_bin .git/scripted-clean.expect \ + ".git/scripted-$witness_query.actual" || return 1 + done && + GIT_TRACE2_EVENT="$PWD/.git/scripted-enable.trace" \ + git update-index --fsmonitor +} + +test_index_witness_scripted_prime () { + witness_prime_label=$1 && + GIT_OPTIONAL_LOCKS=1 GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TRACE2_EVENT="$PWD/.git/$witness_prime_label.prime.trace" \ + git status --porcelain=v2 >".git/$witness_prime_label.prime" && + test_index_witness_full_proof .git/index \ + "$index_witness_scripted_token" \ + >".git/$witness_prime_label.proof" +} + +# Check the real, issued CSHS v2 source alias. Perl exposes the ordinary stat +# fields; Darwin stat adds the durable birth time and inode generation. The +# nanosecond fields remain in the authenticated record and are range-checked. +test_index_witness_scripted_source () { + perl - "$checkpoint" "$witness" \ + "$(git rev-parse --show-object-format)" \ + "$index_witness_scripted_token" <<-\EOF + use strict; + use warnings; + use Digest::SHA qw(sha1 sha256); + my ($checkpoint, $witness, $algo, $token) = @ARGV; + my $rawsz = $algo eq "sha256" ? 32 : 20; + sub digest { return $rawsz == 32 ? sha256($_[0]) : sha1($_[0]); } + sub read_file { + open my $fh, "<", $_[0] or die "cannot read $_[0]: $!\n"; + binmode $fh; + local $/; + return <$fh>; + } + my $source = read_file(".git/scripted-source.index"); + my $record = read_file($checkpoint); + my $end = length($record) - $rawsz; + my $offset = 12 + 2 * $rawsz; + die "bad source checksum\n" if length($source) < 12 + $rawsz || + digest(substr($source, 0, -$rawsz)) ne substr($source, -$rawsz); + die "bad CSHS checksum\n" if $end < $offset + 112 + 8 + $rawsz + 16 || + digest(substr($record, 0, $end)) ne substr($record, $end); + my ($magic, $version, $flags) = unpack("a4NN", substr($record, 0, 12)); + die "missing complete CSHS v2 source alias\n" + if $magic ne "CSHS" || $version != 2 || $flags != 15; + my $namespace = unpack("H*", substr($record, 12, $rawsz)); + die "checkpoint and witness namespaces differ\n" + if $checkpoint !~ /\.csh1\.\Q$namespace\E\z/ || + $witness !~ /\.cswi\.\Q$namespace\E\z/; + my @identity = unpack("Q>*", substr($record, $offset, 112)); + $offset += 112; + my @stat = split(/\s+/, read_file(".git/scripted-source.stat")); + my @fields = (0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 13); + die "incomplete source stat\n" if @stat != @fields; + for my $i (0 .. $#fields) { + die "source identity field $fields[$i] differs\n" + if $identity[$fields[$i]] != $stat[$i]; + } + die "source is not an owned regular single-link index\n" + if ($identity[2] & 0170000) != 0100000 || + $identity[3] != 1 || $identity[4] != $>; + for my $i (8, 10, 12) { + die "invalid source nanoseconds\n" if $identity[$i] >= 1000000000; + } + my ($source_version, $source_nr) = unpack("NN", substr($record, $offset, 8)); + $offset += 8; + die "source header differs\n" + if substr($source, 0, 4) ne "DIRC" || + substr($source, 4, 8) ne pack("NN", $source_version, $source_nr); + die "source trailer differs\n" + if substr($record, $offset, $rawsz) ne substr($source, -$rawsz); + $offset += $rawsz; + my @lengths = unpack("N4", substr($record, $offset, 16)); + $offset += 16; + my %ext; + for my $name (qw(FSMN UNTR FSCF FSUC)) { + my $len = shift @lengths; + die "short checkpoint $name\n" if !$len || $len > $end - $offset; + $ext{$name} = substr($record, $offset, $len); + $offset += $len; + } + die "trailing checkpoint bytes\n" if $offset != $end; + my $proof = $ext{FSCF}; + die "short checkpoint FSCF\n" if length($proof) < 20; + my ($pv, $pmagic, $pf, $token_len, $manifest_len) = + unpack("N5", substr($proof, 0, 20)); + die "checkpoint does not carry FULL15\n" + if ($pv != 1 && $pv != 2) || $pmagic != 0x46534331 || $pf != 15 || + $token_len != length($token) || substr($proof, 20, $token_len) ne $token || + length($proof) != 20 + $token_len + $manifest_len + + ($pv == 2 ? 5 : 4) * $rawsz || + digest(substr($proof, 0, -$rawsz)) ne substr($proof, -$rawsz); + for my $name (qw(FSMN FSUC)) { + my $body = $ext{$name}; + my $want_version = $name eq "FSMN" ? 2 : 1; + die "checkpoint has an unbound $name\n" + if substr($body, 0, 4) ne pack("N", $want_version) || + substr($body, 4, length($token) + 1) ne "$token\0"; + } + print "CSHS v2 source $identity[0]:$identity[1] ", + "birth $identity[11] generation $identity[13] ", + "index $source_version entries $source_nr checksum ", + unpack("H*", substr($source, -$rawsz)), "\n"; + EOF +} + +test_index_witness_scripted_issue_history () { + perl -e ' + use strict; + use warnings; + my @st = lstat($ARGV[0]); + die "cannot stat source index: $!\n" if !@st; + print join(" ", @st[0, 1, 2, 3, 4, 5, 7, 9, 10]), "\n"; + ' .git/index >.git/scripted-source.stat && + /usr/bin/stat -f "%DB %Uv" .git/index >>.git/scripted-source.stat && + cp .git/index .git/scripted-source.index && + test_index_witness_issue_history "$1" "$index_witness_scripted_token" && + test_cmp_bin .git/scripted-source.index .git/witness.good && + test_index_witness_scripted_source >.git/scripted-source.proof +} + +# A FIFO regression must fail instead of hanging the whole test suite. Keep +# the child's actual exit status, and reserve 124 for a killed timeout. +test_index_witness_watchdog () { + perl -e ' + use strict; + use warnings; + use Errno qw(EINTR); + my $seconds = shift @ARGV; + my $pid = fork(); + die "cannot fork watchdog: $!\n" if !defined($pid); + if (!$pid) { + exec @ARGV or die "cannot exec $ARGV[0]: $!\n"; + } + my $timed_out = 0; + $SIG{ALRM} = sub { $timed_out = 1; kill "KILL", $pid; }; + alarm $seconds; + my $waited; + do { $waited = waitpid($pid, 0); } while $waited < 0 && $! == EINTR; + my $status = $?; + alarm 0; + die "cannot reap watchdog child: $!\n" if $waited != $pid; + if ($timed_out) { + warn "index witness command timed out after $seconds seconds\n"; + exit 124; + } + exit(($status & 127) ? 128 + ($status & 127) : $status >> 8); + ' "$@" +} + +test_index_witness_scripted_restore () { + cp .git/index.foreign .git/index && + cp .git/checkpoint.good "$checkpoint" && + rm -f "$witness" && + cp .git/witness.good "$witness" +} + +test_index_witness_scripted_status () ( + witness_status_label=$1 && + witness_status_key=$2 && + witness_status_expected=$3 && + if GIT_TRACE2_EVENT="$PWD/.git/$witness_status_label.trace" \ + test_index_witness_watchdog 20 git --no-optional-locks \ + status --porcelain=v2 >".git/$witness_status_label.actual" \ + 2>".git/$witness_status_label.err" + then + echo 0 >".git/$witness_status_label.exit" + else + witness_status_ret=$? && + echo "$witness_status_ret" >".git/$witness_status_label.exit" && + cat ".git/$witness_status_label.err" >&2 + return 1 + fi && + test_cmp .git/expect ".git/$witness_status_label.actual" && + test_cmp_bin .git/index.foreign .git/index && + test_cmp_bin .git/checkpoint.good "$checkpoint" && + test_grep ! '"key":"query/incompatible-daemon"' \ + ".git/$witness_status_label.trace" && + test_grep ! '"argv":.*"fsmonitor--daemon","run","--detach"' \ + ".git/$witness_status_label.trace" && + if test "$witness_status_expected" = restored + then + test_trace2_data fsmonitor "$witness_status_key" 1 \ + <".git/$witness_status_label.trace" + else + ! test_trace2_data fsmonitor "$witness_status_key" 1 \ + <".git/$witness_status_label.trace" + fi +) + +test_index_witness_scripted_recovery () ( + witness_recovery_key=$1 && + test_index_witness_scripted_restore && + test_index_witness_scripted_status valid-before \ + "$witness_recovery_key" restored || return 1 + witness_recovery_failed=0 + for witness_kind in truncated-header unknown-flags excessive-strip + do + if test_index_witness_scripted_restore && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + "$witness_kind" >"$witness" && + test_index_witness_scripted_status "$witness_kind" \ + "$witness_recovery_key" miss + then + : + else + witness_recovery_failed=1 + fi + done + if test_index_witness_scripted_restore && + rm -f "$witness" && + mkfifo "$witness" && + test_index_witness_scripted_status fifo \ + "$witness_recovery_key" miss && + test -p "$witness" + then + : + else + witness_recovery_failed=1 + fi + # Even a pre-fix failure must reach the FIFO and closing positive control. + if test_index_witness_scripted_restore && + test_index_witness_scripted_status valid-after \ + "$witness_recovery_key" restored + then + : + else + witness_recovery_failed=1 + fi + test "$witness_recovery_failed" = 0 +) + +test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,INDEX_WITNESS_SCRIPTED_IPC,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS,PIPE \ + 'scripted-provider semantic recovery ignores damaged optional witnesses' ' + test_when_finished "test-tool -C scripted-recovery simple-ipc stop-daemon --name=.git/fsmonitor--daemon.ipc --max-wait=5 2>/dev/null || :" && + test_create_repo scripted-recovery && + ( + cd scripted-recovery && + test_commit base tracked && + test_write_lines stable >stable && + git add stable && + git commit -qm stable && + test_write_lines changed >.git/replacement && + git hash-object -w --stdin <.git/replacement >.git/replacement.oid && + test_index_witness_scripted_prepare tracked stable && + test_index_witness_scripted_start && + test_index_witness_scripted_prime semantic && + test_must_be_empty .git/semantic.prime && + test_index_witness_scripted_issue_history semantic && + test_must_be_empty .git/semantic.issue && + # Only the index changes; the provider can truthfully stay at its token. + git update-index --cacheinfo \ + "100644,$(cat .git/replacement.oid),tracked" && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + strip-proofs <.git/index >.git/index.foreign && + GIT_INDEX_FILE="$PWD/.git/index.foreign" \ + git -c core.fsmonitor=false -c core.untrackedCache=false \ + --no-optional-locks status --porcelain=v2 >.git/expect && + test_line_count = 1 .git/expect && + test_grep "^1 MM .* tracked$" .git/expect && + test_index_witness_scripted_recovery \ + history/external-semantic-restored + ) +' + +test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,INDEX_WITNESS_SCRIPTED_IPC,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS,PIPE \ + 'scripted-provider bootstrap recovery ignores damaged optional witnesses' ' + test_when_finished "test-tool -C scripted-bootstrap simple-ipc stop-daemon --name=.git/fsmonitor--daemon.ipc --max-wait=5 2>/dev/null || :" && + test_create_repo scripted-bootstrap && + ( + cd scripted-bootstrap && + test_commit base tracked && + test_write_lines "tracked diff=old" >.gitattributes && + git add .gitattributes && + git commit -qm attributes && + test_index_witness_scripted_prepare tracked .gitattributes && + git -c core.fsmonitor=false --no-optional-locks \ + rev-parse :.gitattributes >.git/attributes.old-oid && + # This edit predates the synthetic provider; no later query may omit it. + test_write_lines "tracked diff=new" >.gitattributes && + test-tool chmtime -120 .gitattributes && + test_index_witness_scripted_start && + test_index_witness_scripted_prime bootstrap && + test_grep "^1 \\.M .* .gitattributes$" .git/bootstrap.prime && + git -c core.fsmonitor=false --no-optional-locks \ + rev-parse :.gitattributes >.git/attributes.still-staged && + test_cmp .git/attributes.old-oid .git/attributes.still-staged && + test_index_witness_scripted_issue_history bootstrap && + test_write_lines " M .gitattributes" >.git/bootstrap.expect && + test_cmp .git/bootstrap.expect .git/bootstrap.issue && + GIT_INDEX_FILE="$PWD/.git/witness.good" \ + git -c core.fsmonitor=false --no-optional-locks \ + rev-parse :.gitattributes >.git/attributes.witness-oid && + test_cmp .git/attributes.old-oid .git/attributes.witness-oid && + git -c core.fsmonitor=false update-index --add .gitattributes && + perl ../make-index.pl "$(git rev-parse --show-object-format)" \ + unbind-proof <.git/index >.git/index.foreign && + GIT_INDEX_FILE="$PWD/.git/index.foreign" \ + git -c core.fsmonitor=false -c core.untrackedCache=false \ + --no-optional-locks status --porcelain=v2 >.git/expect && + test_line_count = 1 .git/expect && + test_grep "^1 M\\. .* .gitattributes$" .git/expect && + test_index_witness_scripted_recovery \ + history/external-bootstrap-manifest + ) +' + +test_done From 176a9b2792bcb7ce3fde529b5b0d17fc5f8ceeb7 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 14:10:04 -0500 Subject: [PATCH 60/73] t1602: require a healthy native cookie before recovery tests A native fsmonitor daemon can start and return a builtin token while cookie synchronization is failing. The two native witness-recovery tests then fail while establishing their source proof, before they can exercise the optional index reader. Probe a fresh repository for a real, nontrivial cookie response before running those tests. Require an observed cookie and reject a daemon trace containing a timeout. Keep the assertions inside each recovery test, and leave the portable parser tests and deterministic IPC recovery controls unconditional. --- t/t1602-index-witness.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/t/t1602-index-witness.sh b/t/t1602-index-witness.sh index 37b20663ff928b..50943332f8eb4a 100755 --- a/t/t1602-index-witness.sh +++ b/t/t1602-index-witness.sh @@ -478,7 +478,21 @@ test_index_witness_issue_history () { cp "$witness" .git/witness.good } -test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ +test_lazy_prereq INDEX_WITNESS_HEALTHY_NATIVE_COOKIE ' + test_have_prereq INDEX_WITNESS_APFS,FSMONITOR_DAEMON && + test_create_repo index-witness-native-cookie-prerequisite && + ( + cd index-witness-native-cookie-prerequisite && + trap "git fsmonitor--daemon stop >/dev/null 2>&1 || :" 0 && + git config core.fsmonitor true && + GIT_TRACE_FSMONITOR="$PWD/.git/witness-daemon.trace" \ + git fsmonitor--daemon start --start-timeout=10 && + test_index_witness_cookie_health native-prerequisite && + test_grep ! "cookie_wait timed out" .git/witness-daemon.trace + ) +' + +test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS,INDEX_WITNESS_HEALTHY_NATIVE_COOKIE \ 'corrupt external semantic witnesses fall back with a valid main index' ' test_when_finished "git -C recovery fsmonitor--daemon stop 2>/dev/null || :" && test_create_repo recovery && @@ -532,7 +546,7 @@ test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC ) ' -test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS \ +test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN,PERL_TEST_HELPERS,INDEX_WITNESS_HEALTHY_NATIVE_COOKIE \ 'bootstrap manifest recovery treats a damaged witness as a miss' ' test_when_finished "git -C bootstrap fsmonitor--daemon stop 2>/dev/null || :" && test_create_repo bootstrap && From 20d54cc2f4dde32c304a431ab96eb612f22a4aa4 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 14:23:44 -0500 Subject: [PATCH 61/73] t: use test_env for shell-function invocations The new witness and sidecar regressions put environment assignments directly before shell functions. Some shells do not export those assignments to commands called by the function, and test-lint rejects the nonportable form. Use test_env so the watchdog and bulk-status helpers pass the intended variables to Git. Keep the command output, failure handling, and proof assertions unchanged. --- t/t1602-index-witness.sh | 2 +- t/t7530-status-clean-sidecar.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/t/t1602-index-witness.sh b/t/t1602-index-witness.sh index 50943332f8eb4a..efbc04db0b0a03 100755 --- a/t/t1602-index-witness.sh +++ b/t/t1602-index-witness.sh @@ -811,7 +811,7 @@ test_index_witness_scripted_status () ( witness_status_label=$1 && witness_status_key=$2 && witness_status_expected=$3 && - if GIT_TRACE2_EVENT="$PWD/.git/$witness_status_label.trace" \ + if test_env GIT_TRACE2_EVENT="$PWD/.git/$witness_status_label.trace" \ test_index_witness_watchdog 20 git --no-optional-locks \ status --porcelain=v2 >".git/$witness_status_label.actual" \ 2>".git/$witness_status_label.err" diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 14c80d19d4d373..a60fbc54cfa5dd 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -2632,11 +2632,11 @@ test_expect_success PERL_TEST_HELPERS \ git config core.fsmonitor true && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ git update-index --fsmonitor && - GIT_INDEX_FILE="$PWD/.git/index" \ + test_env GIT_INDEX_FILE="$PWD/.git/index" \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ bulk_status status --porcelain=v2 >.git/prime && test_must_be_empty .git/prime && - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ bulk_status status --porcelain=v2 >.git/issue && test_must_be_empty .git/issue && test_path_is_file .git/index.csts && From c5a07e68b5e892140a6d47a3e9ca28a2389b9789 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 14:29:52 -0500 Subject: [PATCH 62/73] t7519: expect the index-only cached diff fallback 88f1686b27 (fsmonitor: recognize equivalent index-only reader requests, 2026-08-17) lets cached diff-index avoid restoring external worktree history. Its output depends only on the indexed objects, so rebuilding that history or hashing an old checkpoint is unnecessary. The earlier plumbing-diff regression still requires every command to restore external history. Require the conservative scoped-reader fallback for its cached case instead, together with no restoration, logical digest, manifest scan, or physical index write. Keep the restoration requirements for commands that inspect the worktree. --- t/t7519-status-fsmonitor.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index c56393794a5b28..5aebea5b4ed65b 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -4729,8 +4729,20 @@ test_expect_success MACOS,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_must_be_empty ".git/$diff_case.actual" fi && test_cmp_bin .git/index.before .git/index && - test_trace2_data fsmonitor history/external-restored 1 \ - <".git/$diff_case.trace" && + if test "$diff_case" = cached + then + test_trace2_data fsmonitor \ + semantic/scoped-reader-stat-fallback 1 \ + <".git/$diff_case.trace" && + ! test_trace2_data fsmonitor \ + history/external-restored 1 \ + <".git/$diff_case.trace" && + test_region ! fsmonitor history_logical_digest \ + ".git/$diff_case.trace" + else + test_trace2_data fsmonitor history/external-restored 1 \ + <".git/$diff_case.trace" + fi && ! test_trace2_data fsmonitor semantic/manifest-scan-count \ <".git/$diff_case.trace" && test_grep ! "\"label\":\"do_write_index\"" \ From 19fc55206f31ee6a6a426eba9d3243116e251c1c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 14:29:55 -0500 Subject: [PATCH 63/73] t7535: wait for the listener before removing its cookie directory The background start command can report a listening socket before the filesystem listener has started serving requests. The test currently removes its cookie directory before proving that the listener is ready, which mixes startup failures with the intended cookie-creation error. Complete the existing flush request before renaming the directory. IPC workers start only after the platform listener is ready, and a flush does not depend on cookie-event delivery. The following query still fails to create its cookie and must retire the old provider token. --- t/t7535-fsmonitor-cookie-reset.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t/t7535-fsmonitor-cookie-reset.sh b/t/t7535-fsmonitor-cookie-reset.sh index d83013928c80a2..f2861b405ba8f5 100755 --- a/t/t7535-fsmonitor-cookie-reset.sh +++ b/t/t7535-fsmonitor-cookie-reset.sh @@ -15,15 +15,15 @@ test_expect_success 'a failed cookie permanently invalidates the old token' ' test_create_repo cookie-reset && GIT_TRACE2_EVENT="$PWD/cookie-daemon.trace" \ git -C cookie-reset fsmonitor--daemon start --start-timeout=10 && + test-tool -C cookie-reset fsmonitor-client flush >before && + nul_to_q before.q && + test_grep "^builtin:.*:0Q/Q$" before.q && + old_token=$(sed "s/Q.*//" before.q) && mv cookie-reset/.git/fsmonitor--daemon/cookies \ cookie-reset/.git/fsmonitor--daemon/cookies.saved && test_when_finished "test ! -d cookie-reset/.git/fsmonitor--daemon/cookies.saved || mv cookie-reset/.git/fsmonitor--daemon/cookies.saved \ cookie-reset/.git/fsmonitor--daemon/cookies" && - test-tool -C cookie-reset fsmonitor-client flush >before && - nul_to_q before.q && - test_grep "^builtin:.*:0Q/Q$" before.q && - old_token=$(sed "s/Q.*//" before.q) && test-tool -C cookie-reset fsmonitor-client query \ --token "$old_token" >failed && nul_to_q failed.q && From be3eacfd884d229ea7d859e77f4624e5c69f29fa Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 14:53:51 -0500 Subject: [PATCH 64/73] read-cache: decode extension signatures as unsigned integers CACHE_EXT() shifts bytes loaded through a char pointer. On targets where char is signed, a byte with its high bit set becomes a negative integer, and shifting it has undefined behavior. The EOIE reader probes a possible signature before it knows whether the extension exists. That position can contain arbitrary payload bytes in an otherwise valid index. The new SHA-256 witness tests expose this under UBSan while reading an ordinary index. Use get_be32() to assemble the signature from unsigned bytes. Cover both the speculative EOIE probe and an optional extension with a high-bit byte in its signature, in both object formats. Leave EOIE layout and entry-thread selection unchanged. --- read-cache.c | 2 +- t/t1602-index-witness.sh | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/read-cache.c b/read-cache.c index 923e8748bd45ea..ed9bf5b24276be 100644 --- a/read-cache.c +++ b/read-cache.c @@ -69,7 +69,7 @@ * is outside the range, to cause the reader to abort. */ -#define CACHE_EXT(s) ( (s[0]<<24)|(s[1]<<16)|(s[2]<<8)|(s[3]) ) +#define CACHE_EXT(s) get_be32(s) #define CACHE_EXT_TREE 0x54524545 /* "TREE" */ #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ diff --git a/t/t1602-index-witness.sh b/t/t1602-index-witness.sh index efbc04db0b0a03..5b13f0c6657b62 100755 --- a/t/t1602-index-witness.sh +++ b/t/t1602-index-witness.sh @@ -120,6 +120,13 @@ test_expect_success PERL_TEST_HELPERS 'write index fixture generator' ' $extra .= pack("a4N", $_, 4) . "junk" for qw(TREE UNTR FSMN FSCF FSUC IEOT EOIE ZZZZ); } + elsif ($case eq "high-bit-extension") { + my $size = 12 + $rawsz; + $extra = pack("a4N", "ZZZZ", $size) . ("\x95" x $size); + } + elsif ($case eq "high-bit-signature") { + $extra = pack("a4N", "Z\x95ZZ", 0); + } elsif ($case eq "bad-signature") { $signature = "NOPE"; } elsif ($case eq "bad-version") { $version = 5; } elsif ($case eq "bad-count") { $count = 0xffffffff; } @@ -957,4 +964,39 @@ test_expect_success INDEX_WITNESS_APFS,FSMONITOR_DAEMON,INDEX_WITNESS_SCRIPTED_I ) ' +# An EOIE lookup probes backwards from the checksum before it knows whether +# the bytes are an extension signature. Keep both the current SHA-1-sized +# probe and a hash-sized probe inside a fixed high-bit optional payload. +# Also exercise the ordinary reader with a high-bit byte after the uppercase +# first byte of an optional extension signature. +for algo in sha1 sha256 +do + test_expect_success PTHREADS,PERL_TEST_HELPERS \ + "$algo index extension signatures use unsigned bytes" ' + repo=high-bit-extension-$algo && + git init --object-format="$algo" "$repo" && + perl make-index.pl "$algo" high-bit-extension >"$repo/.git/witness" && + GIT_INDEX_FILE="$PWD/$repo/.git/witness" \ + git -C "$repo" -c core.fsmonitor=false \ + -c core.untrackedCache=false -c index.threads=1 \ + --no-optional-locks ls-files --stage >expect 2>serial.err && + test_line_count = 1 expect && + test_grep "^100644 .*alpha$" expect && + test_grep "ignoring ZZZZ extension" serial.err && + GIT_INDEX_FILE="$PWD/$repo/.git/witness" \ + git -C "$repo" -c core.fsmonitor=false \ + -c core.untrackedCache=false -c index.threads=2 \ + --no-optional-locks ls-files --stage >actual 2>threaded.err && + test_cmp expect actual && + test_grep "ignoring ZZZZ extension" threaded.err && + perl make-index.pl "$algo" high-bit-signature >"$repo/.git/witness" && + GIT_INDEX_FILE="$PWD/$repo/.git/witness" \ + git -C "$repo" -c core.fsmonitor=false \ + -c core.untrackedCache=false -c index.threads=1 \ + --no-optional-locks ls-files --stage >actual 2>signature.err && + test_cmp expect actual && + test_grep "ignoring .* extension" signature.err + ' +done + test_done From a56d683649698eae71d3dc0e2cb867af8e6c7dea Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 15:58:48 -0500 Subject: [PATCH 65/73] t7536: export the watch-limit fixture environment The test library wraps perl in a shell function. Under dash, temporary assignments before that function do not reach the Git process started by the watchdog. The status output still matches the independent oracle, but the missing Trace2 file makes all three invalid-marker tests fail in linux32 and linux-TEST-vars. Use test_env to export the existing provider and Trace2 settings. Keep the watchdog, dirty fixture, oracle, and marker assertions unchanged. --- t/t7536-fsmonitor-watch-limit-backoff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7536-fsmonitor-watch-limit-backoff.sh b/t/t7536-fsmonitor-watch-limit-backoff.sh index 0f3eb532cbb5fa..911f1a29425dce 100755 --- a/t/t7536-fsmonitor-watch-limit-backoff.sh +++ b/t/t7536-fsmonitor-watch-limit-backoff.sh @@ -50,7 +50,7 @@ setup_backoff_marker_fixture () { check_rejected_backoff_marker () { ( cd "$1" && - GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ + test_env GIT_TEST_FSMONITOR_INOTIFY_BACKOFF=1 \ GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCCCCCC \ GIT_TEST_FSMONITOR_QUERY_PATH=// \ GIT_TRACE2_EVENT="$PWD/.git/$2.trace" \ From 7d315041ac43ab97ee5ddb83e69e4f7efa6d7b17 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 15:58:54 -0500 Subject: [PATCH 66/73] unit-tests: use a nonzero mode bit in namespace identity test The directory-identity test changes a synthetic stat buffer to check that a permission change invalidates the identity. S_IXGRP is zero on Windows, so the buffer remains unchanged and the assertion fails. Toggle S_IXUSR instead. It is nonzero on every supported platform and tests the same identity field without changing filesystem permissions. --- t/unit-tests/u-path-namespace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/unit-tests/u-path-namespace.c b/t/unit-tests/u-path-namespace.c index b3735df2c4a4a1..b705daaf107b5a 100644 --- a/t/unit-tests/u-path-namespace.c +++ b/t/unit-tests/u-path-namespace.c @@ -96,7 +96,7 @@ void test_path_namespace__directory_identity_ignores_unrelated_entries(void) changed.st_ino++; cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); changed = original; - changed.st_mode ^= S_IXGRP; + changed.st_mode ^= S_IXUSR; cl_assert(!path_namespace_directory_stat_equal(&original, &changed)); changed = original; changed.st_uid++; From fa49e0f77a23253131e478164093d65644d1e58c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 16:04:17 -0500 Subject: [PATCH 67/73] status: skip an unclosable terminal index refresh The provider-token closure loop permits three queries. After each changed or trivial response, it refreshes the entire tracked index before trying again. The refresh after the third query cannot be closed: the loop immediately exits and the conservative fallback invalidates that work and refreshes the index again. Keep the attribute and manifest rechecks, but stop before that final refresh when the query budget is exhausted. The fallback still rejects the pending token, invalidates tracked and untracked proofs, and checks the worktree normally. A scripted repeated-reset test matches an independent stat-based oracle and reduces captured proof epochs from four to three without accepting or publishing the failed boundary. --- t/t7519-status-fsmonitor.sh | 47 +++++++++++++++++++++++++++++++++++++ wt-status.c | 6 +++++ 2 files changed, 53 insertions(+) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 5aebea5b4ed65b..5d01ca89c9d118 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -6662,4 +6662,51 @@ test_expect_success LINUX_SCOPED_HISTORY,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORE ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'exhausted provider resets skip an unclosable final index refresh' ' + test_when_finished "rm -rf builtin-closure-terminal-reset" && + prepare_builtin_closure_repo builtin-closure-terminal-reset untracked && + ( + cd builtin-closure-terminal-reset && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_fsmonitor_full_proof .git/index paired && + test_write_lines modified >tracked && + test_write_lines "tracked -text" >.gitattributes && + test_write_lines visible >visible && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expected && + test_grep "^1 \\.M .* tracked$" .git/expected && + test_grep "^? \\.gitattributes$" .git/expected && + test_grep "^? visible$" .git/expected && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TTTT \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expected .git/actual && + test_trace2_data fsmonitor token_closure/trivial 1 \ + <.git/status.trace >.git/trivial && + test_line_count = 3 .git/trivial && + test_trace2_data fsmonitor semantic/proof-epoch-captured 1 \ + <.git/status.trace >.git/epochs && + test_line_count = 3 .git/epochs && + test_trace2_data status \ + fsmonitor_token/terminal-rescan-skipped 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "^fsmonitor last update builtin:test:2$" \ + .git/fsmonitor && + ! test_fsmonitor_full_proof .git/index paired \ + 2>.git/unbound-proof && + test_grep "^unbound FSCF flags 9$" .git/unbound-proof + ) +' + test_done diff --git a/wt-status.c b/wt-status.c index 356bc948c0711d..e45cf6f9d2c4b1 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2031,6 +2031,12 @@ static int wt_status_close_ordinary_fsmonitor_token( wt_status_reset_attr_snapshot_if_changed(s); if (wt_status_refresh_invalidated_manifest(s)) break; + if (closure->queries >= FSMONITOR_TOKEN_MAX_QUERIES) { + trace2_data_intmax("status", s->repo, + "fsmonitor_token/terminal-rescan-skipped", + 1); + break; + } if (validate_epoch) { wt_status_refresh_for_token( s, closure->refresh_flags, &scan_epoch, From bd208f35619f7b6e7d9e5d70038a7f2a025a9109 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 16:04:28 -0500 Subject: [PATCH 68/73] status: retain the identity of an index it rewrites A zero-checksum index cannot be authenticated by its trailer alone. The clean-status sidecar therefore binds it to the durable identity recorded when the index was read. If status repairs the stat cache and commits a new index inode, its postwrite sidecar attempt still compares against the old identity. The repair succeeds, but sidecar issuance fails and the next status remains cold. Let this caller request a receipt for its own canonical index write. Keep a close-on-exec duplicate of the lockfile descriptor, then record the installed file's identity after the rename and before invoking post-index-change. After the hook returns, adopt that identity only if the held descriptor and canonical pathname still match every recorded field, header, entry count, and trailer. A hook's in-place rewrite or atomic replacement therefore cannot be mistaken for our own write. Leave the original source descriptor and logical digest intact, and keep generic zero-checksum pinning strict. Unsupported, private, split, sparse, failed, and skipped writes do not produce receipts. Also schedule sidecar reissuance when its opening provider query loses the boundary, even if the old sidecar's source identity still matches. Cover zero-checksum repair and first-reset publication, self-removing hooks, foreign replacement, no-op writes, and private indexes. The clean controls require the following status to hit the new sidecar. --- builtin/commit.c | 11 +- clean-status-index.c | 155 +++++++++++ clean-status-index.h | 30 +++ read-cache-ll.h | 10 + read-cache.c | 68 ++++- t/t7530-status-clean-sidecar.sh | 442 ++++++++++++++++++++++++++++++++ 6 files changed, 701 insertions(+), 15 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 363797652eef2f..86211af7996fc0 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1892,6 +1892,8 @@ struct repository *repo UNUSED) struct clean_status_index_snapshot scoped_history_source = { .fd = -1, }; + struct clean_status_index_write_receipt written_index = + CLEAN_STATUS_INDEX_WRITE_RECEIPT_INIT; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -2020,7 +2022,8 @@ struct repository *repo UNUSED) clean_status_identity_is_durable()) reissue_clean_sidecar = clean_status_sidecar_needs_reissue( - the_repository, repository_inputs_changed); + the_repository, repository_inputs_changed || + sidecar_provider_reset); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) { @@ -2219,7 +2222,10 @@ struct repository *repo UNUSED) "history/scoped-source-epoch-mismatch", 1); } if (0 <= fd) { - repo_update_index_if_able(the_repository, &index_lock); + repo_update_index_if_able_with_receipt(the_repository, &index_lock, + &written_index); + clean_status_index_adopt_write_receipt(the_repository->index, + &written_index); if (save_history_after_write && !hook_exists(the_repository, "post-index-change") && repo_hold_locked_index(the_repository, &index_lock, 0) >= 0) { @@ -2239,6 +2245,7 @@ struct repository *repo UNUSED) rollback_lock_file(&index_lock); } } + clean_status_index_write_receipt_release(&written_index); clean_status_index_snapshot_release(&scoped_history_source); if (s.relative_paths) diff --git a/clean-status-index.c b/clean-status-index.c index 50609cf323d693..e9eb4a940d79d4 100644 --- a/clean-status-index.c +++ b/clean-status-index.c @@ -3,6 +3,7 @@ #include "clean-status-index.h" #include "clean-status-internal.h" #include "clean-status-sidecar.h" +#include "environment.h" #include "hash-framing.h" #include "object.h" #include "read-cache-ll.h" @@ -228,6 +229,160 @@ void clean_status_index_snapshot_release( snapshot->fd = -1; } +static int write_receipt_owner_matches(const struct stat *st) +{ +#ifdef __APPLE__ + return st->st_uid == geteuid(); +#else + (void)st; + return 0; +#endif +} + +static int write_receipt_is_eligible(const struct index_state *istate) +{ + const struct clean_status_state *state; + + if (!istate || !istate->initialized || !istate->repo || + !istate->repo->initialized) + return 0; + state = istate->clean_status; + return clean_status_identity_is_durable() && fstat_is_reliable() && + state && state->config_enforced && state->current_config_valid && + state->source_identity_valid && + clean_status_external_history_enabled(istate) && + !getenv(INDEX_ENVIRONMENT) && istate == istate->repo->index && + !istate->split_index && istate->sparse_index == INDEX_EXPANDED && + !repo_config_values(istate->repo)->apply_sparse_checkout && + is_null_oid(&istate->oid); +} + +int clean_status_index_prepare_write_receipt( + struct index_state *istate, int lock_fd, + struct clean_status_index_write_receipt *receipt) +{ +#if defined(__APPLE__) && defined(F_DUPFD_CLOEXEC) && \ + defined(F_GETFL) && defined(O_ACCMODE) + struct clean_status_identity initial, held; + struct stat before, after; + int owned, flags; + + if (!receipt || receipt->snapshot.fd >= 0 || receipt->istate || + receipt->recorded || !write_receipt_is_eligible(istate) || + lock_fd < 0) + return -1; + flags = fcntl(lock_fd, F_GETFL); + if (flags < 0 || (flags & O_ACCMODE) != O_RDWR || + fstat(lock_fd, &before) || !write_receipt_owner_matches(&before) || + clean_status_identity_from_stat(&initial, &before)) + return -1; + /* The writer closes its descriptor before committing the lockfile. */ + owned = fcntl(lock_fd, F_DUPFD_CLOEXEC, 0); + if (owned < 0) + return -1; + if (fstat(owned, &after) || + clean_status_identity_from_stat(&held, &after) || + !clean_status_identity_equal(&initial, &held)) { + close(owned); + return -1; + } + receipt->snapshot.fd = owned; + receipt->source_identity = istate->clean_status->source_identity; + receipt->istate = istate; + return 0; +#else + (void)istate; + (void)lock_fd; + (void)receipt; + return -1; +#endif +} + +void clean_status_index_record_write_receipt( + struct index_state *istate, + struct clean_status_index_write_receipt *receipt) +{ + struct clean_status_index_snapshot snapshot; + struct stat st; + + if (!receipt || receipt->snapshot.fd < 0) + return; + if (receipt->recorded || receipt->istate != istate || + !write_receipt_is_eligible(istate) || + !clean_status_identity_equal( + &receipt->source_identity, + &istate->clean_status->source_identity)) + goto fail; + + /* Capture ctime after our rename, but before any hook can change it. */ + snapshot = receipt->snapshot; + if (fstat(snapshot.fd, &st) || !write_receipt_owner_matches(&st) || + clean_status_identity_from_stat(&snapshot.identity, &st) || + snapshot_read(snapshot.fd, &st, istate->repo->hash_algo, + &snapshot.version, &snapshot.cache_nr, + &snapshot.checksum) || + snapshot.version != istate->version || + snapshot.cache_nr != istate->cache_nr || + !oideq(&snapshot.checksum, &istate->oid) || + !clean_status_index_snapshot_still_matches_path( + &snapshot, istate->repo->index_file, + istate->repo->hash_algo)) + goto fail; + receipt->snapshot = snapshot; + receipt->recorded = 1; + trace2_data_intmax("fsmonitor", istate->repo, + "history/own-write-source-recorded", 1); + return; + +fail: + clean_status_index_write_receipt_release(receipt); +} + +int clean_status_index_adopt_write_receipt( + struct index_state *istate, + struct clean_status_index_write_receipt *receipt) +{ + struct clean_status_state *state; + struct stat st; + int adopted = 0; + + if (!receipt || !receipt->recorded || receipt->istate != istate || + !write_receipt_is_eligible(istate)) + goto done; + state = istate->clean_status; + if (!clean_status_identity_equal(&receipt->source_identity, + &state->source_identity) || + receipt->snapshot.version != istate->version || + receipt->snapshot.cache_nr != istate->cache_nr || + !oideq(&receipt->snapshot.checksum, &istate->oid) || + fstat(receipt->snapshot.fd, &st) || + !write_receipt_owner_matches(&st) || + !clean_status_index_snapshot_still_matches_path( + &receipt->snapshot, istate->repo->index_file, + istate->repo->hash_algo)) + goto done; + + /* The original descriptor and logical digest still name the old source. */ + state->source_identity = receipt->snapshot.identity; + trace2_data_intmax("fsmonitor", istate->repo, + "history/own-write-source-adopted", 1); + adopted = 1; + +done: + clean_status_index_write_receipt_release(receipt); + return adopted; +} + +void clean_status_index_write_receipt_release( + struct clean_status_index_write_receipt *receipt) +{ + if (!receipt) + return; + clean_status_index_snapshot_release(&receipt->snapshot); + memset(receipt, 0, sizeof(*receipt)); + receipt->snapshot.fd = -1; +} + int clean_status_index_entries_are_certifiable( const struct index_state *istate) { diff --git a/clean-status-index.h b/clean-status-index.h index 1728b6021ddb3c..370792b66f7e4b 100644 --- a/clean-status-index.h +++ b/clean-status-index.h @@ -14,6 +14,36 @@ struct clean_status_index_snapshot { int fd; }; +/* + * An opt-in receipt for a canonical index write. Only the index writer may + * record it, after committing its lockfile and before running hooks. The + * caller must initialize and release it, even if no write was performed. + */ +struct clean_status_index_write_receipt { + struct clean_status_index_snapshot snapshot; + struct clean_status_identity source_identity; + const struct index_state *istate; + unsigned int recorded : 1; +}; + +#define CLEAN_STATUS_INDEX_WRITE_RECEIPT_INIT \ + { .snapshot = { .fd = -1 } } + +/* Writer-only lifecycle: prepare duplicates lock_fd; record fails closed. */ +int clean_status_index_prepare_write_receipt( + struct index_state *istate, int lock_fd, + struct clean_status_index_write_receipt *receipt); +void clean_status_index_record_write_receipt( + struct index_state *istate, + struct clean_status_index_write_receipt *receipt); + +/* Consumes the receipt and returns whether the written source was adopted. */ +int clean_status_index_adopt_write_receipt( + struct index_state *istate, + struct clean_status_index_write_receipt *receipt); +void clean_status_index_write_receipt_release( + struct clean_status_index_write_receipt *receipt); + int clean_status_index_snapshot_open( struct clean_status_index_snapshot *snapshot, const char *path, const struct git_hash_algo *algo); diff --git a/read-cache-ll.h b/read-cache-ll.h index a6899217158916..05c1db3d1c0e82 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -31,6 +31,7 @@ struct cache_entry { char name[FLEX_ARRAY]; /* more */ }; +struct clean_status_index_write_receipt; struct clean_status_proof_epoch; struct preload_bulk_stat_update; @@ -360,6 +361,15 @@ int is_index_unborn(struct index_state *); */ int write_locked_index(struct index_state *, struct lock_file *lock, unsigned flags); +/* + * Like repo_update_index_if_able(), with an optional receipt for the canonical + * file actually written. The receipt must be initialized by the caller and + * remains empty if the write is skipped, fails, or is not eligible. + */ +void repo_update_index_if_able_with_receipt( + struct repository *repo, struct lock_file *lock, + struct clean_status_index_write_receipt *receipt); + void discard_index(struct index_state *); void move_index_extensions(struct index_state *dst, struct index_state *src); int unmerged_index(const struct index_state *); diff --git a/read-cache.c b/read-cache.c index ed9bf5b24276be..f2ed21be6c169c 100644 --- a/read-cache.c +++ b/read-cache.c @@ -17,6 +17,7 @@ #include "lockfile.h" #include "cache-tree.h" #include "clean-status.h" +#include "clean-status-index.h" #include "refs.h" #include "dir.h" #include "object-file.h" @@ -3343,17 +3344,31 @@ int has_racy_timestamp(struct index_state *istate) return 0; } -void repo_update_index_if_able(struct repository *repo, - struct lock_file *lockfile) +static int write_locked_index_with_receipt( + struct index_state *istate, struct lock_file *lock, + unsigned flags, struct clean_status_index_write_receipt *receipt); + +void repo_update_index_if_able_with_receipt( + struct repository *repo, struct lock_file *lockfile, + struct clean_status_index_write_receipt *receipt) { + if (receipt) + clean_status_index_write_receipt_release(receipt); if ((repo->index->cache_changed || has_racy_timestamp(repo->index)) && repo_verify_index(repo)) - write_locked_index(repo->index, lockfile, COMMIT_LOCK); + write_locked_index_with_receipt(repo->index, lockfile, + COMMIT_LOCK, receipt); else rollback_lock_file(lockfile); } +void repo_update_index_if_able(struct repository *repo, + struct lock_file *lockfile) +{ + repo_update_index_if_able_with_receipt(repo, lockfile, NULL); +} + static int record_eoie(void) { int val; @@ -3795,17 +3810,25 @@ static int commit_locked_index(struct lock_file *lk) return commit_lock_file(lk); } -static int do_write_locked_index(struct index_state *istate, - struct lock_file *lock, - unsigned flags, - enum write_extensions write_extensions) +static int do_write_locked_index( + struct index_state *istate, struct lock_file *lock, unsigned flags, + enum write_extensions write_extensions, + struct clean_status_index_write_receipt *receipt) { int ret; int was_full = istate->sparse_index == INDEX_EXPANDED; + int receipt_prepared = 0; + + if (receipt && (flags & COMMIT_LOCK) && !alternate_index_output && + !(write_extensions & WRITE_SPLIT_INDEX_EXTENSION)) + receipt_prepared = !clean_status_index_prepare_write_receipt( + istate, get_lock_file_fd(lock), receipt); ret = convert_to_sparse(istate, 0); if (ret) { + if (receipt_prepared) + clean_status_index_write_receipt_release(receipt); warning(_("failed to convert to a sparse-index")); return ret; } @@ -3819,12 +3842,21 @@ static int do_write_locked_index(struct index_state *istate, if (was_full) ensure_full_index(istate); - if (ret) + if (ret) { + if (receipt_prepared) + clean_status_index_write_receipt_release(receipt); return ret; + } if (flags & COMMIT_LOCK) ret = commit_locked_index(lock); else ret = close_lock_file_gently(lock); + if (receipt_prepared) { + if (!ret) + clean_status_index_record_write_receipt(istate, receipt); + else + clean_status_index_write_receipt_release(receipt); + } run_hooks_l(the_repository, "post-index-change", istate->updated_workdir ? "1" : "0", @@ -3841,7 +3873,8 @@ static int write_split_index(struct index_state *istate, { int ret; prepare_to_write_split_index(istate); - ret = do_write_locked_index(istate, lock, flags, WRITE_ALL_EXTENSIONS); + ret = do_write_locked_index(istate, lock, flags, WRITE_ALL_EXTENSIONS, + NULL); finish_writing_split_index(istate); return ret; } @@ -3983,8 +4016,9 @@ static int too_many_not_shared_entries(struct index_state *istate) return (int64_t)istate->cache_nr * max_split < (int64_t)not_shared * 100; } -int write_locked_index(struct index_state *istate, struct lock_file *lock, - unsigned flags) +static int write_locked_index_with_receipt( + struct index_state *istate, struct lock_file *lock, + unsigned flags, struct clean_status_index_write_receipt *receipt) { int new_shared_index, ret, test_split_index_env; struct split_index *si = istate->split_index; @@ -4008,7 +4042,8 @@ int write_locked_index(struct index_state *istate, struct lock_file *lock, alternate_index_output || (istate->cache_changed & ~EXTMASK)) { ret = do_write_locked_index(istate, lock, flags, - ~WRITE_SPLIT_INDEX_EXTENSION); + ~WRITE_SPLIT_INDEX_EXTENSION, + receipt); goto out; } @@ -4038,7 +4073,8 @@ int write_locked_index(struct index_state *istate, struct lock_file *lock, free(path); if (!temp) { ret = do_write_locked_index(istate, lock, flags, - ~WRITE_SPLIT_INDEX_EXTENSION); + ~WRITE_SPLIT_INDEX_EXTENSION, + receipt); goto out; } ret = write_shared_index(istate, &temp, flags); @@ -4068,6 +4104,12 @@ int write_locked_index(struct index_state *istate, struct lock_file *lock, return ret; } +int write_locked_index(struct index_state *istate, struct lock_file *lock, + unsigned flags) +{ + return write_locked_index_with_receipt(istate, lock, flags, NULL); +} + /* * Read the index file that is potentially unmerged into given * index_state, dropping any unmerged entries to stage #0 (potentially diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index a60fbc54cfa5dd..937470b0262044 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -2734,4 +2734,446 @@ test_expect_success PERL_TEST_HELPERS \ ) ' +test_expect_success PERL_TEST_HELPERS \ + 'a rewritten skipHash index reissues its clean status sidecar' ' + test_when_finished "rm -rf sidecar-skiphash-postwrite" && + test_create_repo sidecar-skiphash-postwrite && + ( + cd sidecar-skiphash-postwrite && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test-tool chmtime =-180 tracked && + git -c core.fsmonitor=false update-index --refresh && + git config index.version 4 && + git config index.skipHash true && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --fsmonitor && + test_env GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + test_grep FSUC .git/index && + rawsz=$(test_oid rawsz) && + dd if=/dev/zero of=.git/zero-trailer \ + bs="$rawsz" count=1 2>/dev/null && + tail -c "$rawsz" .git/index >.git/trailer && + test_cmp_bin .git/zero-trailer .git/trailer && + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/issue && + test_must_be_empty .git/issue && + test_path_is_file .git/index.csts && + + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status >.git/expected && + cp .git/index .git/index.before-noop && + cp .git/index.csts .git/index.csts.before-noop && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/noop.trace" \ + git status >.git/noop && + test_cmp .git/expected .git/noop && + test_trace2_data status clean-proof/hit 1 \ + <.git/noop.trace && + test_region ! index do_read_index .git/noop.trace && + test_region ! index do_write_index .git/noop.trace && + test_cmp_bin .git/index.before-noop .git/index && + test_cmp_bin .git/index.csts.before-noop .git/index.csts && + + before_inode=$(/usr/bin/stat -f %i .git/index) && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --force-write-index && + foreign_inode=$(/usr/bin/stat -f %i .git/index) && + test "$before_inode" != "$foreign_inode" && + tail -c "$rawsz" .git/index >.git/trailer && + test_cmp_bin .git/zero-trailer .git/trailer && + test_cmp_bin .git/index.csts.before-noop .git/index.csts && + cp .git/index .git/index.foreign && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/foreign.trace" \ + git status >.git/foreign && + test_cmp .git/expected .git/foreign && + test_trace2_data status clean-proof/miss \ + fast-index-mismatch <.git/foreign.trace && + ! test_trace2_data status clean-proof/hit 1 \ + <.git/foreign.trace && + test_region ! index do_write_index .git/foreign.trace && + test_cmp_bin .git/index.foreign .git/index && + test_cmp_bin .git/index.csts.before-noop .git/index.csts && + + test-tool chmtime =-90 tracked && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status >.git/expected && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/repair.trace" \ + git -c core.preloadIndex=true \ + -c core.preloadIndexBulk=true \ + status >.git/actual && + test_cmp .git/expected .git/actual && + test_trace2_data status clean-proof/miss \ + fast-index-mismatch <.git/repair.trace && + test_trace2_data fsmonitor apply_count 1 \ + <.git/repair.trace && + test_region index do_write_index .git/repair.trace && + repaired_inode=$(/usr/bin/stat -f %i .git/index) && + test "$repaired_inode" != "$foreign_inode" && + ! test_cmp_bin .git/index.foreign .git/index && + tail -c "$rawsz" .git/index >.git/trailer && + test_cmp_bin .git/zero-trailer .git/trailer && + test_trace2_data status clean-proof/sidecar 1 \ + <.git/repair.trace && + test_trace2_data status clean-proof/postwrite-reissued 1 \ + <.git/repair.trace && + ! test_trace2_data status clean-proof/miss \ + issue-pinned-inputs <.git/repair.trace && + + cp .git/index .git/index.before-follower && + cp .git/index.csts .git/index.csts.before-follower && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/follower.trace" \ + git -c core.preloadIndex=true \ + -c core.preloadIndexBulk=true \ + status >.git/follower && + test_cmp .git/expected .git/follower && + test_trace2_data status clean-proof/hit 1 \ + <.git/follower.trace && + test_region ! index do_read_index .git/follower.trace && + test_region ! index do_write_index .git/follower.trace && + test_cmp_bin .git/index.before-follower .git/index && + test_cmp_bin .git/index.csts.before-follower .git/index.csts + ) +' + +test_expect_success PERL_TEST_HELPERS \ + 'index write receipts reject in-place and foreign hook mutations' ' + test_when_finished "rm -rf sidecar-receipt-valid sidecar-receipt-same-inode sidecar-receipt-foreign-replace" && + for mode in valid same-inode foreign-replace + do + repo=sidecar-receipt-$mode && + test_create_repo "$repo" && + ( + cd "$repo" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test-tool chmtime =-180 tracked && + git -c core.fsmonitor=false update-index --refresh && + git config index.version 4 && + git config index.skipHash true && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --fsmonitor && + test_env GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + test_grep FSUC .git/index && + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/issued && + test_must_be_empty .git/issued && + test_path_is_file .git/index.csts && + cp .git/index.csts .git/sidecar.before && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --force-write-index && + test_cmp_bin .git/sidecar.before .git/index.csts && + rawsz=$(test_oid rawsz) && + printf "%s\n" "$rawsz" >.git/receipt-rawsz && + dd if=/dev/zero of=.git/zero-trailer \ + bs="$rawsz" count=1 2>/dev/null && + test-tool chmtime =-90 tracked && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status >.git/expected && + + if test "$mode" != valid + then + printf "%s\n" "$mode" >.git/receipt-mode && + cat >.git/receipt-mutate.pl <<-\EOF && + use strict; + use warnings; + my $path = shift; + open(my $index, "+<", $path) or die "open: $!"; + binmode $index; + read($index, my $header, 12) == 12 or die "header"; + substr($header, 0, 4) eq "DIRC" or die "magic"; + unpack("N", substr($header, 8, 4)) or die "entries"; + seek($index, 16, 0) or die "seek"; + read($index, my $ctime, 4) == 4 or die "ctime"; + seek($index, 16, 0) or die "rewind"; + my $next = (unpack("N", $ctime) + 1) % 1000000000; + print {$index} pack("N", $next) or die "write"; + close($index) or die "close"; + EOF + write_script .git/hooks/post-index-change <<-\EOF + mode=$(cat .git/receipt-mode) && + /usr/bin/stat -f %i .git/index >.git/hook-inode-before && + cp .git/index .git/hook-index-before && + rm -f "$0" && + case "$mode" in + same-inode) + perl .git/receipt-mutate.pl .git/index + ;; + foreign-replace) + cp .git/index .git/hook-replacement && + mv .git/hook-replacement .git/index + ;; + *) + exit 1 + ;; + esac && + /usr/bin/stat -f %i .git/index >.git/hook-inode-after && + tail -c "$(cat .git/receipt-rawsz)" .git/index \ + >.git/hook-trailer + EOF + else + : + fi && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git -c core.preloadIndex=true \ + -c core.preloadIndexBulk=true \ + status >.git/actual && + test_cmp .git/expected .git/actual && + test_region index do_write_index .git/status.trace && + test_trace2_data fsmonitor \ + history/own-write-source-recorded 1 \ + <.git/status.trace && + tail -c "$rawsz" .git/index >.git/trailer && + test_cmp_bin .git/zero-trailer .git/trailer && + + case "$mode" in + valid) + test_trace2_data fsmonitor \ + history/own-write-source-adopted 1 \ + <.git/status.trace && + test_trace2_data status \ + clean-proof/postwrite-reissued 1 \ + <.git/status.trace + ;; + same-inode|foreign-replace) + test_path_is_missing .git/hooks/post-index-change && + test_cmp_bin .git/zero-trailer .git/hook-trailer && + ! test_trace2_data fsmonitor \ + history/own-write-source-adopted 1 \ + <.git/status.trace && + ! test_trace2_data status \ + clean-proof/sidecar 1 \ + <.git/status.trace && + ! test_trace2_data status \ + clean-proof/postwrite-reissued 1 \ + <.git/status.trace && + test_cmp_bin .git/sidecar.before .git/index.csts && + if test "$mode" = same-inode + then + test_cmp .git/hook-inode-before \ + .git/hook-inode-after && + ! test_cmp_bin .git/hook-index-before .git/index + else + ! test_cmp .git/hook-inode-before \ + .git/hook-inode-after && + test_cmp_bin .git/hook-index-before .git/index + fi + ;; + esac && + cp .git/index .git/follower-index && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/follower.trace" \ + git -c core.preloadIndex=true \ + -c core.preloadIndexBulk=true \ + status >.git/follower && + test_cmp .git/expected .git/follower && + test_cmp_bin .git/follower-index .git/index && + test_region ! index do_write_index .git/follower.trace && + if test "$mode" = valid + then + test_trace2_data status clean-proof/hit 1 \ + <.git/follower.trace + else + test_trace2_data status clean-proof/miss \ + fast-index-mismatch <.git/follower.trace && + ! test_trace2_data status clean-proof/hit 1 \ + <.git/follower.trace + fi + ) || return 1 + done +' + +test_expect_success PERL_TEST_HELPERS \ + 'index write receipts reject unchanged and private indexes' ' + test_when_finished "rm -rf sidecar-receipt-private" && + test_create_repo sidecar-receipt-private && + ( + cd sidecar-receipt-private && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test-tool chmtime =-180 tracked && + git -c core.fsmonitor=false update-index --refresh && + git config index.version 4 && + git config index.skipHash true && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --fsmonitor && + test_env GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/issued && + test_must_be_empty .git/issued && + test_path_is_file .git/index.csts && + cp .git/index .git/canonical.before && + cp .git/index.csts .git/sidecar.before && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/noop.trace" \ + git status >.git/noop && + test_trace2_data status clean-proof/hit 1 \ + <.git/noop.trace && + ! test_trace2_data fsmonitor \ + history/own-write-source-recorded 1 <.git/noop.trace && + ! test_trace2_data fsmonitor \ + history/own-write-source-adopted 1 <.git/noop.trace && + test_cmp_bin .git/canonical.before .git/index && + test_cmp_bin .git/sidecar.before .git/index.csts && + + cp .git/index .git/private.index && + test-tool chmtime =-90 tracked && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status >.git/expected && + GIT_INDEX_FILE="$PWD/.git/private.index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCCCCCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/private.trace" \ + git status >.git/private.actual && + test_cmp .git/expected .git/private.actual && + test_region index do_write_index .git/private.trace && + ! test_trace2_data fsmonitor \ + history/own-write-source-recorded 1 <.git/private.trace && + ! test_trace2_data fsmonitor \ + history/own-write-source-adopted 1 <.git/private.trace && + test_cmp_bin .git/canonical.before .git/index && + test_cmp_bin .git/sidecar.before .git/index.csts + ) +' + +test_expect_success PERL_TEST_HELPERS \ + 'a provider reset reissues an otherwise current skipHash sidecar' ' + test_when_finished "rm -rf sidecar-receipt-provider-reset" && + test_create_repo sidecar-receipt-provider-reset && + ( + cd sidecar-receipt-provider-reset && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test-tool chmtime =-180 tracked && + git -c core.fsmonitor=false update-index --refresh && + git config index.version 4 && + git config index.skipHash true && + git config core.autocrlf false && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + git update-index --fsmonitor && + test_env GIT_INDEX_FILE="$PWD/.git/index" \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + test_grep FSUC .git/index && + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + bulk_status status --porcelain=v2 >.git/issued && + test_must_be_empty .git/issued && + test_path_is_file .git/index.csts && + rawsz=$(test_oid rawsz) && + dd if=/dev/zero of=.git/zero-trailer \ + bs="$rawsz" count=1 2>/dev/null && + tail -c "$rawsz" .git/index >.git/trailer && + test_cmp_bin .git/zero-trailer .git/trailer && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + -c core.trustctime=true \ + -c core.checkStat=default \ + status >.git/expected && + cp .git/index .git/index.before && + cp .git/index.csts .git/sidecar.before && + before_inode=$(/usr/bin/stat -f %i .git/index) && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/hit.trace" \ + git status >.git/hit && + test_cmp .git/expected .git/hit && + test_trace2_data status clean-proof/hit 1 <.git/hit.trace && + test_region ! index do_read_index .git/hit.trace && + test_region ! index do_write_index .git/hit.trace && + test_cmp_bin .git/index.before .git/index && + test_cmp_bin .git/sidecar.before .git/index.csts && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/reset.trace" \ + git -c core.preloadIndex=true \ + -c core.preloadIndexBulk=true \ + status >.git/actual && + test_cmp .git/expected .git/actual && + test_trace2_data status clean-proof/miss \ + fast-provider-changed <.git/reset.trace && + test_trace2_data status clean-proof/provider-reset-carried 1 \ + <.git/reset.trace && + test_region index do_write_index .git/reset.trace && + after_inode=$(/usr/bin/stat -f %i .git/index) && + test "$before_inode" != "$after_inode" && + tail -c "$rawsz" .git/index >.git/trailer && + test_cmp_bin .git/zero-trailer .git/trailer && + test_trace2_data fsmonitor \ + history/own-write-source-recorded 1 <.git/reset.trace && + test_trace2_data fsmonitor \ + history/own-write-source-adopted 1 <.git/reset.trace && + test_trace2_data status clean-proof/sidecar 1 \ + <.git/reset.trace && + test_trace2_data status clean-proof/postwrite-reissued 1 \ + <.git/reset.trace && + ! test_cmp_bin .git/sidecar.before .git/index.csts && + cp .git/index .git/index.after && + cp .git/index.csts .git/sidecar.after && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCCCCCC \ + GIT_TRACE2_EVENT="$PWD/.git/follower.trace" \ + git -c core.preloadIndex=true \ + -c core.preloadIndexBulk=true \ + status >.git/follower && + test_cmp .git/expected .git/follower && + test_trace2_data status clean-proof/hit 1 \ + <.git/follower.trace && + test_region ! index do_read_index .git/follower.trace && + test_region ! index do_write_index .git/follower.trace && + test_cmp_bin .git/index.after .git/index && + test_cmp_bin .git/sidecar.after .git/index.csts + ) +' + test_done From 880b232f8aeb50a6590dbc25508470e27b8f97f4 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 16:49:28 -0500 Subject: [PATCH 69/73] status: trace provider resets in the clean sidecar probe The clean sidecar reports fast-provider-changed for both a nonempty delta and an unavailable provider boundary. Unlike the ordinary index reader, it does not record when the latter was a TRIVIAL response. A trace stopped before the index refresh therefore cannot distinguish a real worktree change from a provider reset. Emit the existing query/trivial-response event as soon as the sidecar probe receives TRIVIAL. Reuse the parsed response without another IPC request, and keep errors and nonempty deltas out of this counter. The existing reset-propagation test checks all three outcomes, including a second TRIVIAL from the ordinary reader. --- clean-status-fast.c | 3 +++ t/t7530-status-clean-sidecar.sh | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/clean-status-fast.c b/clean-status-fast.c index 53f05ce1df41a8..0ec70fc62f92e2 100644 --- a/clean-status-fast.c +++ b/clean-status-fast.c @@ -284,6 +284,9 @@ int clean_status_try_sidecar( record.sidecar.token, record.sidecar.token_len); if (query_builtin_fsmonitor(query_token, &query) != FSMONITOR_QUERY_DELTA) { + if (query.outcome == FSMONITOR_QUERY_TRIVIAL) + trace2_data_intmax("fsm_client", NULL, + "query/trivial-response", 1); /* A later successful query cannot erase this lost boundary. */ *provider_reset = 1; trace_miss(repo, "fast-provider-changed"); diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index 937470b0262044..55a7dd7a3f38e6 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -2681,6 +2681,22 @@ test_expect_success PERL_TEST_HELPERS \ <".git/$outcome.trace" && test_trace2_data status clean-proof/provider-reset-carried 1 \ <".git/$outcome.trace" && + case "$outcome" in + E) + ! test_trace2_data fsm_client query/trivial-response 1 \ + <".git/$outcome.trace" + ;; + *) + test_trace2_data fsm_client query/trivial-response 1 \ + <".git/$outcome.trace" >.git/trivial-responses && + if test "$outcome" = TT + then + test_line_count = 2 .git/trivial-responses + else + test_line_count = 1 .git/trivial-responses + fi + ;; + esac && test_grep ! "\"key\":\"clean-proof/hit\"" \ ".git/$outcome.trace" && test_cmp_bin .git/index.before .git/index && @@ -2718,6 +2734,8 @@ test_expect_success PERL_TEST_HELPERS \ test_cmp .git/expect .git/actual && ! test_trace2_data status clean-proof/provider-reset-carried 1 \ <.git/delta.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/delta.trace && test_cmp_bin .git/index.before .git/index && GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCCCCCCC \ GIT_TRACE2_EVENT="$PWD/.git/writable.trace" \ From 798fb21ad82bdae76e56b7e72e51d62ae6b432e5 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 16:56:48 -0500 Subject: [PATCH 70/73] fsmonitor: require failed-cookie token retirement 2444e35250 (fsmonitor: retire the provider token after a failed cookie, 2026-08-17) prevents a failed synchronization cookie from leaving an old token usable. Its daemon still advertises the same capabilities and token prefix as earlier versions. A new client can therefore keep using an old daemon which lacks that guarantee. On macOS, the existing token-prefix shortcut also bypasses a separate capability check. Advertise cookie-token-retirement-v1 and mark every daemon token with cookie-v1. Require the marker in the actual query response, including an attested legacy response. Discard unmarked bytes before checking capabilities: a reply from a replacement daemon cannot authenticate a response from the preceding process. Reuse the existing bounded restart or retry path, and add no capability query to marked warm responses. Preserve the existing macOS prefix and raw, query-v1, and query-v2 interfaces so older clients can use the upgraded daemon. Update the deterministic token fixtures too; test tokens have no acceptance bypass. Add backend-only regressions for a fully capable pre-retirement daemon, an advertised capability with an unmarked response, and warm queries which do not repeat capability negotiation. --- builtin/fsmonitor--daemon.c | 12 +- fsmonitor-ipc.c | 49 +++++--- fsmonitor-ipc.h | 3 + t/helper/test-simple-ipc.c | 66 ++++++++-- t/meson.build | 1 + t/t1602-index-witness.sh | 7 +- t/t7527-builtin-fsmonitor.sh | 35 ++++-- t/t7537-fsmonitor-cookie-compat.sh | 189 +++++++++++++++++++++++++++++ 8 files changed, 321 insertions(+), 41 deletions(-) create mode 100755 t/t7537-fsmonitor-cookie-compat.sh diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index b52421f672c930..1c53a5af4dd6df 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -410,15 +410,18 @@ static struct fsmonitor_token_data *fsmonitor_new_token_data(void) if (test_env_value < 0) test_env_value = git_env_bool("GIT_TEST_FSMONITOR_TOKEN", 0); +#ifdef __APPLE__ + strbuf_addstr(&token->token_id, + FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX); +#endif + strbuf_addstr(&token->token_id, + FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_PREFIX); + if (!test_env_value) { struct timeval tv; struct tm tm; time_t secs; -#ifdef __APPLE__ - strbuf_addstr(&token->token_id, - FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX); -#endif gettimeofday(&tv, NULL); secs = tv.tv_sec; gmtime_r(&secs, &tm); @@ -754,6 +757,7 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, if (!strcmp(command, FSMONITOR_IPC_CAPABILITY_COMMAND)) { static const char capabilities[] = FSMONITOR_IPC_QUERY_VERSION "\n" + FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_CAPABILITY "\n" #ifdef __APPLE__ FSMONITOR_IPC_HARDLINK_QUERY_VERSION "\n" FSMONITOR_IPC_DIR_METADATA_CAPABILITY "\n" diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 799db433cf86f6..5ddd7503ca60f4 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -437,39 +437,42 @@ static int server_supports_bound_queries(void) static int server_supports_required_capabilities(void) { -#ifdef __APPLE__ struct strbuf answer = STRBUF_INIT; int ret; ret = !try_send_command(FSMONITOR_IPC_CAPABILITY_COMMAND, &answer, NULL, 1) && has_capability(&answer, FSMONITOR_IPC_QUERY_VERSION) && + has_capability(&answer, + FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_CAPABILITY); +#ifdef __APPLE__ + ret = ret && has_capability(&answer, FSMONITOR_IPC_HARDLINK_QUERY_VERSION) && has_capability(&answer, FSMONITOR_IPC_DIR_METADATA_CAPABILITY) && has_capability(&answer, FSMONITOR_IPC_HARDLINK_INODE_CAPABILITY); +#endif strbuf_release(&answer); return ret; -#else - return server_supports_bound_queries(); -#endif } -#ifdef __APPLE__ -static int query_identifies_filtered_daemon(const char *token, - const struct strbuf *answer) +static int response_identifies_cookie_retiring_daemon( + const struct strbuf *answer) { static const char prefix[] = - "builtin:" FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX; + "builtin:" +#ifdef __APPLE__ + FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX +#endif + FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_PREFIX; const char *end = memchr(answer->buf, '\0', answer->len); - return starts_with(token, prefix) && end && + return end && (size_t)(end - answer->buf) >= sizeof(prefix) - 1 && !memcmp(answer->buf, prefix, sizeof(prefix) - 1); } -#endif #if defined(__APPLE__) || defined(__linux__) static int legacy_peer_credentials( @@ -940,26 +943,32 @@ int fsmonitor_ipc__send_query(const char *since_token, trace2_data_intmax("fsm_client", NULL, "query/response-length", answer->len); -#ifdef __APPLE__ - if (!ret && !query_identifies_filtered_daemon(tok, answer) && - !server_supports_required_capabilities()) { + if (!ret && + !response_identifies_cookie_retiring_daemon(answer)) { + int compatible = server_supports_required_capabilities(); + + trace2_data_intmax("fsm_client", NULL, + "query/unmarked-response", 1); strbuf_reset(answer); ret = -1; if (lifecycle_attempts++ >= FSMONITOR_RESTART_ATTEMPTS || - restart_incompatible_daemon()) + (!compatible && restart_incompatible_daemon())) goto done; - options.wait_if_not_found = 1; + options.wait_if_not_found = !compatible; goto try_again; } -#endif if (!ret && is_trivial_response(answer) && !server_supports_bound_queries()) { if (!try_send_attested_legacy_query( tok, &identity, answer)) { - if (legacy_worktree_authenticated) - *legacy_worktree_authenticated = 1; - ret = 0; - goto done; + if (response_identifies_cookie_retiring_daemon(answer)) { + if (legacy_worktree_authenticated) + *legacy_worktree_authenticated = 1; + ret = 0; + goto done; + } + trace2_data_intmax("fsm_client", NULL, + "query/unmarked-response", 1); } /* * A daemon predating bound queries treats query-v1 as diff --git a/fsmonitor-ipc.h b/fsmonitor-ipc.h index 39d8d9cdb6c5f8..ba1c05eea5c065 100644 --- a/fsmonitor-ipc.h +++ b/fsmonitor-ipc.h @@ -16,6 +16,9 @@ struct repository; #define FSMONITOR_IPC_HARDLINK_INODE_CAPABILITY "hardlink-inode-v1" #define FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX \ FSMONITOR_IPC_DIR_METADATA_TOKEN_PREFIX "inode-v1." +#define FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_CAPABILITY \ + "cookie-token-retirement-v1" +#define FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_PREFIX "cookie-v1." #define FSMONITOR_IPC_WORKTREE_ID_HEX 64 /* Hash the canonical worktree root and its stable filesystem identity. */ diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index d1f2149740fbcc..a7e4750fc9be2a 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -3,6 +3,7 @@ */ #include "test-tool.h" +#include "fsmonitor-ipc.h" #include "gettext.h" #include "simple-ipc.h" #include "parse-options.h" @@ -162,6 +163,8 @@ static int my_app_data = 42; static int fsmonitor_legacy; static int fsmonitor_capability_superset; static int fsmonitor_pre_dir_metadata; +static int fsmonitor_pre_cookie_retirement; +static int fsmonitor_unmarked_response; static int fsmonitor_disconnect_first; static ipc_server_application_cb test_app_cb; @@ -172,35 +175,69 @@ static int app__fsmonitor_capability_superset( struct ipc_server_reply_data *reply_data) { static const char capability_command[] = "get-capabilities"; - static const char capabilities[] = "query-v1\nquery-v2\n" + static const char capabilities[] = + FSMONITOR_IPC_QUERY_VERSION "\n" + FSMONITOR_IPC_HARDLINK_QUERY_VERSION "\n" + FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_CAPABILITY "\n" #ifdef __APPLE__ - "dir-metadata-filter-v1\n" - "hardlink-inode-v1\n" + FSMONITOR_IPC_DIR_METADATA_CAPABILITY "\n" + FSMONITOR_IPC_HARDLINK_INODE_CAPABILITY "\n" #endif ; - static const char pre_dir_metadata_capabilities[] = "query-v1\n"; - static const char token[] = "builtin:test-capable:0"; + static const char pre_cookie_capabilities[] = + FSMONITOR_IPC_QUERY_VERSION "\n" +#ifdef __APPLE__ + FSMONITOR_IPC_HARDLINK_QUERY_VERSION "\n" + FSMONITOR_IPC_DIR_METADATA_CAPABILITY "\n" + FSMONITOR_IPC_HARDLINK_INODE_CAPABILITY "\n" +#endif + ; + static const char pre_dir_metadata_capabilities[] = + FSMONITOR_IPC_QUERY_VERSION "\n"; + static const char current_token[] = + "builtin:" +#ifdef __APPLE__ + FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX +#endif + FSMONITOR_IPC_COOKIE_TOKEN_RETIREMENT_PREFIX "test-capable:0"; + static const char old_token[] = + "builtin:" +#ifdef __APPLE__ + FSMONITOR_IPC_HARDLINK_INODE_TOKEN_PREFIX +#endif + "test-pre-cookie:0"; + const char *token; const char *query; - size_t query_len; + size_t token_len, query_len; int ret; + trace2_data_string("fsmonitor", NULL, "request", command); + if (command_len == sizeof(capability_command) - 1 && !memcmp(command, capability_command, command_len)) { if (fsmonitor_pre_dir_metadata) return reply_cb(reply_data, pre_dir_metadata_capabilities, sizeof(pre_dir_metadata_capabilities) - 1); + if (fsmonitor_pre_cookie_retirement) + return reply_cb(reply_data, + pre_cookie_capabilities, + sizeof(pre_cookie_capabilities) - 1); return reply_cb(reply_data, capabilities, sizeof(capabilities) - 1); } + token = fsmonitor_pre_dir_metadata || + fsmonitor_pre_cookie_retirement || fsmonitor_unmarked_response ? + old_token : current_token; + token_len = strlen(token); query = memchr(command, '\n', command_len); query_len = query ? command_len - (query + 1 - command) : 0; - ret = reply_cb(reply_data, token, sizeof(token)); + ret = reply_cb(reply_data, token, token_len + 1); if (!ret && ((!starts_with(command, "query-v1 ") && !starts_with(command, "query-v2 ")) || - query_len != sizeof(token) - 1 || + query_len != token_len || memcmp(query + 1, token, query_len))) ret = reply_cb(reply_data, "/", 2); return ret; @@ -251,7 +288,8 @@ static int test_app_cb(void *application_data, return SIMPLE_IPC_QUIT; } - if (fsmonitor_capability_superset || fsmonitor_pre_dir_metadata) + if (fsmonitor_capability_superset || fsmonitor_pre_dir_metadata || + fsmonitor_pre_cookie_retirement || fsmonitor_unmarked_response) return app__fsmonitor_capability_superset( command, command_len, reply_cb, reply_data); @@ -380,6 +418,10 @@ static int daemon__start_server(void) strvec_push(&cp.args, "--fsmonitor-capability-superset"); if (fsmonitor_pre_dir_metadata) strvec_push(&cp.args, "--fsmonitor-pre-dir-metadata"); + if (fsmonitor_pre_cookie_retirement) + strvec_push(&cp.args, "--fsmonitor-pre-cookie-retirement"); + if (fsmonitor_unmarked_response) + strvec_push(&cp.args, "--fsmonitor-unmarked-response"); if (fsmonitor_disconnect_first) strvec_push(&cp.args, "--fsmonitor-disconnect-first"); @@ -682,6 +724,12 @@ int cmd__simple_ipc(int argc, const char **argv) OPT_BOOL(0, "fsmonitor-pre-dir-metadata", &fsmonitor_pre_dir_metadata, N_("emulate a daemon without directory metadata filtering")), + OPT_BOOL(0, "fsmonitor-pre-cookie-retirement", + &fsmonitor_pre_cookie_retirement, + N_("emulate a daemon without failed-cookie token retirement")), + OPT_BOOL(0, "fsmonitor-unmarked-response", + &fsmonitor_unmarked_response, + N_("advertise token retirement but return an unmarked token")), OPT_BOOL(0, "fsmonitor-disconnect-first", &fsmonitor_disconnect_first, N_("disconnect while handling the first fsmonitor query")), diff --git a/t/meson.build b/t/meson.build index a0e542915d4d32..b5aa03844cbff8 100644 --- a/t/meson.build +++ b/t/meson.build @@ -970,6 +970,7 @@ integration_tests = [ 't7534-status-scoped-readers.sh', 't7535-fsmonitor-cookie-reset.sh', 't7536-fsmonitor-watch-limit-backoff.sh', + 't7537-fsmonitor-cookie-compat.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t1602-index-witness.sh b/t/t1602-index-witness.sh index 5b13f0c6657b62..88c8793bcf035f 100755 --- a/t/t1602-index-witness.sh +++ b/t/t1602-index-witness.sh @@ -613,7 +613,12 @@ test_lazy_prereq INDEX_WITNESS_SCRIPTED_IPC ' # This provider exists in the pre-fix runtime too. Its one stable token is # truthful only while the worktree is unchanged: make every worktree edit # before starting it, and keep all later fixture writes inside .git. -index_witness_scripted_token=builtin:test-capable:0 +if test_have_prereq MACOS +then + index_witness_scripted_token=builtin:dirmeta-v1.inode-v1.cookie-v1.test-capable:0 +else + index_witness_scripted_token=builtin:cookie-v1.test-capable:0 +fi test_index_witness_scripted_prepare () { sane_unset GIT_INDEX_FILE GIT_INDEX_VERSION \ diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index c10f1635ff5a98..04c188ac50132a 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -74,6 +74,14 @@ then test_done fi +if test_have_prereq MACOS +then + fsmonitor_pre_cookie_token_prefix=dirmeta-v1.inode-v1. +else + fsmonitor_pre_cookie_token_prefix= +fi +fsmonitor_cookie_token_prefix=${fsmonitor_pre_cookie_token_prefix}cookie-v1. + stop_daemon_delete_repo () { r=$1 && { maybe_timeout 30 git -C $r fsmonitor--daemon stop 2>/dev/null || :; } && @@ -630,13 +638,17 @@ test_expect_success 'flush cached data' ' # then a few (probably platform-specific number of) events in _1. # These should both have the same . - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_0 && + test-tool -C test_flush fsmonitor-client query \ + --token "builtin:${fsmonitor_cookie_token_prefix}test_00000001:0" \ + >actual_0 && nul_to_q actual_q0 && >test_flush/file_1 && >test_flush/file_2 && - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000001:0" >actual_1 && + test-tool -C test_flush fsmonitor-client query \ + --token "builtin:${fsmonitor_cookie_token_prefix}test_00000001:0" \ + >actual_1 && nul_to_q actual_q1 && test_grep "file_1" actual_q1 && @@ -647,16 +659,24 @@ test_expect_success 'flush cached data' ' test-tool -C test_flush fsmonitor-client flush >flush_0 && nul_to_q flush_q0 && - test_grep "^builtin:test_00000002:0Q/Q$" flush_q0 && + test_grep \ + "^builtin:${fsmonitor_cookie_token_prefix}test_00000002:0Q/Q$" \ + flush_q0 && - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_2 && + test-tool -C test_flush fsmonitor-client query \ + --token "builtin:${fsmonitor_cookie_token_prefix}test_00000002:0" \ + >actual_2 && nul_to_q actual_q2 && - test_grep "^builtin:test_00000002:0Q$" actual_q2 && + test_grep \ + "^builtin:${fsmonitor_cookie_token_prefix}test_00000002:0Q$" \ + actual_q2 && >test_flush/file_3 && - test-tool -C test_flush fsmonitor-client query --token "builtin:test_00000002:0" >actual_3 && + test-tool -C test_flush fsmonitor-client query \ + --token "builtin:${fsmonitor_cookie_token_prefix}test_00000002:0" \ + >actual_3 && nul_to_q actual_q3 && test_grep "file_3" actual_q3 @@ -2306,7 +2326,8 @@ test_expect_success 'bound query accepts a capability superset' ' GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ git status >.git/status.out && test_trace2_data fsm_client query/command \ - "builtin:test-capable:0" <.git/status.trace && + "builtin:${fsmonitor_cookie_token_prefix}test-capable:0" \ + <.git/status.trace && test_grep ! \ "\"key\":\"query/incompatible-daemon\"" \ .git/status.trace && diff --git a/t/t7537-fsmonitor-cookie-compat.sh b/t/t7537-fsmonitor-cookie-compat.sh new file mode 100755 index 00000000000000..29d1eca87ec641 --- /dev/null +++ b/t/t7537-fsmonitor-cookie-compat.sh @@ -0,0 +1,189 @@ +#!/bin/sh + +test_description='fsmonitor cookie-retirement daemon compatibility' + +. ./test-lib.sh + +if ! test_have_prereq FSMONITOR_DAEMON +then + skip_all='fsmonitor--daemon is not supported on this platform' + test_done +fi + +if test_have_prereq MACOS +then + fsmonitor_pre_cookie_token_prefix=dirmeta-v1.inode-v1. +else + fsmonitor_pre_cookie_token_prefix= +fi +fsmonitor_cookie_token_prefix=${fsmonitor_pre_cookie_token_prefix}cookie-v1. + +stop_cookie_compat_daemon () { + cookie_compat_repo=$1 && + test -d "$cookie_compat_repo/.git" || return 0 + cookie_compat_ipc=$( + git -C "$cookie_compat_repo" \ + rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc 2>/dev/null + ) || return 0 + test-tool simple-ipc stop-daemon \ + --name="$cookie_compat_ipc" --max-wait=5 \ + >/dev/null 2>&1 || : +} + +have_t2_data_event () { + grep -e '"event":"data".*"category":"'"$1"'".*"key":"'"$2"'"' +} + +# Unlike test_when_finished, these still stop our private daemons under -i. +test_atexit 'stop_cookie_compat_daemon cookie-retirement-upgrade' +test_atexit 'stop_cookie_compat_daemon cookie-retirement-unmarked' + +test_expect_success \ + 'a marked provider boundary replaces pre-retirement daemons once' ' + test_when_finished \ + "stop_cookie_compat_daemon cookie-retirement-upgrade" && + test_create_repo cookie-retirement-upgrade && + ( + cd cookie-retirement-upgrade && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.preloadIndex false && + git config core.untrackedCache true && + test_write_lines modified >tracked && + test_write_lines visible >visible && + GIT_OPTIONAL_LOCKS=0 \ + git -c core.fsmonitor=false \ + -c core.untrackedCache=false \ + status --porcelain=v2 >.git/expected && + test_grep "^1 \\.M .* tracked$" .git/expected && + test_grep "^? visible$" .git/expected && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --max-wait=10 \ + --fsmonitor-pre-cookie-retirement && + test-tool simple-ipc send --name="$ipc_path" \ + --token=get-capabilities >.git/old-capabilities && + test_grep "^query-v1$" .git/old-capabilities && + test_grep ! "^cookie-token-retirement-v1$" \ + .git/old-capabilities && + old_token="builtin:${fsmonitor_pre_cookie_token_prefix}test-pre-cookie:0" && + GIT_TRACE2_EVENT="$PWD/.git/upgrade.trace" \ + test-tool fsmonitor-client query \ + --token "$old_token" >.git/upgrade.raw && + nul_to_q <.git/upgrade.raw >.git/upgrade.response && + test_grep "^builtin:${fsmonitor_cookie_token_prefix}" \ + .git/upgrade.response && + test_trace2_data fsm_client query/command \ + "$old_token" <.git/upgrade.trace && + test_trace2_data fsm_client query/unmarked-response 1 \ + <.git/upgrade.trace && + test_trace2_data fsm_client query/incompatible-daemon 1 \ + <.git/upgrade.trace >.git/restarts && + test_line_count = 1 .git/restarts && + test_grep \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/upgrade.trace && + GIT_TRACE2_EVENT="$PWD/.git/prime.trace" \ + git status --porcelain=v2 >.git/prime && + test_cmp .git/expected .git/prime && + current_token=$(sed -n "1s/Q.*//p" .git/upgrade.response) && + worktree_identity=$( + sed -n \ + "s/.*\"category\":\"fsmonitor\",\"key\":\"request\",\"value\":\"query-v[12] \\([0-9a-f]*\\)\\\\n.*/\\1/p" \ + .git/upgrade.trace | + sed -n 1p + ) && + test ${#worktree_identity} = 64 && + for legacy_protocol in raw query-v1 query-v2 + do + if test "$legacy_protocol" = raw + then + legacy_command=$current_token + else + legacy_command=$(printf "%s %s\\n%s" \ + "$legacy_protocol" "$worktree_identity" \ + "$current_token") + fi && + test-tool simple-ipc send --name="$ipc_path" \ + --token="$legacy_command" \ + >".git/legacy-$legacy_protocol.response" && + test_grep "^builtin:${fsmonitor_cookie_token_prefix}" \ + ".git/legacy-$legacy_protocol.response" || return 1 + done && + test-tool simple-ipc stop-daemon \ + --name="$ipc_path" --max-wait=5 && + GIT_TRACE2_EVENT="$PWD/.git/marked-provider.trace" \ + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --max-wait=10 \ + --fsmonitor-capability-superset && + test-tool simple-ipc send --name="$ipc_path" \ + --token=get-capabilities >.git/marked-capabilities && + test_grep "^cookie-token-retirement-v1$" \ + .git/marked-capabilities && + test_trace2_data fsmonitor request get-capabilities \ + <.git/marked-provider.trace \ + >.git/capabilities.before && + test_line_count = 1 .git/capabilities.before && + marked_token="builtin:${fsmonitor_cookie_token_prefix}test-capable:0" && + printf "%s\\000" "$marked_token" >.git/warm.expected && + for warm_query in first repeated + do + GIT_TRACE2_EVENT="$PWD/.git/warm-$warm_query.trace" \ + test-tool fsmonitor-client query \ + --token "$marked_token" \ + >".git/warm-$warm_query.actual" && + test_cmp_bin .git/warm.expected \ + ".git/warm-$warm_query.actual" && + have_t2_data_event fsm_client query/response-length \ + <".git/warm-$warm_query.trace" && + ! test_trace2_data fsm_client query/unmarked-response 1 \ + <".git/warm-$warm_query.trace" && + ! test_trace2_data fsm_client query/incompatible-daemon 1 \ + <".git/warm-$warm_query.trace" || return 1 + done && + test_trace2_data fsmonitor request get-capabilities \ + <.git/marked-provider.trace \ + >.git/capabilities.after && + test_cmp .git/capabilities.before .git/capabilities.after + ) +' + +test_expect_success \ + 'an advertised capability never authenticates an unmarked response' ' + test_when_finished \ + "stop_cookie_compat_daemon cookie-retirement-unmarked" && + test_create_repo cookie-retirement-unmarked && + ( + cd cookie-retirement-unmarked && + test_commit base tracked && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --max-wait=10 \ + --fsmonitor-unmarked-response && + test-tool simple-ipc send --name="$ipc_path" \ + --token=get-capabilities >.git/capabilities && + test_grep "^cookie-token-retirement-v1$" \ + .git/capabilities && + old_token="builtin:${fsmonitor_pre_cookie_token_prefix}test-pre-cookie:0" && + test_must_fail env \ + GIT_TRACE2_EVENT="$PWD/.git/unmarked.trace" \ + test-tool fsmonitor-client query \ + --token "$old_token" \ + >.git/unmarked.raw \ + 2>.git/unmarked.err && + test_must_be_empty .git/unmarked.raw && + test_trace2_data fsm_client query/unmarked-response 1 \ + <.git/unmarked.trace >.git/rejected-responses && + test_line_count = 4 .git/rejected-responses && + ! test_trace2_data fsm_client query/incompatible-daemon 1 \ + <.git/unmarked.trace && + test-tool simple-ipc is-active --name="$ipc_path" + ) +' + +test_done From 68abca224f29ab5a5fd9863bea21d1fbc50b6fd8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 17:26:48 -0500 Subject: [PATCH 71/73] status: stop retrying consecutive provider resets A TRIVIAL closing query discards the current proof and starts another full refresh. The terminal retry cap prevents an unclosable last scan, but an unavailable provider still makes status repeat expensive tracked and untracked work before falling back to an ordinary refresh. Stop closing the token after two consecutive TRIVIAL responses. Share the counter between semantic and ordinary closure, and clear it when a query returns another result. A single transient reset can still be rescanned and accepted, and changed-path replies retain their existing three-query limit. Discard staged results and recheck attributes and the manifest before entering the existing conservative fallback. The regression compares dirty tracked, attributes, and untracked output with an independent fsmonitor-disabled status, and verifies that no FULL proof is issued. --- t/t7519-status-fsmonitor.sh | 8 ++++---- wt-status.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 5d01ca89c9d118..9454c11695077f 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -6663,7 +6663,7 @@ test_expect_success LINUX_SCOPED_HISTORY,UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORE ' test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ - 'exhausted provider resets skip an unclosable final index refresh' ' + 'repeated provider resets fall back before an unclosable rescan' ' test_when_finished "rm -rf builtin-closure-terminal-reset" && prepare_builtin_closure_repo builtin-closure-terminal-reset untracked && ( @@ -6689,12 +6689,12 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_cmp .git/expected .git/actual && test_trace2_data fsmonitor token_closure/trivial 1 \ <.git/status.trace >.git/trivial && - test_line_count = 3 .git/trivial && + test_line_count = 2 .git/trivial && test_trace2_data fsmonitor semantic/proof-epoch-captured 1 \ <.git/status.trace >.git/epochs && - test_line_count = 3 .git/epochs && + test_line_count = 2 .git/epochs && test_trace2_data status \ - fsmonitor_token/terminal-rescan-skipped 1 \ + fsmonitor_token/repeated-trivial-fallback 1 \ <.git/status.trace && test_trace2_data fsmonitor token_closure/rejected 1 \ <.git/status.trace && diff --git a/wt-status.c b/wt-status.c index e45cf6f9d2c4b1..061c75d51f1c67 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1746,6 +1746,7 @@ struct wt_status_token_closure { int staged_output_matches_status; int refresh_result; int queries; + int consecutive_trivial; }; static void wt_status_discard_staged_untracked( @@ -1846,6 +1847,19 @@ static int fsmonitor_token_requires_rescan(enum fsmonitor_token_result result) result == FSMONITOR_TOKEN_TRIVIAL; } +static enum fsmonitor_token_result wt_status_query_pending_token( + struct wt_status_token_closure *closure, int untracked_ready) +{ + enum fsmonitor_token_result result = fsmonitor_query_pending_token( + closure->status->repo->index, untracked_ready); + + if (result == FSMONITOR_TOKEN_TRIVIAL) + closure->consecutive_trivial++; + else + closure->consecutive_trivial = 0; + return result; +} + static void wt_status_release_attr_snapshot(struct wt_status *s); static int wt_status_attr_snapshot_matches(struct wt_status *s) @@ -1984,9 +1998,8 @@ static int wt_status_close_ordinary_fsmonitor_token( istate, scan_epoch)) break; closure->queries++; - result = fsmonitor_query_pending_token( - istate, - wt_status_untracked_cache_valid(closure)); + result = wt_status_query_pending_token( + closure, wt_status_untracked_cache_valid(closure)); if (result == FSMONITOR_TOKEN_CLEAN) { if (validate_epoch && !clean_status_proof_epoch_matches( @@ -2031,6 +2044,12 @@ static int wt_status_close_ordinary_fsmonitor_token( wt_status_reset_attr_snapshot_if_changed(s); if (wt_status_refresh_invalidated_manifest(s)) break; + if (closure->consecutive_trivial >= 2) { + trace2_data_intmax("status", s->repo, + "fsmonitor_token/repeated-trivial-fallback", + 1); + break; + } if (closure->queries >= FSMONITOR_TOKEN_MAX_QUERIES) { trace2_data_intmax("status", s->repo, "fsmonitor_token/terminal-rescan-skipped", @@ -2089,8 +2108,8 @@ wt_status_close_semantic_fsmonitor_token( /* The first query closes the tracked scan and its semantic proof. */ closure->queries++; - result = fsmonitor_query_pending_token( - istate, defer_untracked ? 0 : + result = wt_status_query_pending_token( + closure, defer_untracked ? 0 : wt_status_untracked_cache_valid(closure)); if (result != FSMONITOR_TOKEN_CLEAN) { wt_status_discard_semantic_verify( @@ -2136,9 +2155,8 @@ wt_status_close_semantic_fsmonitor_token( /* A second query closes the subsequent untracked scan. */ closure->queries++; clean_status_manifest_begin_directory_delta(istate, *proof); - result = fsmonitor_query_pending_token( - istate, - wt_status_untracked_cache_valid(closure)); + result = wt_status_query_pending_token( + closure, wt_status_untracked_cache_valid(closure)); directory_delta_reused = clean_status_manifest_end_directory_delta(istate); if (result != FSMONITOR_TOKEN_CLEAN) { From a8d82e3790880fdaa1842d1d2093eab6cc44754e Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Tue, 21 Jul 2026 17:04:56 -0400 Subject: [PATCH 72/73] fsmonitor: flush pending FSEvents before cookie wait 56cef9cb1a (fsmonitor: use pthread_cond_timedwait for cookie wait, 2026-04-15) limits the cookie wait to one second so that a filesystem which never delivers events cannot hang fsmonitor clients. A client that times out receives a trivial response and scans the entire index. FSEvents can defer delivery while it batches notifications and does not guarantee that its queue is drained in one latency interval. A loaded macOS system can therefore time out even though the event stream is working. On an Apple M4 Max (16 cores, 128 GiB RAM) running macOS 26.5.2, two worktrees with a 1,001,178-entry index timed out 484 of 545 and 297 of 365 fsmonitor requests. One status call performed 934,519 lstat() calls during a 47-second preload and took 52 seconds overall. Ask FSEvents to flush pending notifications after creating the cookie and before starting the timed wait. Use the asynchronous form because the client handler holds main_lock, which the listener callback also acquires. Keep the timeout and the behavior of the other backends unchanged. Signed-off-by: Tamir Duberstein Signed-off-by: Junio C Hamano (cherry picked from commit 08b12d90cde7a833221e132b1b2b7a9c724af234) --- builtin/fsmonitor--daemon.c | 3 +++ compat/fsmonitor/fsm-darwin-gcc.h | 1 + compat/fsmonitor/fsm-listen-darwin.c | 5 +++++ compat/fsmonitor/fsm-listen-linux.c | 4 ++++ compat/fsmonitor/fsm-listen-win32.c | 4 ++++ compat/fsmonitor/fsm-listen.h | 6 ++++++ 6 files changed, 23 insertions(+) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 1c53a5af4dd6df..cb4fdb7f64aae2 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -217,6 +217,9 @@ static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( close(fd); unlink(cookie_pathname.buf); + /* The listener callback takes main_lock, so this must not block. */ + fsm_listen__flush_async(state); + /* * Wait for the listener thread to observe the cookie file. * Time out after a short interval so that the client diff --git a/compat/fsmonitor/fsm-darwin-gcc.h b/compat/fsmonitor/fsm-darwin-gcc.h index 959bc88f8f765a..b749012c959ca8 100644 --- a/compat/fsmonitor/fsm-darwin-gcc.h +++ b/compat/fsmonitor/fsm-darwin-gcc.h @@ -97,6 +97,7 @@ CFRunLoopRef CFRunLoopGetCurrent(void); extern CFStringRef kCFRunLoopDefaultMode; void FSEventStreamSetDispatchQueue(FSEventStreamRef stream, dispatch_queue_t q); unsigned char FSEventStreamStart(FSEventStreamRef stream); +FSEventStreamEventId FSEventStreamFlushAsync(FSEventStreamRef stream); void FSEventStreamStop(FSEventStreamRef stream); void FSEventStreamInvalidate(FSEventStreamRef stream); void FSEventStreamRelease(FSEventStreamRef stream); diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index f25d7cdd907af9..b7d091453c2730 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -586,6 +586,11 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) pthread_mutex_unlock(&data->dq_lock); } +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state) +{ + FSEventStreamFlushAsync(state->listen_data->stream); +} + void fsm_listen__loop(struct fsmonitor_daemon_state *state) { struct fsm_listen_data *data; diff --git a/compat/fsmonitor/fsm-listen-linux.c b/compat/fsmonitor/fsm-listen-linux.c index 6181dcba51472d..ec46ef721c0031 100644 --- a/compat/fsmonitor/fsm-listen-linux.c +++ b/compat/fsmonitor/fsm-listen-linux.c @@ -481,6 +481,10 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) state->listen_data->shutdown = SHUTDOWN_STOP; } +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state UNUSED) +{ +} + /* * Process a single inotify event and queue for publication. */ diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 9a6efc9bea340b..039d7970004d46 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -290,6 +290,10 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) SetEvent(state->listen_data->hListener[LISTENER_SHUTDOWN]); } +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state UNUSED) +{ +} + static struct one_watch *create_watch(const char *path) { struct one_watch *watch = NULL; diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h index 41650bf8972217..cfeca1f4b63204 100644 --- a/compat/fsmonitor/fsm-listen.h +++ b/compat/fsmonitor/fsm-listen.h @@ -38,6 +38,12 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state); */ void fsm_listen__loop(struct fsmonitor_daemon_state *state); +/* + * Prompt the listener to deliver queued filesystem events, if supported. + * This does not wait for the events to be processed. + */ +void fsm_listen__flush_async(struct fsmonitor_daemon_state *state); + /* * Gently request that the fsmonitor listener thread shutdown. * It does not wait for it to stop. The caller should do a JOIN From f7be5b1d18f97c2b66256370b608f684641c80e1 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 17 Aug 2026 18:07:24 -0500 Subject: [PATCH 73/73] Revert "fsmonitor: flush pending FSEvents before cookie wait" a8d82e3790 (fsmonitor: flush pending FSEvents before cookie wait, 2026-07-21) backported a topic that upstream has since retracted. It also does not repair the workload motivating this backport: a matched 48-query test observes 12 timeouts in each 24-query arm, and status in a 1,160,465-entry macOS worktree still loses its closing cookie after one second. Remove the unqualified delivery prompt. Keep failed-cookie token retirement, response-bound capability checks, and the bounded status fallback unchanged. This reverts commit a8d82e3790880fdaa1842d1d2093eab6cc44754e. Link: https://github.com/git/git/blob/eb17606c718c93a69a64a8c3fd764899a68ba9f2/whats-cooking.txt#L1208-L1217 --- builtin/fsmonitor--daemon.c | 3 --- compat/fsmonitor/fsm-darwin-gcc.h | 1 - compat/fsmonitor/fsm-listen-darwin.c | 5 ----- compat/fsmonitor/fsm-listen-linux.c | 4 ---- compat/fsmonitor/fsm-listen-win32.c | 4 ---- compat/fsmonitor/fsm-listen.h | 6 ------ 6 files changed, 23 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index cb4fdb7f64aae2..1c53a5af4dd6df 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -217,9 +217,6 @@ static enum fsmonitor_cookie_item_result with_lock__wait_for_cookie( close(fd); unlink(cookie_pathname.buf); - /* The listener callback takes main_lock, so this must not block. */ - fsm_listen__flush_async(state); - /* * Wait for the listener thread to observe the cookie file. * Time out after a short interval so that the client diff --git a/compat/fsmonitor/fsm-darwin-gcc.h b/compat/fsmonitor/fsm-darwin-gcc.h index b749012c959ca8..959bc88f8f765a 100644 --- a/compat/fsmonitor/fsm-darwin-gcc.h +++ b/compat/fsmonitor/fsm-darwin-gcc.h @@ -97,7 +97,6 @@ CFRunLoopRef CFRunLoopGetCurrent(void); extern CFStringRef kCFRunLoopDefaultMode; void FSEventStreamSetDispatchQueue(FSEventStreamRef stream, dispatch_queue_t q); unsigned char FSEventStreamStart(FSEventStreamRef stream); -FSEventStreamEventId FSEventStreamFlushAsync(FSEventStreamRef stream); void FSEventStreamStop(FSEventStreamRef stream); void FSEventStreamInvalidate(FSEventStreamRef stream); void FSEventStreamRelease(FSEventStreamRef stream); diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index b7d091453c2730..f25d7cdd907af9 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -586,11 +586,6 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) pthread_mutex_unlock(&data->dq_lock); } -void fsm_listen__flush_async(struct fsmonitor_daemon_state *state) -{ - FSEventStreamFlushAsync(state->listen_data->stream); -} - void fsm_listen__loop(struct fsmonitor_daemon_state *state) { struct fsm_listen_data *data; diff --git a/compat/fsmonitor/fsm-listen-linux.c b/compat/fsmonitor/fsm-listen-linux.c index ec46ef721c0031..6181dcba51472d 100644 --- a/compat/fsmonitor/fsm-listen-linux.c +++ b/compat/fsmonitor/fsm-listen-linux.c @@ -481,10 +481,6 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) state->listen_data->shutdown = SHUTDOWN_STOP; } -void fsm_listen__flush_async(struct fsmonitor_daemon_state *state UNUSED) -{ -} - /* * Process a single inotify event and queue for publication. */ diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 039d7970004d46..9a6efc9bea340b 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -290,10 +290,6 @@ void fsm_listen__stop_async(struct fsmonitor_daemon_state *state) SetEvent(state->listen_data->hListener[LISTENER_SHUTDOWN]); } -void fsm_listen__flush_async(struct fsmonitor_daemon_state *state UNUSED) -{ -} - static struct one_watch *create_watch(const char *path) { struct one_watch *watch = NULL; diff --git a/compat/fsmonitor/fsm-listen.h b/compat/fsmonitor/fsm-listen.h index cfeca1f4b63204..41650bf8972217 100644 --- a/compat/fsmonitor/fsm-listen.h +++ b/compat/fsmonitor/fsm-listen.h @@ -38,12 +38,6 @@ void fsm_listen__dtor(struct fsmonitor_daemon_state *state); */ void fsm_listen__loop(struct fsmonitor_daemon_state *state); -/* - * Prompt the listener to deliver queued filesystem events, if supported. - * This does not wait for the events to be processed. - */ -void fsm_listen__flush_async(struct fsmonitor_daemon_state *state); - /* * Gently request that the fsmonitor listener thread shutdown. * It does not wait for it to stop. The caller should do a JOIN