fsmonitor: preserve proofs and avoid repeated recovery scans

Documentation/config/core.adoc

@@ -738,7 +738,10 @@ but may cost more than normal preload depending on filesystem and cache
 state. Inconclusive scans are discarded before continuing with the normal
 preload. Currently this is supported on APFS, ext-family filesystems, and
 XFS, and only has an effect when `core.preloadIndex` is enabled. Defaults
-to false.
+to false, except that a whole-worktree, read-only `git status` may use it
+after the native file system monitor discards a legacy
+untracked cache. Setting this option explicitly to false also disables
+that recovery optimization.
 
 core.unsetenvvars::
 	Windows-only: comma-separated list of environment variables'

Makefile

@@ -1579,6 +1579,7 @@ CLAR_TEST_SUITES += u-clean-status-history-store
 CLAR_TEST_SUITES += u-clean-status-identity
 CLAR_TEST_SUITES += u-clean-status-index
 CLAR_TEST_SUITES += u-clean-status-manifest
+CLAR_TEST_SUITES += u-clean-status-progress
 CLAR_TEST_SUITES += u-clean-status-sidecar
 CLAR_TEST_SUITES += u-clean-status-store
 CLAR_TEST_SUITES += u-ctype

apply.c

@@ -13,12 +13,14 @@
 #include "git-compat-util.h"
 #include "abspath.h"
 #include "base85.h"
+#include "clean-status.h"
 #include "config.h"
 #include "odb.h"
 #include "delta.h"
 #include "diff.h"
 #include "dir.h"
 #include "environment.h"
+#include "fsmonitor-settings.h"
 #include "gettext.h"
 #include "hex.h"
 #include "xdiff-interface.h"
@@ -31,6 +33,7 @@
 #include "path.h"
 #include "quote.h"
 #include "read-cache.h"
+#include "replace-object.h"
 #include "repository.h"
 #include "rerere.h"
 #include "apply.h"
@@ -4440,9 +4443,66 @@ static void patch_stats(struct apply_state *state, struct patch *patch)
 	}
 }
 
+static int patch_preserves_clean_history(struct apply_state *state,
+					 struct patch *patch)
+{
+	struct index_state *istate = state->repo->index;
+	const struct cache_entry *old;
+	int pos;
+
+	if (!state->update_index || state->ita_only || state->threeway ||
+	    state->apply_with_reject || state->fake_ancestor ||
+	    state->index_file || !fstat_is_reliable() ||
+	    getenv(INDEX_ENVIRONMENT) ||
+	    getenv(GIT_WORK_TREE_ENVIRONMENT) ||
+	    getenv(GIT_COMMON_DIR_ENVIRONMENT) ||
+	    getenv(DB_ENVIRONMENT) || getenv(ALTERNATE_DB_ENVIRONMENT) ||
+	    istate != istate->repo->index || istate->split_index ||
+	    istate->sparse_index != INDEX_EXPANDED ||
+	    repo_config_values(istate->repo)->apply_sparse_checkout ||
+	    !istate->repo->config_values_private_.trust_ctime ||
+	    !istate->repo->config_values_private_.check_stat ||
+	    fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC ||
+	    repo_has_replace_refs_uncached(istate->repo) ||
+	    patch->is_new > 0 || patch->is_delete > 0 || patch->is_copy ||
+	    patch->is_rename || patch->conflicted_threeway ||
+	    !patch->old_name || !patch->new_name ||
+	    strcmp(patch->old_name, patch->new_name) ||
+	    !S_ISREG(patch->old_mode) || !S_ISREG(patch->new_mode) ||
+	    create_ce_mode(patch->old_mode) !=
+		create_ce_mode(patch->new_mode) ||
+	    !clean_status_external_history_enabled(istate) ||
+	    !clean_status_has_persistent_fsmonitor_semantic_history(istate) ||
+	    !clean_status_revalidated_token_matches(istate) ||
+	    !istate->fsmonitor_token_valid ||
+	    !istate->fsmonitor_untracked_valid ||
+	    !istate->fsmonitor_untracked_extension_seen ||
+	    istate->fsmonitor_untracked_extension_invalid ||
+	    !istate->fsmonitor_last_update ||
+	    !istate->fsmonitor_untracked_token ||
+	    strcmp(istate->fsmonitor_last_update,
+		   istate->fsmonitor_untracked_token) ||
+	    !istate->untracked || !istate->untracked->use_fsmonitor ||
+	    !istate->untracked->root)
+		return 0;
+
+	pos = index_name_pos(istate, patch->old_name,
+			     strlen(patch->old_name));
+	if (pos < 0)
+		return 0;
+	old = istate->cache[pos];
+	return S_ISREG(old->ce_mode) &&
+		old->ce_mode == create_ce_mode(patch->new_mode) &&
+		clean_status_index_entry_is_semantically_safe(
+			istate, old, old);
+}
+
 static int remove_file(struct apply_state *state, struct patch *patch, int rmdir_empty)
 {
-	if (state->update_index && !state->ita_only) {
+	if (state->update_index && !state->ita_only &&
+	    !patch_preserves_clean_history(state, patch)) {
+		if (clean_status_external_history_enabled(state->repo->index))
+			clean_status_invalidate_current_proof(state->repo->index);
 		if (remove_file_from_index(state->repo->index, patch->old_name) < 0)
 			return error(_("unable to remove %s from index"), patch->old_name);
 	}
@@ -4455,6 +4515,7 @@ static int remove_file(struct apply_state *state, struct patch *patch, int rmdir
 }
 
 static int add_index_file(struct apply_state *state,
+			  struct patch *patch,
 			  const char *path,
 			  unsigned mode,
 			  void *buf,
@@ -4463,6 +4524,7 @@ static int add_index_file(struct apply_state *state,
 	struct stat st;
 	struct cache_entry *ce;
 	int namelen = strlen(path);
+	int options = ADD_CACHE_OK_TO_ADD;
 
 	ce = make_empty_cache_entry(state->repo->index, namelen);
 	memcpy(ce->name, path, namelen);
@@ -4497,7 +4559,13 @@ static int add_index_file(struct apply_state *state,
 				       "for newly created file %s"), path);
 		}
 	}
-	if (add_index_entry(state->repo->index, ce, ADD_CACHE_OK_TO_ADD) < 0) {
+	if (patch_preserves_clean_history(state, patch)) {
+		options |= ADD_CACHE_OK_TO_REPLACE |
+			ADD_CACHE_PRESERVE_CLEAN_HISTORY;
+	} else if (clean_status_external_history_enabled(state->repo->index)) {
+		clean_status_invalidate_current_proof(state->repo->index);
+	}
+	if (add_index_entry(state->repo->index, ce, options) < 0) {
 		discard_cache_entry(ce);
 		return error(_("unable to add cache entry for %s"), path);
 	}
@@ -4697,7 +4765,7 @@ static int create_file(struct apply_state *state, struct patch *patch)
 	if (patch->conflicted_threeway)
 		return add_conflicted_stages_file(state, patch);
 	else if (state->check_index || (state->ita_only && patch->is_new > 0))
-		return add_index_file(state, path, mode, buf, size);
+		return add_index_file(state, patch, path, mode, buf, size);
 	return 0;
 }
 
@@ -4828,6 +4896,18 @@ static int write_out_results(struct apply_state *state, struct patch *list)
 	struct patch *l;
 	struct string_list cpath = STRING_LIST_INIT_DUP;
 
+	if (state->update_index &&
+	    clean_status_external_history_enabled(state->repo->index)) {
+		for (l = list; l; l = l->next) {
+			if (l->rejected ||
+			    !patch_preserves_clean_history(state, l)) {
+				clean_status_invalidate_current_proof(
+					state->repo->index);
+				break;
+			}
+		}
+	}
+
 	for (phase = 0; phase < 2; phase++) {
 		l = list;
 		while (l) {

builtin/am.c

@@ -9,9 +9,12 @@
 #include "builtin.h"
 #include "abspath.h"
 #include "advice.h"
+#include "clean-status-config.h"
+#include "clean-status.h"
 #include "config.h"
 #include "editor.h"
 #include "environment.h"
+#include "fsmonitor-settings.h"
 #include "gettext.h"
 #include "hex.h"
 #include "parse-options.h"
@@ -2315,6 +2318,7 @@ int cmd_am(int argc,
 	   struct repository *repo UNUSED)
 {
 	struct am_state state;
+	struct clean_status_config_digest clean_digest;
 	int binary = -1;
 	int keep_cr = -1;
 	int patch_format = PATCH_FORMAT_UNKNOWN;
@@ -2464,6 +2468,13 @@ int cmd_am(int argc,
 	/* Ensure a valid committer ident can be constructed */
 	git_committer_info(IDENT_STRICT);
 
+	if (fstat_is_reliable() && !getenv(INDEX_ENVIRONMENT) &&
+	    fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC &&
+	    !clean_status_config_read_repository(the_repository, &clean_digest)) {
+		clean_status_enable_external_history(the_repository);
+		clean_status_set_config_digest(the_repository, &clean_digest);
+	}
+
 	if (repo_read_index_preload(the_repository, NULL, 0) < 0)
 		die(_("failed to read the index"));
 

builtin/apply.c

@@ -1,7 +1,12 @@
 #define USE_THE_REPOSITORY_VARIABLE
 #include "builtin.h"
+#include "clean-status-config.h"
+#include "clean-status.h"
+#include "environment.h"
+#include "fsmonitor-settings.h"
 #include "gettext.h"
 #include "hash.h"
+#include "replace-object.h"
 #include "apply.h"
 
 static const char * const apply_usage[] = {
@@ -17,6 +22,7 @@ int cmd_apply(int argc,
 	int force_apply = 0;
 	int options = 0;
 	int ret;
+	struct clean_status_config_digest clean_digest;
 	struct apply_state state;
 
 	if (init_apply_state(&state, the_repository, prefix))
@@ -43,6 +49,25 @@ int cmd_apply(int argc,
 	if (check_apply_state(&state, force_apply))
 		exit(128);
 
+	if (state.apply && state.check_index && !state.threeway &&
+	    !state.apply_with_reject && !state.ita_only &&
+	    !state.fake_ancestor && !state.index_file &&
+	    !getenv(INDEX_ENVIRONMENT) &&
+	    !getenv(GIT_WORK_TREE_ENVIRONMENT) &&
+	    !getenv(GIT_COMMON_DIR_ENVIRONMENT) &&
+	    !getenv(DB_ENVIRONMENT) &&
+	    !getenv(ALTERNATE_DB_ENVIRONMENT) && fstat_is_reliable() &&
+	    !repo_config_values(the_repository)->apply_sparse_checkout &&
+	    the_repository->config_values_private_.trust_ctime &&
+	    the_repository->config_values_private_.check_stat &&
+	    fsm_settings__get_mode(the_repository) == FSMONITOR_MODE_IPC &&
+	    !repo_has_replace_refs_uncached(the_repository) &&
+	    !clean_status_config_read_repository(the_repository,
+					       &clean_digest)) {
+		clean_status_enable_external_history(the_repository);
+		clean_status_set_config_digest(the_repository, &clean_digest);
+	}
+
 	ret = apply_all_patches(&state, argc, argv, options);
 
 	clear_apply_state(&state);

builtin/check-attr.c

@@ -3,6 +3,7 @@
 #include "config.h"
 #include "attr.h"
 #include "environment.h"
+#include "fsmonitor.h"
 #include "gettext.h"
 #include "object-name.h"
 #include "quote.h"
@@ -115,6 +116,7 @@ int cmd_check_attr(int argc,
 	struct attr_check *check;
 	struct object_id initialized_oid;
 	int cnt, i, doubledash, filei;
+	int scoped_bootstrap = 0;
 
 	if (!is_bare_repository(the_repository))
 		setup_work_tree(the_repository);
@@ -127,13 +129,6 @@ int cmd_check_attr(int argc,
 	prepare_repo_settings(the_repository);
 	the_repository->settings.command_requires_full_index = 0;
 
-	if (repo_read_index(the_repository) < 0) {
-		die("invalid cache");
-	}
-
-	if (cached_attrs)
-		git_attr_set_direction(GIT_ATTR_INDEX);
-
 	doubledash = -1;
 	for (i = 0; doubledash < 0 && i < argc; i++) {
 		if (!strcmp(argv[i], "--"))
@@ -176,6 +171,18 @@ int cmd_check_attr(int argc,
 			error_with_usage("No file specified");
 	}
 
+	scoped_bootstrap = !stdin_paths && !source &&
+		argc - filei > 0 && argc - filei <= 64;
+	if (scoped_bootstrap)
+		fsmonitor_begin_scoped_bootstrap(the_repository->index);
+	if (repo_read_index(the_repository) < 0)
+		die("invalid cache");
+	if (scoped_bootstrap)
+		fsmonitor_end_scoped_bootstrap(the_repository->index);
+
+	if (cached_attrs)
+		git_attr_set_direction(GIT_ATTR_INDEX);
+
 	check = attr_check_alloc();
 	if (!all_attrs) {
 		for (i = 0; i < cnt; i++) {

builtin/checkout-index.c

@@ -12,6 +12,7 @@
 #include "clean-status-config.h"
 #include "config.h"
 #include "environment.h"
+#include "fsmonitor-settings.h"
 #include "gettext.h"
 #include "hook.h"
 #include "lockfile.h"
@@ -277,13 +278,8 @@ int cmd_checkout_index(int argc,
 	prepare_repo_settings(repo);
 	repo->settings.command_requires_full_index = 0;
 
-	if (repo_read_index(repo) < 0) {
-		die("invalid cache");
-	}
-
 	argc = parse_options(argc, argv, prefix, builtin_checkout_index_options,
 			builtin_checkout_index_usage, 0);
-	state.istate = repo->index;
 	state.force = force;
 	state.quiet = quiet;
 	state.not_new = not_new;
@@ -298,6 +294,15 @@ int cmd_checkout_index(int argc,
 		die(_("options '%s' and '%s' cannot be used together"),
 		    "--stage=all", "--no-temp");
 
+	if (index_opt && !state.base_dir_len && !to_tempfile &&
+	    !checkout_stage && !getenv(INDEX_ENVIRONMENT) &&
+	    fstat_is_reliable() &&
+	    fsm_settings__get_mode(repo) == FSMONITOR_MODE_IPC)
+		clean_status_enable_external_history(repo);
+	if (repo_read_index(repo) < 0)
+		die("invalid cache");
+	state.istate = repo->index;
+
 	/*
 	 * when --prefix is specified we do not want to update cache.
 	 */